Add native self-hosted instance connection to fluxer_desktop
Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 + tools/ci) with a "Connect to a Different Server" menu item and popout that lets the desktop app switch to any self-hosted Fluxer instance, plus fixes for well-known discovery on single-domain self-hosted deployments and a false-positive ERR_ABORTED on same-origin client redirects during the switch. Defaults to chat.fluxr.chat and uses an isolated userData directory from the official build.
This commit is contained in:
+6092
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,135 @@
|
||||
[package]
|
||||
name = "fluxer_webrtc_sender"
|
||||
version = "0.0.0"
|
||||
edition = "2024"
|
||||
license = "AGPL-3.0-or-later"
|
||||
publish = false
|
||||
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
exclude = [
|
||||
"vendor/tract-linalg-0.19.16",
|
||||
"vendor/tract-linalg-0.23.1",
|
||||
]
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib", "rlib"]
|
||||
|
||||
[features]
|
||||
# `publisher` pulls the LiveKit native SDK (libwebrtc). It is on by default for
|
||||
# the real addon build, but `cargo test --no-default-features` builds only the
|
||||
# pure YUV-conversion core so the unit tests run anywhere (no libwebrtc).
|
||||
default = ["publisher"]
|
||||
publisher = ["dep:livekit", "dep:tokio", "dep:futures-util"]
|
||||
camera-native = ["dep:nokhwa"]
|
||||
# Exposes pub re-exports of private camera-background hot-path internals so the
|
||||
# criterion bench target can call them; never enabled for the shipped addon.
|
||||
bench-internals = []
|
||||
|
||||
[dependencies]
|
||||
napi = {version = "3.9.1", default-features = false, features = ["dyn-symbols", "napi8", "tokio_rt"]}
|
||||
napi-derive = "3.5.6"
|
||||
crossbeam-queue = "0.3"
|
||||
|
||||
# LiveKit native SDK (wraps libwebrtc). Optional / feature-gated. The exact
|
||||
# version is resolved on the Windows build host (where the libwebrtc prebuilt
|
||||
# links); pinned loosely here.
|
||||
# Keep LiveKit's default features (default = ["tokio"] -> the runtime +
|
||||
# signal-client-tokio) AND add a TLS backend for the signal WebSocket -- without
|
||||
# one, connecting to a wss:// SFU fails with "TLS support not compiled in".
|
||||
# rustls-tls-native-roots = pure-Rust TLS using the OS root store (no OpenSSL).
|
||||
livekit = {version = "0.7", optional = true, features = ["rustls-tls-native-roots"]}
|
||||
tokio = {version = "1", optional = true, features = ["rt-multi-thread", "sync", "macros", "time"]}
|
||||
parking_lot = "0.12"
|
||||
fluxer_screen_frame_bus = { path = "../screen-frame-bus" }
|
||||
futures-core = "0.3"
|
||||
# StreamExt::next() for draining inbound NativeAudioStream/NativeVideoStream.
|
||||
futures-util = {version = "0.3", optional = true}
|
||||
libloading = "0.9.0"
|
||||
fluxer_desktop_native = { path = "../rust" }
|
||||
image = {version = "0.25.10", default-features = false, features = ["jpeg", "png", "webp", "gif"]}
|
||||
# System-camera capture for the native camera-publish path. Optional /
|
||||
# feature-gated for the shipped desktop builds. `input-native` maps to V4L2 on
|
||||
# Linux, AVFoundation on macOS, and Media Foundation on Windows.
|
||||
# BUILD-HOST NOTE: `decoding` pulls `mozjpeg-sys`, which statically builds
|
||||
# libjpeg and -- with its default `nasm_simd` -- needs NASM + a C compiler on the
|
||||
# build host.
|
||||
nokhwa = {version = "0.10", optional = true, default-features = false, features = ["input-native", "decoding"]}
|
||||
|
||||
# Person segmentation for camera background blur / replacement on platforms
|
||||
# without a native segmentation API (Windows, Linux) and as the macOS fallback
|
||||
# when Vision is unavailable. tract is a pure-Rust ONNX inference engine: no
|
||||
# C++ runtime to link or ship per target, deterministic results, and the unit
|
||||
# tests exercise the real model on any build host. It runs the bundled Apache-2.0
|
||||
# MediaPipe Selfie Segmenter landscape model (models/, ~450 KB, embedded into the
|
||||
# addon) in ~10 ms per 256x144 frame on a laptop core; an inference-cadence
|
||||
# governor halves the rate on machines that miss the frame budget. Cost: ~40
|
||||
# transitive crates, all pure Rust, compile-time only impact elsewhere.
|
||||
tract-onnx = "0.23"
|
||||
|
||||
# DeepFilterNet3 noise suppression for the native device-microphone publish
|
||||
# path, replacing the WebRTC-APM fallback when the renderer requests the
|
||||
# deepFilter profile (parity with the web build's deepfilternet3-noise-filter
|
||||
# WASM chain). The crates.io release (0.2.5, 2022) predates DFN3, so this pins
|
||||
# the upstream git repo at the v0.5.6 release tag -- the combination the shipped
|
||||
# deep-filter binaries were built from. `tract` + `default-model` run the
|
||||
# bundled DFN3 ONNX models (~2 MB, embedded into the addon) on the same
|
||||
# pure-Rust tract inference engine already used for person segmentation, albeit
|
||||
# at tract 0.19 (deep_filter's tested series) alongside our 0.23 tree:
|
||||
# compile-time cost only, no new native or C++ dependencies.
|
||||
deep_filter = {git = "https://github.com/Rikorose/DeepFilterNet", rev = "978576aa8400552a4ce9730838c635aa30db5e61", default-features = false, features = ["tract", "default-model"]}
|
||||
# deep_filter declares tract ^0.19.4, but newer releases in that semver range
|
||||
# regress DFN3 model codegen (0.21.4 fails with "duplicate name .../Conv.bias"
|
||||
# on the transposed convolutions). These renamed dependency keys exist solely
|
||||
# to pin deep_filter's tract subtree to 0.19.16, the version in upstream's own
|
||||
# v0.5.6 lockfile; tract's internal `=` pins drag tract-core/hir/data/linalg
|
||||
# along. Never import these directly: the 0.23 `tract-onnx` entry above is the
|
||||
# one the rest of this crate uses.
|
||||
tract-onnx-deepfilter-pin = {package = "tract-onnx", version = "=0.19.16"}
|
||||
tract-pulse-deepfilter-pin = {package = "tract-pulse", version = "=0.19.16"}
|
||||
# Pinned to the 0.15 family to match deep_filter's public API surface
|
||||
# (DfTract::process takes ndarray 0.15 ArrayView2 arguments).
|
||||
ndarray = "0.15"
|
||||
|
||||
# macOS person segmentation for camera background blur / replacement. Vision's
|
||||
# VNGeneratePersonSegmentationRequest produces the per-pixel person mask that the
|
||||
# camera background transform composites with; without it the transform falls
|
||||
# back to a fixed portrait ellipse. Pinned to the objc2 0.6 / framework 0.3
|
||||
# family already used across fluxer_desktop/native, so no new dependency tree.
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
objc2 = "0.6"
|
||||
objc2-foundation = {version = "0.3", features = ["NSArray", "NSDictionary", "NSError", "NSObject", "NSString"]}
|
||||
objc2-vision = {version = "0.3", default-features = false, features = ["std", "VNRequest", "VNStatefulRequest", "VNGeneratePersonSegmentationRequest", "VNRequestHandler", "VNObservation", "VNTypes", "objc2-core-video", "objc2-core-foundation"]}
|
||||
objc2-core-video = {version = "0.3", features = ["CVPixelBuffer", "CVImageBuffer", "CVBuffer", "CVReturn", "CVBase"]}
|
||||
objc2-core-foundation = {version = "0.3", features = ["CFDictionary", "CFBase"]}
|
||||
|
||||
[build-dependencies]
|
||||
napi-build = "2.3.2"
|
||||
|
||||
[dev-dependencies]
|
||||
# Test-only (never linked into the shipped addon): parses the committed golden
|
||||
# event fixtures so the renderer's (eventType, jsonPayload) parser contract is
|
||||
# regression-locked offline, without a live SFU or a second identity.
|
||||
serde_json = "1"
|
||||
criterion = {version = "0.8", default-features = false, features = ["cargo_bench_support", "html_reports"]}
|
||||
|
||||
[[bench]]
|
||||
name = "frame_bus"
|
||||
harness = false
|
||||
required-features = []
|
||||
|
||||
[[bench]]
|
||||
name = "camera_background"
|
||||
harness = false
|
||||
required-features = ["bench-internals"]
|
||||
|
||||
[[bench]]
|
||||
name = "deep_filter"
|
||||
harness = false
|
||||
required-features = ["bench-internals"]
|
||||
|
||||
[patch.crates-io]
|
||||
libwebrtc = { path = "vendor/libwebrtc" }
|
||||
webrtc-sys = { path = "vendor/webrtc-sys" }
|
||||
tract-linalg-019 = { package = "tract-linalg", path = "vendor/tract-linalg-0.19.16" }
|
||||
tract-linalg-023 = { package = "tract-linalg", path = "vendor/tract-linalg-0.23.1" }
|
||||
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"measured_at": "85e057a273fd",
|
||||
"host": "darwin-arm64-apple-silicon",
|
||||
"regression_budget_percent": 5.0,
|
||||
"criterion_args": {
|
||||
"warm_up_time_sec": 2,
|
||||
"measurement_time_sec": 5
|
||||
},
|
||||
"benches": {
|
||||
"frame_bus::get_sink": {
|
||||
"median_ns": 13.292,
|
||||
"low_ns": 13.189,
|
||||
"high_ns": 13.408,
|
||||
"budget_percent_override": 10.0,
|
||||
"note": "~13ns op; noise floor reasoning."
|
||||
},
|
||||
"frame_bus::get_sink_miss": {
|
||||
"median_ns": 3.9658,
|
||||
"low_ns": 3.9444,
|
||||
"high_ns": 3.9918,
|
||||
"budget_percent_override": 15.0,
|
||||
"note": "~4ns op; below the practical resolution of repeated criterion runs."
|
||||
},
|
||||
"frame_bus::native_handle_enqueue/nv12_320x180": {
|
||||
"median_ns": 1.808,
|
||||
"low_ns": 1.7961,
|
||||
"high_ns": 1.8205,
|
||||
"budget_percent_override": 15.0,
|
||||
"note": "~2ns op; below the practical resolution of repeated criterion runs."
|
||||
},
|
||||
"frame_bus::native_handle_enqueue/nv12_1920x1080": {
|
||||
"median_ns": 1.8086,
|
||||
"low_ns": 1.8008,
|
||||
"high_ns": 1.8171,
|
||||
"budget_percent_override": 15.0,
|
||||
"note": "~2ns op; below the practical resolution of repeated criterion runs."
|
||||
},
|
||||
"frame_bus::native_handle_enqueue/nv12_3840x2160": {
|
||||
"median_ns": 1.8141,
|
||||
"low_ns": 1.7964,
|
||||
"high_ns": 1.8333,
|
||||
"budget_percent_override": 15.0,
|
||||
"note": "~2ns op; below the practical resolution of repeated criterion runs."
|
||||
},
|
||||
"frame_bus::native_handle_enqueue/bgra_320x180": {
|
||||
"median_ns": 1.8261,
|
||||
"low_ns": 1.8172,
|
||||
"high_ns": 1.8361,
|
||||
"budget_percent_override": 15.0,
|
||||
"note": "~2ns op; below the practical resolution of repeated criterion runs."
|
||||
},
|
||||
"frame_bus::native_handle_enqueue/bgra_1920x1080": {
|
||||
"median_ns": 1.7952,
|
||||
"low_ns": 1.7877,
|
||||
"high_ns": 1.8027,
|
||||
"budget_percent_override": 15.0,
|
||||
"note": "~2ns op; below the practical resolution of repeated criterion runs."
|
||||
},
|
||||
"frame_bus::enqueue_discard/nv12_320x180": {
|
||||
"median_ns": 1391.4,
|
||||
"low_ns": 1342.4,
|
||||
"high_ns": 1455.8,
|
||||
"budget_percent_override": 20.0,
|
||||
"note": "Allocator-bound: bench does vec![0; total] per iter. Run-to-run sigma observed ~15%; budget set above that."
|
||||
},
|
||||
"frame_bus::enqueue_discard/nv12_1920x1080": {
|
||||
"median_ns": 29502.0,
|
||||
"low_ns": 28397.0,
|
||||
"high_ns": 30819.0,
|
||||
"budget_percent_override": 20.0,
|
||||
"note": "Allocator-bound; see nv12_320x180 note."
|
||||
},
|
||||
"frame_bus::enqueue_discard/nv12_3840x2160": {
|
||||
"median_ns": 121900.0,
|
||||
"low_ns": 114880.0,
|
||||
"high_ns": 129680.0,
|
||||
"budget_percent_override": 25.0,
|
||||
"note": "Largest 4K allocator-bound bench; widest budget."
|
||||
},
|
||||
"frame_bus::enqueue_discard/bgra_320x180": {
|
||||
"median_ns": 2807.2,
|
||||
"low_ns": 2695.1,
|
||||
"high_ns": 2937.6,
|
||||
"budget_percent_override": 20.0,
|
||||
"note": "Allocator-bound; see nv12_320x180 note."
|
||||
},
|
||||
"frame_bus::enqueue_discard/bgra_1920x1080": {
|
||||
"median_ns": 83692.0,
|
||||
"low_ns": 78328.0,
|
||||
"high_ns": 90200.0,
|
||||
"budget_percent_override": 25.0,
|
||||
"note": "Allocator-bound 1080p BGRA bench; widest budget."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
|
||||
use fluxer_webrtc_sender::bench_internals::{
|
||||
BlurScratch, MaskRefiner, blur_plane_masked, composite_masked_plane,
|
||||
};
|
||||
use std::hint::black_box;
|
||||
|
||||
const BENCH_WIDTH: usize = 1280;
|
||||
const BENCH_HEIGHT: usize = 720;
|
||||
const BENCH_BLUR_RADIUS_PASS: usize = 6;
|
||||
const BENCH_MASK_FEATHER_PX: usize = 64;
|
||||
|
||||
fn synth_plane(seed: usize) -> Vec<u8> {
|
||||
let mut plane = vec![0u8; BENCH_WIDTH * BENCH_HEIGHT];
|
||||
for (index, value) in plane.iter_mut().enumerate() {
|
||||
*value = ((index * 31 + seed) % 251) as u8;
|
||||
}
|
||||
plane
|
||||
}
|
||||
|
||||
fn synth_person_mask() -> Vec<u8> {
|
||||
let mut mask = vec![0u8; BENCH_WIDTH * BENCH_HEIGHT];
|
||||
let person_edge = BENCH_WIDTH / 2;
|
||||
for row in mask.chunks_exact_mut(BENCH_WIDTH) {
|
||||
for (x, value) in row.iter_mut().enumerate() {
|
||||
*value = if x < person_edge {
|
||||
255
|
||||
} else if x < person_edge + BENCH_MASK_FEATHER_PX {
|
||||
(255 - (x - person_edge) * 255 / BENCH_MASK_FEATHER_PX) as u8
|
||||
} else {
|
||||
0
|
||||
};
|
||||
}
|
||||
}
|
||||
mask
|
||||
}
|
||||
|
||||
fn bench_blur_plane_masked(c: &mut Criterion) {
|
||||
let source = synth_plane(7);
|
||||
let mask = synth_person_mask();
|
||||
let mut plane = source.clone();
|
||||
let mut scratch = BlurScratch::new(BENCH_WIDTH, BENCH_HEIGHT);
|
||||
|
||||
let mut group = c.benchmark_group("camera_background::blur_plane_masked");
|
||||
group.throughput(Throughput::Bytes((BENCH_WIDTH * BENCH_HEIGHT) as u64));
|
||||
group.bench_function("1280x720_radius_pass6", |b| {
|
||||
b.iter(|| {
|
||||
plane.copy_from_slice(&source);
|
||||
blur_plane_masked(
|
||||
black_box(&mut plane),
|
||||
BENCH_WIDTH,
|
||||
BENCH_HEIGHT,
|
||||
black_box(&mask),
|
||||
BENCH_BLUR_RADIUS_PASS,
|
||||
&mut scratch,
|
||||
);
|
||||
black_box(&plane);
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_mask_refine(c: &mut Criterion) {
|
||||
let luma = synth_plane(13);
|
||||
let raw_mask = synth_person_mask();
|
||||
let mut mask = raw_mask.clone();
|
||||
let mut refiner = MaskRefiner::new(BENCH_WIDTH, BENCH_HEIGHT);
|
||||
|
||||
let mut group = c.benchmark_group("camera_background::mask_refine");
|
||||
group.throughput(Throughput::Bytes((BENCH_WIDTH * BENCH_HEIGHT) as u64));
|
||||
group.bench_function("1280x720_guided_refine", |b| {
|
||||
b.iter(|| {
|
||||
mask.copy_from_slice(&raw_mask);
|
||||
refiner.refine(black_box(&luma), black_box(&mut mask));
|
||||
black_box(&mask);
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_composite_masked_plane(c: &mut Criterion) {
|
||||
let source = synth_plane(11);
|
||||
let background = synth_plane(151);
|
||||
let mask = synth_person_mask();
|
||||
let mut plane = source.clone();
|
||||
|
||||
let mut group = c.benchmark_group("camera_background::composite_masked_plane");
|
||||
group.throughput(Throughput::Bytes((BENCH_WIDTH * BENCH_HEIGHT) as u64));
|
||||
group.bench_function("1280x720_feathered_mask", |b| {
|
||||
b.iter(|| {
|
||||
plane.copy_from_slice(&source);
|
||||
composite_masked_plane(
|
||||
black_box(&mut plane),
|
||||
black_box(&background),
|
||||
BENCH_WIDTH,
|
||||
BENCH_HEIGHT,
|
||||
&mask,
|
||||
);
|
||||
black_box(&plane);
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_blur_plane_masked,
|
||||
bench_mask_refine,
|
||||
bench_composite_masked_plane
|
||||
);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,44 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
|
||||
use fluxer_webrtc_sender::bench_internals::{
|
||||
DEEP_FILTER_FRAME_SAMPLES, DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX, DeepFilterProcessor,
|
||||
};
|
||||
use std::hint::black_box;
|
||||
|
||||
fn next_noise_sample(seed: &mut u32) -> i16 {
|
||||
*seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
((*seed >> 16) as u16 as i16) / 4
|
||||
}
|
||||
|
||||
fn noise_frame(seed: &mut u32) -> Vec<i16> {
|
||||
let mut frame = vec![0i16; DEEP_FILTER_FRAME_SAMPLES];
|
||||
for sample in frame.iter_mut() {
|
||||
*sample = next_noise_sample(seed);
|
||||
}
|
||||
frame
|
||||
}
|
||||
|
||||
fn bench_deep_filter_process_frame(c: &mut Criterion) {
|
||||
let mut processor = DeepFilterProcessor::new(DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX)
|
||||
.expect("embedded model must initialize");
|
||||
let mut seed = 0x9e37_79b9u32;
|
||||
let mut group = c.benchmark_group("deep_filter");
|
||||
group.throughput(Throughput::Elements(DEEP_FILTER_FRAME_SAMPLES as u64));
|
||||
group.bench_function("process_frame_10ms", |bencher| {
|
||||
bencher.iter_batched(
|
||||
|| noise_frame(&mut seed),
|
||||
|mut frame| {
|
||||
processor
|
||||
.process_frame(black_box(&mut frame))
|
||||
.expect("processing must succeed");
|
||||
frame
|
||||
},
|
||||
criterion::BatchSize::SmallInput,
|
||||
);
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_deep_filter_process_frame);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,220 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
|
||||
use fluxer_screen_frame_bus::{
|
||||
BgraFrame, EnqueueOutcome, NATIVE_SCREEN_FRAME_SINK_ACCEPTED,
|
||||
NATIVE_SCREEN_FRAME_SINK_HANDLE_MAGIC, NATIVE_SCREEN_FRAME_SINK_HANDLE_VERSION,
|
||||
NativeScreenFrameSinkHandle, Nv12Frame, ScreenFrame, ScreenFrameSink, get_sink, register_sink,
|
||||
unregister_sink,
|
||||
};
|
||||
use std::ffi::c_void;
|
||||
use std::hint::black_box;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
struct DiscardSink(AtomicU64);
|
||||
impl ScreenFrameSink for DiscardSink {
|
||||
fn enqueue(&self, _frame: ScreenFrame) -> EnqueueOutcome {
|
||||
self.0.fetch_add(1, Ordering::Relaxed);
|
||||
EnqueueOutcome::Accepted
|
||||
}
|
||||
}
|
||||
|
||||
struct NativeDiscardSink(AtomicU64);
|
||||
|
||||
unsafe extern "C" fn retain_native_discard_sink(context: *const c_void) {
|
||||
unsafe { Arc::increment_strong_count(context.cast::<NativeDiscardSink>()) };
|
||||
}
|
||||
|
||||
unsafe extern "C" fn release_native_discard_sink(context: *const c_void) {
|
||||
unsafe { drop(Arc::from_raw(context.cast::<NativeDiscardSink>())) };
|
||||
}
|
||||
|
||||
unsafe extern "C" fn enqueue_native_discard_nv12(
|
||||
context: *const c_void,
|
||||
data: *const u8,
|
||||
data_len: usize,
|
||||
_width: u32,
|
||||
_height: u32,
|
||||
_stride_y: u32,
|
||||
_stride_uv: u32,
|
||||
_timestamp_us: i64,
|
||||
) -> u32 {
|
||||
if !context.is_null() && !data.is_null() && data_len > 0 {
|
||||
unsafe {
|
||||
(*context.cast::<NativeDiscardSink>())
|
||||
.0
|
||||
.fetch_add(1, Ordering::Relaxed)
|
||||
};
|
||||
}
|
||||
NATIVE_SCREEN_FRAME_SINK_ACCEPTED
|
||||
}
|
||||
|
||||
unsafe extern "C" fn enqueue_native_discard_bgra(
|
||||
context: *const c_void,
|
||||
data: *const u8,
|
||||
data_len: usize,
|
||||
_width: u32,
|
||||
_height: u32,
|
||||
_stride: u32,
|
||||
_timestamp_us: i64,
|
||||
) -> u32 {
|
||||
if !context.is_null() && !data.is_null() && data_len > 0 {
|
||||
unsafe {
|
||||
(*context.cast::<NativeDiscardSink>())
|
||||
.0
|
||||
.fetch_add(1, Ordering::Relaxed)
|
||||
};
|
||||
}
|
||||
NATIVE_SCREEN_FRAME_SINK_ACCEPTED
|
||||
}
|
||||
|
||||
fn native_discard_handle() -> (
|
||||
Arc<NativeDiscardSink>,
|
||||
fluxer_screen_frame_bus::NativeScreenFrameSinkHandleRef,
|
||||
) {
|
||||
let sink = Arc::new(NativeDiscardSink(AtomicU64::new(0)));
|
||||
let raw_context = Arc::into_raw(sink.clone()).cast::<c_void>();
|
||||
let handle = NativeScreenFrameSinkHandle {
|
||||
magic: NATIVE_SCREEN_FRAME_SINK_HANDLE_MAGIC,
|
||||
version: NATIVE_SCREEN_FRAME_SINK_HANDLE_VERSION,
|
||||
context: raw_context,
|
||||
retain: retain_native_discard_sink,
|
||||
release: release_native_discard_sink,
|
||||
enqueue_nv12: Some(enqueue_native_discard_nv12),
|
||||
enqueue_bgra: Some(enqueue_native_discard_bgra),
|
||||
enqueue_mac_cv_pixel_buffer: None,
|
||||
enqueue_dmabuf: None,
|
||||
enqueue_shared_texture: None,
|
||||
};
|
||||
let retained = unsafe { handle.retain_ref().expect("valid native sink handle") };
|
||||
unsafe { release_native_discard_sink(raw_context) };
|
||||
(sink, retained)
|
||||
}
|
||||
|
||||
fn bench_registry_lookup(c: &mut Criterion) {
|
||||
let id = format!("bench:registry:{}", std::process::id());
|
||||
register_sink(
|
||||
id.clone(),
|
||||
Arc::new(DiscardSink(AtomicU64::new(0))) as Arc<dyn ScreenFrameSink>,
|
||||
);
|
||||
|
||||
c.bench_function("frame_bus::get_sink", |b| {
|
||||
b.iter(|| {
|
||||
let s = get_sink(black_box(id.as_str()));
|
||||
black_box(s);
|
||||
})
|
||||
});
|
||||
|
||||
unregister_sink(&id);
|
||||
}
|
||||
|
||||
fn synth_nv12(width: u32, height: u32) -> ScreenFrame {
|
||||
let total = (width * height) as usize + (width * (height / 2)) as usize;
|
||||
let data = vec![0x80u8; total];
|
||||
ScreenFrame::Nv12(Nv12Frame {
|
||||
data: data.into(),
|
||||
width,
|
||||
height,
|
||||
stride_y: width,
|
||||
stride_uv: width,
|
||||
timestamp_us: 1,
|
||||
})
|
||||
}
|
||||
|
||||
fn synth_bgra(width: u32, height: u32) -> ScreenFrame {
|
||||
let total = (width * height * 4) as usize;
|
||||
let data = vec![0xff; total];
|
||||
ScreenFrame::Bgra(BgraFrame {
|
||||
data,
|
||||
width,
|
||||
height,
|
||||
stride: width * 4,
|
||||
timestamp_us: 1,
|
||||
})
|
||||
}
|
||||
|
||||
fn bench_discard_sink_enqueue(c: &mut Criterion) {
|
||||
let id = format!("bench:discard:{}", std::process::id());
|
||||
let sink = Arc::new(DiscardSink(AtomicU64::new(0)));
|
||||
register_sink(id.clone(), sink.clone() as Arc<dyn ScreenFrameSink>);
|
||||
|
||||
let mut group = c.benchmark_group("frame_bus::enqueue_discard");
|
||||
for (label, w, h) in [
|
||||
("nv12_320x180", 320u32, 180u32),
|
||||
("nv12_1920x1080", 1920, 1080),
|
||||
("nv12_3840x2160", 3840, 2160),
|
||||
("bgra_320x180", 320, 180),
|
||||
("bgra_1920x1080", 1920, 1080),
|
||||
] {
|
||||
let bytes = if label.starts_with("nv12") {
|
||||
((w * h) + (w * (h / 2))) as u64
|
||||
} else {
|
||||
(w * h * 4) as u64
|
||||
};
|
||||
group.throughput(Throughput::Bytes(bytes));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(label), &(), |b, _| {
|
||||
b.iter(|| {
|
||||
let frame = if label.starts_with("nv12") {
|
||||
synth_nv12(w, h)
|
||||
} else {
|
||||
synth_bgra(w, h)
|
||||
};
|
||||
let s = get_sink(id.as_str()).unwrap();
|
||||
let outcome = s.enqueue(black_box(frame));
|
||||
black_box(outcome);
|
||||
})
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
unregister_sink(&id);
|
||||
}
|
||||
|
||||
fn bench_native_handle_enqueue(c: &mut Criterion) {
|
||||
let (_sink, handle) = native_discard_handle();
|
||||
let mut group = c.benchmark_group("frame_bus::native_handle_enqueue");
|
||||
for (label, w, h) in [
|
||||
("nv12_320x180", 320u32, 180u32),
|
||||
("nv12_1920x1080", 1920, 1080),
|
||||
("nv12_3840x2160", 3840, 2160),
|
||||
("bgra_320x180", 320, 180),
|
||||
("bgra_1920x1080", 1920, 1080),
|
||||
] {
|
||||
let bytes = if label.starts_with("nv12") {
|
||||
((w * h) + (w * (h / 2))) as usize
|
||||
} else {
|
||||
(w * h * 4) as usize
|
||||
};
|
||||
let data = vec![0x80u8; bytes];
|
||||
group.throughput(Throughput::Bytes(bytes as u64));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(label), &(), |b, _| {
|
||||
b.iter(|| {
|
||||
let outcome = if label.starts_with("nv12") {
|
||||
handle.enqueue_nv12_copy(black_box(&data), w, h, w, w, 1)
|
||||
} else {
|
||||
handle.enqueue_bgra_copy(black_box(&data), w, h, w * 4, 1)
|
||||
};
|
||||
black_box(outcome);
|
||||
})
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_registry_lookup_miss(c: &mut Criterion) {
|
||||
c.bench_function("frame_bus::get_sink_miss", |b| {
|
||||
b.iter(|| {
|
||||
let s = get_sink(black_box("nonexistent"));
|
||||
black_box(s);
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_registry_lookup,
|
||||
bench_registry_lookup_miss,
|
||||
bench_native_handle_enqueue,
|
||||
bench_discard_sink_enqueue
|
||||
);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,99 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=vendor/webrtc-sys/src/nvidia/NvCodec/include/cuda.h");
|
||||
println!("cargo:rustc-check-cfg=cfg(fluxer_linux_nvenc)");
|
||||
println!("cargo:rustc-check-cfg=cfg(fluxer_windows_nvenc)");
|
||||
println!("cargo:rustc-check-cfg=cfg(fluxer_windows_nvenc_encoder)");
|
||||
println!("cargo:rustc-check-cfg=cfg(fluxer_macos_videotoolbox)");
|
||||
|
||||
let cuda_include_dir = vendored_cuda_include_dir();
|
||||
let cuda_version = cuda_include_dir
|
||||
.as_deref()
|
||||
.and_then(read_cuda_version)
|
||||
.unwrap_or(0);
|
||||
let is_macos = env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos");
|
||||
let has_linux_nvenc = env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux")
|
||||
&& supports_webrtc_sys_nvenc_arch()
|
||||
&& cuda_version > 0;
|
||||
let has_windows_nvenc = env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows")
|
||||
&& supports_webrtc_sys_nvenc_arch();
|
||||
let has_windows_nvenc_encoder = has_windows_nvenc;
|
||||
|
||||
println!(
|
||||
"cargo:rustc-env=FLUXER_LINUX_NVENC_COMPILED={}",
|
||||
if has_linux_nvenc { "1" } else { "0" }
|
||||
);
|
||||
println!(
|
||||
"cargo:rustc-env=FLUXER_WINDOWS_NVENC_COMPILED={}",
|
||||
if has_windows_nvenc { "1" } else { "0" }
|
||||
);
|
||||
println!(
|
||||
"cargo:rustc-env=FLUXER_WINDOWS_NVENC_ENCODER_COMPILED={}",
|
||||
if has_windows_nvenc_encoder { "1" } else { "0" }
|
||||
);
|
||||
println!("cargo:rustc-env=FLUXER_CUDA_VERSION={cuda_version}");
|
||||
if has_linux_nvenc {
|
||||
println!("cargo:rustc-cfg=fluxer_linux_nvenc");
|
||||
}
|
||||
if has_windows_nvenc {
|
||||
println!("cargo:rustc-cfg=fluxer_windows_nvenc");
|
||||
}
|
||||
if has_windows_nvenc_encoder {
|
||||
println!("cargo:rustc-cfg=fluxer_windows_nvenc_encoder");
|
||||
}
|
||||
println!(
|
||||
"cargo:rustc-env=FLUXER_MACOS_VIDEOTOOLBOX_COMPILED={}",
|
||||
if is_macos { "1" } else { "0" }
|
||||
);
|
||||
if is_macos {
|
||||
println!("cargo:rustc-cfg=fluxer_macos_videotoolbox");
|
||||
println!("cargo:rustc-link-lib=framework=CoreFoundation");
|
||||
println!("cargo:rustc-link-lib=framework=CoreMedia");
|
||||
println!("cargo:rustc-link-lib=framework=VideoToolbox");
|
||||
}
|
||||
|
||||
configure_darwin_objc_linking();
|
||||
napi_build::setup();
|
||||
}
|
||||
|
||||
fn configure_darwin_objc_linking() {
|
||||
if env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") {
|
||||
return;
|
||||
}
|
||||
|
||||
println!("cargo:rustc-link-arg=-ObjC");
|
||||
}
|
||||
|
||||
fn vendored_cuda_include_dir() -> Option<PathBuf> {
|
||||
let manifest_dir = env::var("CARGO_MANIFEST_DIR").ok()?;
|
||||
Some(
|
||||
PathBuf::from(manifest_dir)
|
||||
.join("vendor")
|
||||
.join("webrtc-sys")
|
||||
.join("src")
|
||||
.join("nvidia")
|
||||
.join("NvCodec")
|
||||
.join("include"),
|
||||
)
|
||||
}
|
||||
|
||||
fn read_cuda_version(include_dir: &Path) -> Option<u32> {
|
||||
let content = std::fs::read_to_string(include_dir.join("cuda.h")).ok()?;
|
||||
content.lines().find_map(|line| {
|
||||
let mut parts = line.split_whitespace();
|
||||
match (parts.next(), parts.next(), parts.next()) {
|
||||
(Some("#define"), Some("CUDA_VERSION"), Some(value)) => value.parse().ok(),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn supports_webrtc_sys_nvenc_arch() -> bool {
|
||||
let Ok(arch) = env::var("CARGO_CFG_TARGET_ARCH") else {
|
||||
return false;
|
||||
};
|
||||
matches!(arch.as_str(), "x86_64" | "i686" | "aarch64") || arch.contains("arm")
|
||||
}
|
||||
+363
@@ -0,0 +1,363 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
export interface AudioOutputDevice {
|
||||
deviceId: string;
|
||||
label: string;
|
||||
isDefault: boolean;
|
||||
}
|
||||
|
||||
export interface AudioInputDevice {
|
||||
deviceId: string;
|
||||
label: string;
|
||||
isDefault: boolean;
|
||||
}
|
||||
|
||||
export interface PublishMicrophoneOptions {
|
||||
deviceId?: string;
|
||||
echoCancellation?: boolean;
|
||||
noiseSuppression?: boolean;
|
||||
autoGainControl?: boolean;
|
||||
deepFilter?: boolean;
|
||||
deepFilterNoiseReductionLevel?: number;
|
||||
maxBitrateBps?: number;
|
||||
}
|
||||
|
||||
export interface VoiceEngineV2BridgeCapabilities {
|
||||
microphoneCapture: boolean;
|
||||
syntheticMicrophonePcm: boolean;
|
||||
cameraCapture: boolean;
|
||||
nativeCameraBackgrounds: boolean;
|
||||
screenShare: boolean;
|
||||
screenShareEncodingUpdate: boolean;
|
||||
screenShareAudio: boolean;
|
||||
deviceLists: boolean;
|
||||
outputDeviceSelection: boolean;
|
||||
participantVolume: boolean;
|
||||
remoteTrackSubscription: boolean;
|
||||
dataChannel: boolean;
|
||||
connectionStats: boolean;
|
||||
nativeVideoFrames: boolean;
|
||||
hardwareEncoderCapabilities: boolean;
|
||||
}
|
||||
|
||||
export type VoiceEngineTrackKind = 'audio' | 'video';
|
||||
export type VoiceEngineTrackSource =
|
||||
| 'unknown'
|
||||
| 'camera'
|
||||
| 'microphone'
|
||||
| 'screen_share'
|
||||
| 'screen_share_audio'
|
||||
| 'screenshare'
|
||||
| 'screenshareAudio';
|
||||
export type VoiceEngineSubscriptionStatus = 'desired' | 'subscribed' | 'unsubscribed';
|
||||
export type VoiceEngineConnectionQuality = 'excellent' | 'good' | 'poor' | 'lost';
|
||||
|
||||
export interface VoiceEngineParticipantEventPayload {
|
||||
sid: string;
|
||||
identity: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface VoiceEngineTrackEventPayload {
|
||||
participantSid: string;
|
||||
identity: string;
|
||||
participantName: string;
|
||||
trackSid: string;
|
||||
trackName: string;
|
||||
kind: VoiceEngineTrackKind;
|
||||
source: VoiceEngineTrackSource;
|
||||
muted: boolean;
|
||||
}
|
||||
|
||||
export interface VoiceEngineSubscribedTrackEventPayload extends VoiceEngineTrackEventPayload {
|
||||
subscribed: boolean;
|
||||
subscriptionStatus: VoiceEngineSubscriptionStatus;
|
||||
}
|
||||
|
||||
export interface VoiceEngineTrackSubscriptionFailedEventPayload {
|
||||
participantSid: string;
|
||||
identity: string;
|
||||
participantName: string;
|
||||
trackSid: string;
|
||||
error: string;
|
||||
trackName?: string;
|
||||
kind?: VoiceEngineTrackKind;
|
||||
source?: VoiceEngineTrackSource;
|
||||
muted?: boolean;
|
||||
subscribed?: boolean;
|
||||
subscriptionStatus?: VoiceEngineSubscriptionStatus;
|
||||
}
|
||||
|
||||
export interface VoiceEngineLocalTrackRepublishedEventPayload extends VoiceEngineTrackEventPayload {
|
||||
previousTrackSid: string;
|
||||
}
|
||||
|
||||
export interface VoiceEngineV2BridgeEventPayloads {
|
||||
connected: Record<keyof any, never>;
|
||||
connectionState: {state: string};
|
||||
disconnected: {reason: string};
|
||||
participantJoined: VoiceEngineParticipantEventPayload;
|
||||
participantLeft: VoiceEngineParticipantEventPayload;
|
||||
participantNameChanged: {sid: string; identity: string; oldName: string; name: string};
|
||||
participantMetadataChanged: {
|
||||
sid: string;
|
||||
identity: string;
|
||||
name: string;
|
||||
oldMetadata: string;
|
||||
metadata: string;
|
||||
attributes: Record<string, string>;
|
||||
};
|
||||
participantAttributesChanged: {
|
||||
sid: string;
|
||||
identity: string;
|
||||
name: string;
|
||||
attributes: Record<string, string>;
|
||||
changedAttributes: Record<string, string>;
|
||||
};
|
||||
trackPublished: VoiceEngineSubscribedTrackEventPayload;
|
||||
trackUnpublished: VoiceEngineSubscribedTrackEventPayload;
|
||||
trackSubscribed: VoiceEngineSubscribedTrackEventPayload;
|
||||
trackUnsubscribed: VoiceEngineSubscribedTrackEventPayload;
|
||||
trackSubscriptionFailed: VoiceEngineTrackSubscriptionFailedEventPayload;
|
||||
trackMuted: VoiceEngineTrackEventPayload;
|
||||
trackUnmuted: VoiceEngineTrackEventPayload;
|
||||
localTrackPublished: VoiceEngineTrackEventPayload;
|
||||
localTrackUnpublished: VoiceEngineTrackEventPayload;
|
||||
localTrackRepublished: VoiceEngineLocalTrackRepublishedEventPayload;
|
||||
activeSpeakers: {sids: Array<string>; participants: Array<VoiceEngineParticipantEventPayload>};
|
||||
connectionQuality: {sid: string; identity: string; name: string; quality: VoiceEngineConnectionQuality};
|
||||
dataReceived: {
|
||||
payloadBytes: Array<number>;
|
||||
payloadText?: string;
|
||||
topic?: string;
|
||||
reliable: boolean;
|
||||
kind: 'reliable' | 'lossy';
|
||||
participantSid?: string;
|
||||
identity?: string;
|
||||
participantName?: string;
|
||||
};
|
||||
e2eeState: {sid: string; identity: string; name: string; state: string};
|
||||
stats: VoiceEngineV2BridgeStats;
|
||||
audioPlaybackUnavailable: {message: string};
|
||||
}
|
||||
|
||||
export type VoiceEngineKnownEventType = keyof VoiceEngineV2BridgeEventPayloads;
|
||||
export type VoiceEngineV2BridgeEventType = VoiceEngineKnownEventType | (string & {});
|
||||
|
||||
export interface VoiceEngineOutboundStats {
|
||||
trackSid: string;
|
||||
source: string;
|
||||
kind: VoiceEngineTrackKind;
|
||||
codec?: string;
|
||||
bitrateKbps: number;
|
||||
packetsLost: number;
|
||||
fps?: number;
|
||||
}
|
||||
|
||||
export interface VoiceEngineInboundStats {
|
||||
participantSid: string;
|
||||
trackSid: string;
|
||||
kind: VoiceEngineTrackKind;
|
||||
codec?: string;
|
||||
bitrateKbps: number;
|
||||
packetsLost: number;
|
||||
jitterMs?: number;
|
||||
audioLevel?: number;
|
||||
}
|
||||
|
||||
export interface VoiceEngineV2BridgeStats {
|
||||
rttMs: number | null;
|
||||
outbound: Array<VoiceEngineOutboundStats>;
|
||||
inbound: Array<VoiceEngineInboundStats>;
|
||||
droppedVideoFrameCallbacks?: number;
|
||||
send?: VoiceEngineSendStats | null;
|
||||
}
|
||||
|
||||
export interface VoiceEngineSendStats {
|
||||
outgoingVideoQueueDepth: number;
|
||||
outgoingVideoFramesProduced: number;
|
||||
outgoingVideoFramesAccepted: number;
|
||||
outgoingVideoFramesDropped: number;
|
||||
outgoingVideoFramesCoalesced: number;
|
||||
outgoingVideoFramesCaptured: number;
|
||||
outgoingVideoCaptureFailures: number;
|
||||
outgoingVideoEffectiveFps: number;
|
||||
outgoingVideoTargetFps: number;
|
||||
outgoingVideoMaxQueueAgeMs: number;
|
||||
outgoingVideoMaxPushLatencyMs: number;
|
||||
outgoingAudioBufferTargetMs: number;
|
||||
outgoingAudioBufferMaxMs: number;
|
||||
outgoingAudioUnderruns: number;
|
||||
outgoingAudioRebuffers: number;
|
||||
outgoingAudioMaxFrameGapMs: number;
|
||||
adaptiveSendTier: string;
|
||||
adaptiveSendReason: string;
|
||||
}
|
||||
|
||||
export interface PublishScreenShareOptions {
|
||||
adaptiveSend?: boolean;
|
||||
minVideoFps?: number;
|
||||
maxAudioBufferMs?: number;
|
||||
pacing?: 'sender' | 'source';
|
||||
captureId: string;
|
||||
trackName?: string;
|
||||
}
|
||||
|
||||
export type VoiceEngineRemoteTrackSubscriptionQuality = 'low' | 'medium' | 'high';
|
||||
|
||||
export interface VoiceEngineV2BridgeRemoteTrackSubscriptionOptions {
|
||||
participantIdentity: string;
|
||||
source: string;
|
||||
subscribed: boolean;
|
||||
enabled?: boolean;
|
||||
quality?: VoiceEngineRemoteTrackSubscriptionQuality;
|
||||
}
|
||||
|
||||
export interface PublishCameraOptions {
|
||||
deviceId?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
frameRate?: number;
|
||||
mirror?: boolean;
|
||||
backgroundMode?: 'none' | 'non' | 'blur' | 'custom';
|
||||
backgroundCustomMediaPath?: string;
|
||||
backgroundCustomMediaKind?: 'static' | 'animated' | 'video';
|
||||
backgroundBlurStrength?: number;
|
||||
codec?: '' | 'vp8' | 'vp9' | 'h264' | 'av1' | 'h265' | 'hevc';
|
||||
maxBitrateBps?: number;
|
||||
maxFramerate?: number;
|
||||
}
|
||||
|
||||
export interface PublishProcessedCameraOptions {
|
||||
width: number;
|
||||
height: number;
|
||||
frameRate: number;
|
||||
}
|
||||
|
||||
export interface PublishProcessedCameraResult {
|
||||
trackSid: string;
|
||||
}
|
||||
|
||||
export type NativeCameraFrameSinkHandle = object;
|
||||
|
||||
export interface ProcessedCameraFrame {
|
||||
format: 'i420';
|
||||
width: number;
|
||||
height: number;
|
||||
timestampUs: number;
|
||||
data: Buffer;
|
||||
}
|
||||
|
||||
export interface CameraDeviceInfo {
|
||||
deviceId: string;
|
||||
label: string;
|
||||
description: string;
|
||||
index?: number | null;
|
||||
deviceIdAliases: Array<string>;
|
||||
}
|
||||
|
||||
export interface HardwareEncoderCapability {
|
||||
available: boolean;
|
||||
backend: 'nvenc' | 'videotoolbox' | 'none';
|
||||
compiled: boolean;
|
||||
runtime: boolean;
|
||||
codecs: Array<string>;
|
||||
zeroCopy: boolean;
|
||||
nativeInputs: Array<'dmabuf' | 'd3d11-texture' | string>;
|
||||
reason?: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export interface VoiceEngineV2BridgeConnectOptions {
|
||||
autoSubscribe?: boolean;
|
||||
adaptiveStream?: boolean;
|
||||
dynacast?: boolean;
|
||||
}
|
||||
|
||||
export declare class VoiceEngine {
|
||||
constructor();
|
||||
|
||||
setEventCallback(callback: (eventType: VoiceEngineV2BridgeEventType, jsonPayload: string) => void): void;
|
||||
setVideoFrameCallback(callback: (metaJson: string, data: Buffer) => void): void;
|
||||
clearVideoFrameCallback(): void;
|
||||
setCountInboundAudio(enabled: boolean): void;
|
||||
|
||||
connect(url: string, token: string, e2eeKey?: Buffer, options?: VoiceEngineV2BridgeConnectOptions): Promise<void>;
|
||||
disconnect(): Promise<void>;
|
||||
isConnected(): boolean;
|
||||
|
||||
publishScreenShare(
|
||||
width: number,
|
||||
height: number,
|
||||
codec: '' | 'vp8' | 'vp9' | 'h264' | 'av1' | 'h265' | 'hevc' | undefined,
|
||||
maxBitrateBps: number | undefined,
|
||||
maxFramerate: number | undefined,
|
||||
simulcast: boolean | undefined,
|
||||
options: PublishScreenShareOptions,
|
||||
): Promise<void>;
|
||||
updateScreenShareEncoding(
|
||||
width: number,
|
||||
height: number,
|
||||
maxBitrateBps: number | undefined,
|
||||
maxFramerate: number | undefined,
|
||||
options: PublishScreenShareOptions,
|
||||
): Promise<void>;
|
||||
createScreenFrameSinkHandle(captureId: string): unknown | null;
|
||||
unpublishScreenShare(): Promise<void>;
|
||||
isPublishingScreen(): boolean;
|
||||
|
||||
publishScreenShareAudio(sampleRate: number, numChannels: number): Promise<void>;
|
||||
pushScreenSharePcm(buffer: Buffer, sampleRate: number, numChannels: number): Promise<boolean>;
|
||||
pushScreenShareFloat(buffer: Buffer, sampleRate: number, numChannels: number): Promise<boolean>;
|
||||
unpublishScreenShareAudio(): Promise<void>;
|
||||
isPublishingScreenAudio(): boolean;
|
||||
|
||||
publishMicrophone(sampleRate: number, numChannels: number): Promise<void>;
|
||||
publishDeviceMicrophone(opts?: PublishMicrophoneOptions): Promise<void>;
|
||||
pushPcm(buffer: Buffer, sampleRate: number, numChannels: number): Promise<boolean>;
|
||||
setMicEnabled(enabled: boolean): Promise<void>;
|
||||
setSpeakingDetection(localThresholdRms: number, remoteThresholdRms: number): void;
|
||||
|
||||
publishCamera(opts?: PublishCameraOptions): Promise<void>;
|
||||
updateCameraCapture(opts?: PublishCameraOptions): Promise<void>;
|
||||
publishProcessedCamera(opts: PublishProcessedCameraOptions): Promise<PublishProcessedCameraResult>;
|
||||
publishNativeCameraSink(opts?: PublishCameraOptions): Promise<PublishProcessedCameraResult>;
|
||||
createCameraFrameSinkHandle(): NativeCameraFrameSinkHandle | null;
|
||||
pushProcessedCameraFrame(frame: ProcessedCameraFrame): Promise<boolean>;
|
||||
publishDeviceScreenShare(opts?: PublishCameraOptions): Promise<void>;
|
||||
listCameraDevices(): Array<CameraDeviceInfo>;
|
||||
unpublishCamera(): Promise<void>;
|
||||
isPublishingCamera(): boolean;
|
||||
|
||||
listAudioInputDevices(): Promise<Array<AudioInputDevice>>;
|
||||
listAudioOutputDevices(): Promise<Array<AudioOutputDevice>>;
|
||||
setAudioOutputDevice(deviceId: string): Promise<void>;
|
||||
ensurePlatformAudio(): Promise<void>;
|
||||
|
||||
setParticipantVolume(participantSid: string, volume: number): Promise<void>;
|
||||
setRemoteTrackSubscription(options: VoiceEngineV2BridgeRemoteTrackSubscriptionOptions): Promise<void>;
|
||||
publishData(
|
||||
payload: Buffer | ArrayBuffer | Uint8Array,
|
||||
options?: {reliable?: boolean; topic?: string; destinationIdentities?: Array<string>},
|
||||
): Promise<void>;
|
||||
|
||||
getConnectionStats(): Promise<VoiceEngineV2BridgeStats>;
|
||||
|
||||
inboundAudioFrames(): number;
|
||||
inboundVideoFrames(): number;
|
||||
droppedVideoFrameCallbacks(): number;
|
||||
droppedEngineEvents(): number;
|
||||
}
|
||||
|
||||
export declare function isSupported(): boolean;
|
||||
export declare function getEngineBridgeVersion(): number | null;
|
||||
export declare function assertEngineBridgeVersion(version: number): void;
|
||||
export declare function getHardwareEncoderCapability(): HardwareEncoderCapability;
|
||||
export declare function getHardwareEncoderCapabilities(): HardwareEncoderCapability;
|
||||
export declare function getCapabilities(): VoiceEngineV2BridgeCapabilities;
|
||||
export declare function hasNativeCameraBackgrounds(): boolean;
|
||||
export declare function prewarmVoiceEngine(): void;
|
||||
export declare function probeAudioDeviceModule(): Promise<boolean>;
|
||||
export declare const loadError: Error | null;
|
||||
export declare function __nativeFileNameForTests(platform: string, arch: string): string;
|
||||
export declare function __setBindingForTests(binding: unknown): void;
|
||||
@@ -0,0 +1,509 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
const {existsSync} = require('node:fs');
|
||||
const {join, sep} = require('node:path');
|
||||
|
||||
const MODULE_NAME = '@fluxer/webrtc-sender';
|
||||
|
||||
function resolveNativeRoot() {
|
||||
const asarSegment = `${sep}app.asar${sep}`;
|
||||
if (!__dirname.includes(asarSegment)) return __dirname;
|
||||
const unpackedDir = __dirname.replace(asarSegment, `${sep}app.asar.unpacked${sep}`);
|
||||
return existsSync(unpackedDir) ? unpackedDir : __dirname;
|
||||
}
|
||||
|
||||
function nativeFileName(platform = process.platform, arch = process.arch) {
|
||||
if ((arch !== 'x64' && arch !== 'arm64') || !['darwin', 'linux', 'win32'].includes(platform)) {
|
||||
throw new Error(`${MODULE_NAME} not supported on ${platform}-${arch}`);
|
||||
}
|
||||
if (platform === 'darwin') return `webrtc-sender.darwin-${arch}.node`;
|
||||
if (platform === 'linux') return `webrtc-sender.linux-${arch}-gnu.node`;
|
||||
if (platform === 'win32') return `webrtc-sender.win32-${arch}-msvc.node`;
|
||||
throw new Error(`${MODULE_NAME} not supported on ${platform}-${arch}`);
|
||||
}
|
||||
|
||||
let binding = null;
|
||||
let loadError = null;
|
||||
|
||||
try {
|
||||
const nativePath = join(resolveNativeRoot(), nativeFileName());
|
||||
if (existsSync(nativePath)) {
|
||||
try {
|
||||
binding = require(nativePath);
|
||||
} catch (error) {
|
||||
loadError = error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
} else {
|
||||
loadError = new Error(`${MODULE_NAME} native binary missing: ${nativePath}`);
|
||||
}
|
||||
} catch (error) {
|
||||
loadError = error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
|
||||
function isSupported() {
|
||||
return Boolean(binding);
|
||||
}
|
||||
|
||||
function unavailableHardwareEncoderCapability(reason, detail) {
|
||||
return {
|
||||
available: false,
|
||||
backend: 'none',
|
||||
compiled: false,
|
||||
runtime: false,
|
||||
codecs: [],
|
||||
zeroCopy: false,
|
||||
nativeInputs: [],
|
||||
reason,
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
function getHardwareEncoderCapability() {
|
||||
if (!binding) {
|
||||
return unavailableHardwareEncoderCapability(
|
||||
'native_binding_unavailable',
|
||||
loadError ? loadError.message : `${MODULE_NAME} binding unavailable`,
|
||||
);
|
||||
}
|
||||
if (typeof binding.getHardwareEncoderCapability !== 'function') {
|
||||
return unavailableHardwareEncoderCapability(
|
||||
'native_capability_unavailable',
|
||||
`${MODULE_NAME} native binding does not export getHardwareEncoderCapability`,
|
||||
);
|
||||
}
|
||||
return binding.getHardwareEncoderCapability();
|
||||
}
|
||||
|
||||
function getHardwareEncoderCapabilities() {
|
||||
return getHardwareEncoderCapability();
|
||||
}
|
||||
|
||||
function hasVoiceEngineMethod(name) {
|
||||
const prototype = binding?.VoiceEngine?.prototype;
|
||||
return Boolean(prototype && typeof prototype[name] === 'function');
|
||||
}
|
||||
|
||||
function hasNativeCameraBackgrounds() {
|
||||
if (!binding) return false;
|
||||
if (typeof binding.hasNativeCameraBackgrounds !== 'function') return false;
|
||||
return binding.hasNativeCameraBackgrounds() === true;
|
||||
}
|
||||
|
||||
function getCapabilities() {
|
||||
const hasVoiceEngine = Boolean(binding && typeof binding.VoiceEngine === 'function');
|
||||
return {
|
||||
microphoneCapture: hasVoiceEngineMethod('publishDeviceMicrophone'),
|
||||
syntheticMicrophonePcm: hasVoiceEngineMethod('publishMicrophone'),
|
||||
cameraCapture: hasVoiceEngineMethod('publishCamera') && hasVoiceEngineMethod('listCameraDevices'),
|
||||
nativeCameraBackgrounds: hasNativeCameraBackgrounds(),
|
||||
screenShare: hasVoiceEngineMethod('publishScreenShare') && hasVoiceEngineMethod('unpublishScreenShare'),
|
||||
screenShareEncodingUpdate: hasVoiceEngineMethod('updateScreenShareEncoding'),
|
||||
screenShareAudio:
|
||||
hasVoiceEngineMethod('publishScreenShareAudio') &&
|
||||
hasVoiceEngineMethod('pushScreenSharePcm') &&
|
||||
hasVoiceEngineMethod('pushScreenShareFloat') &&
|
||||
hasVoiceEngineMethod('unpublishScreenShareAudio'),
|
||||
deviceLists: hasVoiceEngineMethod('listAudioInputDevices') && hasVoiceEngineMethod('listAudioOutputDevices'),
|
||||
outputDeviceSelection: hasVoiceEngineMethod('setAudioOutputDevice'),
|
||||
participantVolume: hasVoiceEngineMethod('setParticipantVolume'),
|
||||
remoteTrackSubscription: hasVoiceEngineMethod('setRemoteTrackSubscription'),
|
||||
dataChannel: hasVoiceEngineMethod('publishData'),
|
||||
connectionStats: hasVoiceEngineMethod('getConnectionStats'),
|
||||
nativeVideoFrames: hasVoiceEngineMethod('setVideoFrameCallback'),
|
||||
hardwareEncoderCapabilities: hasVoiceEngine && typeof getHardwareEncoderCapability === 'function',
|
||||
};
|
||||
}
|
||||
|
||||
function requireScreenShareCaptureId(options, operation) {
|
||||
if (!options || typeof options.captureId !== 'string' || options.captureId.trim().length === 0) {
|
||||
throw new Error(`${operation} requires a non-empty captureId`);
|
||||
}
|
||||
}
|
||||
|
||||
function getEngineBridgeVersion() {
|
||||
if (!binding) return null;
|
||||
if (typeof binding.getEngineBridgeVersion !== 'function') return null;
|
||||
return binding.getEngineBridgeVersion();
|
||||
}
|
||||
|
||||
function assertEngineBridgeVersion(version) {
|
||||
if (!binding) {
|
||||
throw loadError || new Error(`${MODULE_NAME} binding unavailable`);
|
||||
}
|
||||
if (typeof binding.assertEngineBridgeVersion !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export assertEngineBridgeVersion`);
|
||||
}
|
||||
binding.assertEngineBridgeVersion(version);
|
||||
}
|
||||
|
||||
function prewarmVoiceEngine() {
|
||||
if (!binding) {
|
||||
if (loadError) throw loadError;
|
||||
return;
|
||||
}
|
||||
if (typeof binding.prewarmVoiceEngine === 'function') {
|
||||
return binding.prewarmVoiceEngine();
|
||||
}
|
||||
}
|
||||
|
||||
function probeAudioDeviceModule() {
|
||||
if (!binding) {
|
||||
if (loadError) throw loadError;
|
||||
return Promise.resolve(false);
|
||||
}
|
||||
if (typeof binding.probeAudioDeviceModule === 'function') {
|
||||
return Promise.resolve(binding.probeAudioDeviceModule());
|
||||
}
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
|
||||
function normalizeCameraOptions(opts = {}) {
|
||||
return {
|
||||
deviceId: opts.deviceId,
|
||||
width: opts.width,
|
||||
height: opts.height,
|
||||
frameRate: opts.frameRate,
|
||||
mirror: opts.mirror,
|
||||
backgroundMode: opts.backgroundMode,
|
||||
backgroundCustomMediaPath: opts.backgroundCustomMediaPath,
|
||||
backgroundCustomMediaKind: opts.backgroundCustomMediaKind,
|
||||
backgroundBlurStrength: opts.backgroundBlurStrength,
|
||||
codec: opts.codec,
|
||||
maxBitrateBps: opts.maxBitrateBps,
|
||||
maxFramerate: opts.maxFramerate,
|
||||
};
|
||||
}
|
||||
|
||||
class VoiceEngine {
|
||||
constructor() {
|
||||
if (!binding) {
|
||||
throw loadError || new Error(`${MODULE_NAME} binding unavailable`);
|
||||
}
|
||||
this.native = new binding.VoiceEngine();
|
||||
}
|
||||
|
||||
setEventCallback(callback) {
|
||||
return this.native.setEventCallback(callback);
|
||||
}
|
||||
|
||||
setVideoFrameCallback(callback) {
|
||||
return this.native.setVideoFrameCallback(callback);
|
||||
}
|
||||
|
||||
clearVideoFrameCallback() {
|
||||
if (typeof this.native.clearVideoFrameCallback === 'function') {
|
||||
return this.native.clearVideoFrameCallback();
|
||||
}
|
||||
if (typeof this.native.setVideoFrameCallback === 'function') {
|
||||
return this.native.setVideoFrameCallback(() => {});
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
setCountInboundAudio(enabled) {
|
||||
return this.native.setCountInboundAudio(enabled);
|
||||
}
|
||||
|
||||
connect(url, token, e2eeKey, options) {
|
||||
return this.native.connect(url, token, e2eeKey, options);
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
return this.native.disconnect();
|
||||
}
|
||||
|
||||
isConnected() {
|
||||
return this.native.isConnected();
|
||||
}
|
||||
|
||||
publishScreenShare(width, height, codec = '', maxBitrateBps, maxFramerate, simulcast, options) {
|
||||
requireScreenShareCaptureId(options, 'Native screen-share publish');
|
||||
return this.native.publishScreenShare(width, height, codec, maxBitrateBps, maxFramerate, simulcast, options);
|
||||
}
|
||||
|
||||
updateScreenShareEncoding(width, height, maxBitrateBps, maxFramerate, options) {
|
||||
requireScreenShareCaptureId(options, 'Native screen-share encoding update');
|
||||
if (typeof this.native.updateScreenShareEncoding === 'function') {
|
||||
return this.native.updateScreenShareEncoding(width, height, maxBitrateBps, maxFramerate, options);
|
||||
}
|
||||
return Promise.reject(new Error('Native screen-share encoding update is unavailable'));
|
||||
}
|
||||
|
||||
createScreenFrameSinkHandle(captureId) {
|
||||
if (typeof this.native.createScreenFrameSinkHandle !== 'function') return null;
|
||||
return this.native.createScreenFrameSinkHandle(captureId);
|
||||
}
|
||||
|
||||
createScreenAudioSinkHandle() {
|
||||
if (typeof this.native.createScreenAudioSinkHandle !== 'function') return null;
|
||||
return this.native.createScreenAudioSinkHandle();
|
||||
}
|
||||
|
||||
unpublishScreenShare() {
|
||||
return this.native.unpublishScreenShare();
|
||||
}
|
||||
|
||||
isPublishingScreen() {
|
||||
return this.native.isPublishingScreen();
|
||||
}
|
||||
|
||||
publishScreenShareAudio(sampleRate, numChannels) {
|
||||
return this.native.publishScreenShareAudio(sampleRate, numChannels);
|
||||
}
|
||||
|
||||
pushScreenSharePcm(buffer, sampleRate, numChannels) {
|
||||
return this.native.pushScreenSharePcm(buffer, sampleRate, numChannels);
|
||||
}
|
||||
|
||||
pushScreenShareFloat(buffer, sampleRate, numChannels) {
|
||||
return this.native.pushScreenShareFloat(buffer, sampleRate, numChannels);
|
||||
}
|
||||
|
||||
unpublishScreenShareAudio() {
|
||||
return this.native.unpublishScreenShareAudio();
|
||||
}
|
||||
|
||||
isPublishingScreenAudio() {
|
||||
return this.native.isPublishingScreenAudio();
|
||||
}
|
||||
|
||||
publishMicrophone(sampleRate, numChannels) {
|
||||
return this.native.publishMicrophone(sampleRate, numChannels);
|
||||
}
|
||||
|
||||
publishDeviceMicrophone(opts = {}) {
|
||||
if (typeof this.native.publishDeviceMicrophone !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export publishDeviceMicrophone`);
|
||||
}
|
||||
return this.native.publishDeviceMicrophone({
|
||||
deviceId: opts.deviceId,
|
||||
echoCancellation: opts.echoCancellation,
|
||||
noiseSuppression: opts.noiseSuppression,
|
||||
autoGainControl: opts.autoGainControl,
|
||||
...(opts.deepFilter !== undefined ? {deepFilter: opts.deepFilter} : {}),
|
||||
...(opts.deepFilterNoiseReductionLevel !== undefined
|
||||
? {deepFilterNoiseReductionLevel: opts.deepFilterNoiseReductionLevel}
|
||||
: {}),
|
||||
maxBitrateBps: opts.maxBitrateBps,
|
||||
});
|
||||
}
|
||||
|
||||
pushPcm(buffer, sampleRate, numChannels) {
|
||||
return this.native.pushPcm(buffer, sampleRate, numChannels);
|
||||
}
|
||||
|
||||
setMicEnabled(enabled) {
|
||||
return this.native.setMicEnabled(enabled);
|
||||
}
|
||||
|
||||
setSpeakingDetection(localThresholdRms, remoteThresholdRms) {
|
||||
if (typeof this.native.setSpeakingDetection !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export setSpeakingDetection`);
|
||||
}
|
||||
return this.native.setSpeakingDetection(localThresholdRms, remoteThresholdRms);
|
||||
}
|
||||
|
||||
publishCamera(opts = {}) {
|
||||
return this.native.publishCamera(normalizeCameraOptions(opts));
|
||||
}
|
||||
|
||||
updateCameraCapture(opts = {}) {
|
||||
if (typeof this.native.updateCameraCapture !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export updateCameraCapture`);
|
||||
}
|
||||
return this.native.updateCameraCapture(normalizeCameraOptions(opts));
|
||||
}
|
||||
|
||||
publishProcessedCamera(opts) {
|
||||
if (typeof this.native.publishProcessedCamera !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export publishProcessedCamera`);
|
||||
}
|
||||
return this.native.publishProcessedCamera({
|
||||
width: opts.width,
|
||||
height: opts.height,
|
||||
frameRate: opts.frameRate,
|
||||
});
|
||||
}
|
||||
|
||||
publishNativeCameraSink(opts = {}) {
|
||||
if (typeof this.native.publishNativeCameraSink !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export publishNativeCameraSink`);
|
||||
}
|
||||
return this.native.publishNativeCameraSink(normalizeCameraOptions(opts));
|
||||
}
|
||||
|
||||
createCameraFrameSinkHandle() {
|
||||
if (typeof this.native.createCameraFrameSinkHandle !== 'function') return null;
|
||||
return this.native.createCameraFrameSinkHandle();
|
||||
}
|
||||
|
||||
pushProcessedCameraFrame(frame) {
|
||||
if (typeof this.native.pushProcessedCameraFrame !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export pushProcessedCameraFrame`);
|
||||
}
|
||||
return this.native.pushProcessedCameraFrame({
|
||||
format: frame.format,
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
timestampUs: frame.timestampUs,
|
||||
data: frame.data,
|
||||
});
|
||||
}
|
||||
|
||||
startCameraPreview(opts = {}) {
|
||||
if (typeof this.native.startCameraPreview !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export startCameraPreview`);
|
||||
}
|
||||
return this.native.startCameraPreview(normalizeCameraOptions(opts));
|
||||
}
|
||||
|
||||
stopCameraPreview() {
|
||||
if (typeof this.native.stopCameraPreview !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export stopCameraPreview`);
|
||||
}
|
||||
return this.native.stopCameraPreview();
|
||||
}
|
||||
|
||||
pushCameraBackgroundFrame(frame) {
|
||||
if (typeof this.native.pushCameraBackgroundFrame !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export pushCameraBackgroundFrame`);
|
||||
}
|
||||
return this.native.pushCameraBackgroundFrame({
|
||||
format: frame.format,
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
timestampUs: frame.timestampUs,
|
||||
data: frame.data,
|
||||
});
|
||||
}
|
||||
|
||||
clearCameraBackgroundFrame() {
|
||||
if (typeof this.native.clearCameraBackgroundFrame !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export clearCameraBackgroundFrame`);
|
||||
}
|
||||
return this.native.clearCameraBackgroundFrame();
|
||||
}
|
||||
|
||||
publishDeviceScreenShare(opts = {}) {
|
||||
if (typeof this.native.publishDeviceScreenShare !== 'function') {
|
||||
throw new Error(`${MODULE_NAME} native binding does not export publishDeviceScreenShare`);
|
||||
}
|
||||
return this.native.publishDeviceScreenShare(normalizeCameraOptions(opts));
|
||||
}
|
||||
|
||||
listCameraDevices() {
|
||||
return this.native.listCameraDevices();
|
||||
}
|
||||
|
||||
unpublishCamera() {
|
||||
return this.native.unpublishCamera();
|
||||
}
|
||||
|
||||
isPublishingCamera() {
|
||||
return this.native.isPublishingCamera();
|
||||
}
|
||||
|
||||
async listAudioInputDevices() {
|
||||
const json = await this.native.listAudioInputDevices();
|
||||
return JSON.parse(json);
|
||||
}
|
||||
|
||||
droppedEngineEvents() {
|
||||
if (typeof this.native.droppedEngineEvents !== 'function') return 0;
|
||||
return this.native.droppedEngineEvents();
|
||||
}
|
||||
|
||||
async listAudioOutputDevices() {
|
||||
const json = await this.native.listAudioOutputDevices();
|
||||
return JSON.parse(json);
|
||||
}
|
||||
|
||||
setAudioOutputDevice(deviceId) {
|
||||
return this.native.setAudioOutputDevice(deviceId || '');
|
||||
}
|
||||
|
||||
ensurePlatformAudio() {
|
||||
if (typeof this.native.ensurePlatformAudio !== 'function') return Promise.resolve();
|
||||
return this.native.ensurePlatformAudio();
|
||||
}
|
||||
|
||||
setParticipantVolume(participantSid, volume) {
|
||||
return this.native.setParticipantVolume(participantSid, volume);
|
||||
}
|
||||
|
||||
setRemoteTrackSubscription(opts = {}) {
|
||||
if (typeof this.native.setRemoteTrackSubscription !== 'function') return Promise.resolve();
|
||||
return this.native.setRemoteTrackSubscription(
|
||||
opts.participantIdentity || '',
|
||||
opts.source || '',
|
||||
opts.subscribed === true,
|
||||
opts.enabled !== false,
|
||||
opts.quality || undefined,
|
||||
);
|
||||
}
|
||||
|
||||
publishData(payload, opts = {}) {
|
||||
if (typeof this.native.publishData !== 'function') {
|
||||
return Promise.reject(new Error(`${MODULE_NAME} native binding does not export publishData`));
|
||||
}
|
||||
const buffer = Buffer.isBuffer(payload)
|
||||
? payload
|
||||
: payload instanceof ArrayBuffer
|
||||
? Buffer.from(payload)
|
||||
: ArrayBuffer.isView(payload)
|
||||
? Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength)
|
||||
: null;
|
||||
if (!buffer) {
|
||||
return Promise.reject(new TypeError('publishData payload must be a Buffer, ArrayBuffer, or typed array'));
|
||||
}
|
||||
return this.native.publishData(
|
||||
buffer,
|
||||
opts.reliable !== false,
|
||||
typeof opts.topic === 'string' ? opts.topic : undefined,
|
||||
Array.isArray(opts.destinationIdentities) ? opts.destinationIdentities : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
getConnectionStats() {
|
||||
const json = this.native.getConnectionStats();
|
||||
try {
|
||||
return JSON.parse(json);
|
||||
} catch {
|
||||
return {rttMs: null, outbound: [], inbound: []};
|
||||
}
|
||||
}
|
||||
|
||||
inboundAudioFrames() {
|
||||
return this.native.inboundAudioFrames();
|
||||
}
|
||||
|
||||
inboundVideoFrames() {
|
||||
return this.native.inboundVideoFrames();
|
||||
}
|
||||
|
||||
droppedVideoFrameCallbacks() {
|
||||
if (typeof this.native.droppedVideoFrameCallbacks !== 'function') return 0;
|
||||
return this.native.droppedVideoFrameCallbacks();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
isSupported,
|
||||
getEngineBridgeVersion,
|
||||
assertEngineBridgeVersion,
|
||||
getHardwareEncoderCapability,
|
||||
getHardwareEncoderCapabilities,
|
||||
getCapabilities,
|
||||
hasNativeCameraBackgrounds,
|
||||
prewarmVoiceEngine,
|
||||
probeAudioDeviceModule,
|
||||
VoiceEngine,
|
||||
get loadError() {
|
||||
return loadError;
|
||||
},
|
||||
__nativeFileNameForTests: nativeFileName,
|
||||
__setBindingForTests(next) {
|
||||
binding = next;
|
||||
loadError = null;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,848 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {createRequire} from 'node:module';
|
||||
import {describe, test} from 'node:test';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const webrtcSender = require('./index.js');
|
||||
|
||||
describe('webrtc-sender loader wrapper', () => {
|
||||
test('resolves the native addon filename for every desktop OS/arch target', () => {
|
||||
const cases = [
|
||||
['win32', 'x64', 'webrtc-sender.win32-x64-msvc.node'],
|
||||
['win32', 'arm64', 'webrtc-sender.win32-arm64-msvc.node'],
|
||||
['darwin', 'x64', 'webrtc-sender.darwin-x64.node'],
|
||||
['darwin', 'arm64', 'webrtc-sender.darwin-arm64.node'],
|
||||
['linux', 'x64', 'webrtc-sender.linux-x64-gnu.node'],
|
||||
['linux', 'arm64', 'webrtc-sender.linux-arm64-gnu.node'],
|
||||
];
|
||||
for (const [platform, arch, expected] of cases) {
|
||||
assert.equal(webrtcSender.__nativeFileNameForTests(platform, arch), expected);
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects unsupported platform/architecture pairs explicitly', () => {
|
||||
assert.throws(() => webrtcSender.__nativeFileNameForTests('linux', 'ia32'), /not supported/);
|
||||
assert.throws(() => webrtcSender.__nativeFileNameForTests('freebsd', 'x64'), /not supported/);
|
||||
});
|
||||
|
||||
test('returns a native hardware encoder capability when the binding exports one', () => {
|
||||
const expected = {
|
||||
available: true,
|
||||
backend: 'nvenc',
|
||||
compiled: true,
|
||||
runtime: true,
|
||||
codecs: ['h264', 'h265'],
|
||||
zeroCopy: true,
|
||||
nativeInputs: ['dmabuf'],
|
||||
};
|
||||
webrtcSender.__setBindingForTests({
|
||||
getHardwareEncoderCapability() {
|
||||
return expected;
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(webrtcSender.getHardwareEncoderCapability(), expected);
|
||||
assert.deepEqual(webrtcSender.getHardwareEncoderCapabilities(), expected);
|
||||
});
|
||||
|
||||
test('delegates dropped video callback metrics through the VoiceEngine wrapper', () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
inboundAudioFrames() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
inboundVideoFrames() {
|
||||
return 3;
|
||||
}
|
||||
|
||||
droppedVideoFrameCallbacks() {
|
||||
return 5;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.equal(engine.inboundAudioFrames(), 2);
|
||||
assert.equal(engine.inboundVideoFrames(), 3);
|
||||
assert.equal(engine.droppedVideoFrameCallbacks(), 5);
|
||||
});
|
||||
|
||||
test('reports VoiceEngine feature capabilities from the wrapped native prototype', () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishDeviceMicrophone() {}
|
||||
publishMicrophone() {}
|
||||
publishCamera() {}
|
||||
listCameraDevices() {}
|
||||
publishScreenShare() {}
|
||||
updateScreenShareEncoding() {}
|
||||
unpublishScreenShare() {}
|
||||
publishScreenShareAudio() {}
|
||||
pushScreenSharePcm() {}
|
||||
pushScreenShareFloat() {}
|
||||
unpublishScreenShareAudio() {}
|
||||
listAudioInputDevices() {}
|
||||
listAudioOutputDevices() {}
|
||||
setAudioOutputDevice() {}
|
||||
setParticipantVolume() {}
|
||||
setRemoteTrackSubscription() {}
|
||||
publishData() {}
|
||||
getConnectionStats() {}
|
||||
setVideoFrameCallback() {}
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(webrtcSender.getCapabilities(), {
|
||||
microphoneCapture: true,
|
||||
syntheticMicrophonePcm: true,
|
||||
cameraCapture: true,
|
||||
nativeCameraBackgrounds: false,
|
||||
screenShare: true,
|
||||
screenShareEncodingUpdate: true,
|
||||
screenShareAudio: true,
|
||||
deviceLists: true,
|
||||
outputDeviceSelection: true,
|
||||
participantVolume: true,
|
||||
remoteTrackSubscription: true,
|
||||
dataChannel: true,
|
||||
connectionStats: true,
|
||||
nativeVideoFrames: true,
|
||||
hardwareEncoderCapabilities: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('does not report screen-share audio capability without the float push method', () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishScreenShareAudio() {}
|
||||
pushScreenSharePcm() {}
|
||||
unpublishScreenShareAudio() {}
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(webrtcSender.getCapabilities().screenShareAudio, false);
|
||||
});
|
||||
|
||||
test('reports native camera background support only from the explicit native probe', () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
hasNativeCameraBackgrounds() {
|
||||
return true;
|
||||
},
|
||||
VoiceEngine: class {
|
||||
publishCamera() {}
|
||||
listCameraDevices() {}
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(webrtcSender.hasNativeCameraBackgrounds(), true);
|
||||
assert.equal(webrtcSender.getCapabilities().nativeCameraBackgrounds, true);
|
||||
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishCamera() {}
|
||||
listCameraDevices() {}
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(webrtcSender.hasNativeCameraBackgrounds(), false);
|
||||
assert.equal(webrtcSender.getCapabilities().nativeCameraBackgrounds, false);
|
||||
});
|
||||
|
||||
test('delegates video-frame callback clearing to the native binding when exported', () => {
|
||||
let clearCalls = 0;
|
||||
const setCalls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
setVideoFrameCallback(callback) {
|
||||
setCalls.push(callback);
|
||||
}
|
||||
|
||||
clearVideoFrameCallback() {
|
||||
clearCalls += 1;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
engine.clearVideoFrameCallback();
|
||||
|
||||
assert.equal(clearCalls, 1);
|
||||
assert.deepEqual(setCalls, []);
|
||||
});
|
||||
|
||||
test('delegates screen-share float audio through the VoiceEngine wrapper', async () => {
|
||||
const calls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
pushScreenShareFloat(buffer, sampleRate, numChannels) {
|
||||
calls.push([buffer, sampleRate, numChannels]);
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
const buffer = Buffer.from(new Float32Array([0.25, -0.25]).buffer);
|
||||
const accepted = await engine.pushScreenShareFloat(buffer, 48000, 2);
|
||||
|
||||
assert.equal(accepted, true);
|
||||
assert.deepEqual(calls, [[buffer, 48000, 2]]);
|
||||
});
|
||||
|
||||
test('falls back to a no-op video-frame callback for older native bindings', () => {
|
||||
const setCalls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
setVideoFrameCallback(callback) {
|
||||
setCalls.push(callback);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
engine.clearVideoFrameCallback();
|
||||
|
||||
assert.equal(setCalls.length, 1);
|
||||
assert.equal(typeof setCalls[0], 'function');
|
||||
assert.equal(setCalls[0]('{}', Buffer.alloc(0)), undefined);
|
||||
});
|
||||
|
||||
test('treats video-frame callback clearing as a no-op when the binding exports neither method', () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
|
||||
assert.equal(engine.clearVideoFrameCallback(), undefined);
|
||||
});
|
||||
|
||||
test('delegates engine bridge version reads and assertions to the native binding', () => {
|
||||
const assertedVersions = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
getEngineBridgeVersion() {
|
||||
return 9;
|
||||
},
|
||||
assertEngineBridgeVersion(version) {
|
||||
assertedVersions.push(version);
|
||||
if (version !== 9) {
|
||||
throw new Error(`voice engine bridge version mismatch: host sent ${version}, native addon expects 9`);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(webrtcSender.getEngineBridgeVersion(), 9);
|
||||
webrtcSender.assertEngineBridgeVersion(9);
|
||||
assert.throws(() => webrtcSender.assertEngineBridgeVersion(8), /bridge version mismatch/);
|
||||
assert.deepEqual(assertedVersions, [9, 8]);
|
||||
});
|
||||
|
||||
test('reports a null engine bridge version and throws on assertion when the binding lacks the exports', () => {
|
||||
webrtcSender.__setBindingForTests({});
|
||||
|
||||
assert.equal(webrtcSender.getEngineBridgeVersion(), null);
|
||||
assert.throws(() => webrtcSender.assertEngineBridgeVersion(9), /does not export assertEngineBridgeVersion/);
|
||||
});
|
||||
|
||||
test('delegates native voice engine prewarm when the binding exports it', () => {
|
||||
let prewarmCalls = 0;
|
||||
webrtcSender.__setBindingForTests({
|
||||
prewarmVoiceEngine() {
|
||||
prewarmCalls += 1;
|
||||
},
|
||||
});
|
||||
|
||||
webrtcSender.prewarmVoiceEngine();
|
||||
|
||||
assert.equal(prewarmCalls, 1);
|
||||
});
|
||||
|
||||
test('delegates device microphone publish through the VoiceEngine wrapper', async () => {
|
||||
const calls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishDeviceMicrophone(opts) {
|
||||
calls.push(opts);
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
await engine.publishDeviceMicrophone({
|
||||
deviceId: 'mic-guid',
|
||||
echoCancellation: false,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: false,
|
||||
});
|
||||
await engine.publishDeviceMicrophone({
|
||||
deviceId: 'mic-guid',
|
||||
echoCancellation: false,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: false,
|
||||
maxBitrateBps: 96_000,
|
||||
});
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
{
|
||||
deviceId: 'mic-guid',
|
||||
echoCancellation: false,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: false,
|
||||
maxBitrateBps: undefined,
|
||||
},
|
||||
{
|
||||
deviceId: 'mic-guid',
|
||||
echoCancellation: false,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: false,
|
||||
maxBitrateBps: 96_000,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('delegates native camera devices and publish options through the VoiceEngine wrapper', async () => {
|
||||
const publishCalls = [];
|
||||
const processedPublishCalls = [];
|
||||
const processedFrameCalls = [];
|
||||
const nativeSinkPublishCalls = [];
|
||||
const nativeSinkHandle = {};
|
||||
const devices = [
|
||||
{
|
||||
deviceId: 'native-camera-id',
|
||||
label: 'Studio Display Camera',
|
||||
description: 'Apple Studio Display Camera',
|
||||
index: 0,
|
||||
deviceIdAliases: ['native-camera-id', '0'],
|
||||
},
|
||||
];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
listCameraDevices() {
|
||||
return devices;
|
||||
}
|
||||
|
||||
publishCamera(opts) {
|
||||
publishCalls.push(opts);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
publishProcessedCamera(opts) {
|
||||
processedPublishCalls.push(opts);
|
||||
return Promise.resolve({trackSid: 'TR_processed_camera'});
|
||||
}
|
||||
|
||||
publishNativeCameraSink(opts) {
|
||||
nativeSinkPublishCalls.push(opts);
|
||||
return Promise.resolve({trackSid: 'TR_native_camera'});
|
||||
}
|
||||
|
||||
createCameraFrameSinkHandle() {
|
||||
return nativeSinkHandle;
|
||||
}
|
||||
|
||||
pushProcessedCameraFrame(frame) {
|
||||
processedFrameCalls.push(frame);
|
||||
return Promise.resolve(true);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.equal(engine.listCameraDevices(), devices);
|
||||
await engine.publishCamera({
|
||||
deviceId: 'native-camera-id',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
mirror: true,
|
||||
codec: 'h265',
|
||||
maxBitrateBps: 6_000_000,
|
||||
maxFramerate: 30,
|
||||
});
|
||||
const processedPublishResult = await engine.publishProcessedCamera({
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
ignored: true,
|
||||
});
|
||||
const nativeSinkPublishResult = await engine.publishNativeCameraSink({
|
||||
deviceId: 'native-camera-id',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
backgroundMode: 'custom',
|
||||
backgroundCustomMediaPath: '/tmp/bg.webp',
|
||||
backgroundCustomMediaKind: 'animated',
|
||||
codec: 'h264',
|
||||
});
|
||||
const cameraFrameSinkHandle = engine.createCameraFrameSinkHandle();
|
||||
const frame = {
|
||||
format: 'i420',
|
||||
width: 4,
|
||||
height: 2,
|
||||
timestampUs: 12_345,
|
||||
data: Buffer.alloc(12),
|
||||
ignored: true,
|
||||
};
|
||||
const processedFrameResult = await engine.pushProcessedCameraFrame(frame);
|
||||
|
||||
assert.deepEqual(publishCalls, [
|
||||
{
|
||||
deviceId: 'native-camera-id',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
mirror: true,
|
||||
backgroundMode: undefined,
|
||||
backgroundCustomMediaPath: undefined,
|
||||
backgroundCustomMediaKind: undefined,
|
||||
backgroundBlurStrength: undefined,
|
||||
codec: 'h265',
|
||||
maxBitrateBps: 6_000_000,
|
||||
maxFramerate: 30,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(processedPublishResult, {trackSid: 'TR_processed_camera'});
|
||||
assert.deepEqual(processedPublishCalls, [
|
||||
{
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(nativeSinkPublishResult, {trackSid: 'TR_native_camera'});
|
||||
assert.deepEqual(nativeSinkPublishCalls, [
|
||||
{
|
||||
deviceId: 'native-camera-id',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
mirror: undefined,
|
||||
backgroundMode: 'custom',
|
||||
backgroundCustomMediaPath: '/tmp/bg.webp',
|
||||
backgroundCustomMediaKind: 'animated',
|
||||
backgroundBlurStrength: undefined,
|
||||
codec: 'h264',
|
||||
maxBitrateBps: undefined,
|
||||
maxFramerate: undefined,
|
||||
},
|
||||
]);
|
||||
assert.equal(cameraFrameSinkHandle, nativeSinkHandle);
|
||||
assert.equal(processedFrameResult, true);
|
||||
assert.deepEqual(processedFrameCalls, [
|
||||
{
|
||||
format: 'i420',
|
||||
width: 4,
|
||||
height: 2,
|
||||
timestampUs: 12_345,
|
||||
data: frame.data,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('reports processed camera publishing as unavailable for older native bindings', async () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.throws(
|
||||
() => engine.publishProcessedCamera({width: 1280, height: 720, frameRate: 30}),
|
||||
/native binding does not export publishProcessedCamera/,
|
||||
);
|
||||
assert.throws(
|
||||
() => engine.publishNativeCameraSink({width: 1280, height: 720, frameRate: 30}),
|
||||
/native binding does not export publishNativeCameraSink/,
|
||||
);
|
||||
assert.equal(engine.createCameraFrameSinkHandle(), null);
|
||||
assert.throws(
|
||||
() =>
|
||||
engine.pushProcessedCameraFrame({
|
||||
format: 'i420',
|
||||
width: 4,
|
||||
height: 2,
|
||||
timestampUs: 12_345,
|
||||
data: Buffer.alloc(12),
|
||||
}),
|
||||
/native binding does not export pushProcessedCameraFrame/,
|
||||
);
|
||||
});
|
||||
|
||||
test('delegates device screen-share publish through the VoiceEngine wrapper', async () => {
|
||||
const publishCalls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishDeviceScreenShare(opts) {
|
||||
publishCalls.push(opts);
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
await engine.publishDeviceScreenShare({
|
||||
deviceId: 'studio-display-camera',
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
frameRate: 60,
|
||||
codec: 'h264',
|
||||
maxBitrateBps: 8_000_000,
|
||||
maxFramerate: 60,
|
||||
});
|
||||
|
||||
assert.deepEqual(publishCalls, [
|
||||
{
|
||||
deviceId: 'studio-display-camera',
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
frameRate: 60,
|
||||
mirror: undefined,
|
||||
backgroundMode: undefined,
|
||||
backgroundCustomMediaPath: undefined,
|
||||
backgroundCustomMediaKind: undefined,
|
||||
backgroundBlurStrength: undefined,
|
||||
codec: 'h264',
|
||||
maxBitrateBps: 8_000_000,
|
||||
maxFramerate: 60,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('delegates camera capture updates with effect strengths through the VoiceEngine wrapper', async () => {
|
||||
const updateCalls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
updateCameraCapture(opts) {
|
||||
updateCalls.push(opts);
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
await engine.updateCameraCapture({
|
||||
deviceId: 'native-camera-id',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
mirror: false,
|
||||
backgroundMode: 'blur',
|
||||
backgroundBlurStrength: 90,
|
||||
ignored: true,
|
||||
});
|
||||
|
||||
assert.deepEqual(updateCalls, [
|
||||
{
|
||||
deviceId: 'native-camera-id',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
frameRate: 30,
|
||||
mirror: false,
|
||||
backgroundMode: 'blur',
|
||||
backgroundCustomMediaPath: undefined,
|
||||
backgroundCustomMediaKind: undefined,
|
||||
backgroundBlurStrength: 90,
|
||||
codec: undefined,
|
||||
maxBitrateBps: undefined,
|
||||
maxFramerate: undefined,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('reports camera capture updates as unavailable for older native bindings', () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.throws(
|
||||
() => engine.updateCameraCapture({deviceId: 'native-camera-id'}),
|
||||
/native binding does not export updateCameraCapture/,
|
||||
);
|
||||
});
|
||||
|
||||
test('reports device screen-share publish as unavailable for older native bindings', async () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.throws(
|
||||
() => engine.publishDeviceScreenShare({deviceId: 'studio-display-camera'}),
|
||||
/native binding does not export publishDeviceScreenShare/,
|
||||
);
|
||||
});
|
||||
|
||||
test('delegates screen-share simulcast selection through the VoiceEngine wrapper', async () => {
|
||||
const calls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishScreenShare(width, height, codec, maxBitrateBps, maxFramerate, simulcast, options) {
|
||||
calls.push({width, height, codec, maxBitrateBps, maxFramerate, simulcast, options});
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
await engine.publishScreenShare(3840, 2160, 'h265', 50_000_000, 60, false, {
|
||||
adaptiveSend: true,
|
||||
minVideoFps: 15,
|
||||
minResolutionScale: 0.5,
|
||||
maxAudioBufferMs: 750,
|
||||
captureId: 'screen-harness-primary',
|
||||
});
|
||||
assert.deepEqual(calls, [
|
||||
{
|
||||
width: 3840,
|
||||
height: 2160,
|
||||
codec: 'h265',
|
||||
maxBitrateBps: 50_000_000,
|
||||
maxFramerate: 60,
|
||||
simulcast: false,
|
||||
options: {
|
||||
adaptiveSend: true,
|
||||
minVideoFps: 15,
|
||||
minResolutionScale: 0.5,
|
||||
maxAudioBufferMs: 750,
|
||||
captureId: 'screen-harness-primary',
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects screen-share publish and encoding update without capture IDs', async () => {
|
||||
const calls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishScreenShare(...args) {
|
||||
calls.push(['publish', args]);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
updateScreenShareEncoding(...args) {
|
||||
calls.push(['update', args]);
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.throws(
|
||||
() => engine.publishScreenShare(1280, 720, 'h264', undefined, 30, false, undefined),
|
||||
/screen-share publish requires a non-empty captureId/,
|
||||
);
|
||||
assert.throws(
|
||||
() => engine.publishScreenShare(1280, 720, 'h264', undefined, 30, false, {captureId: ''}),
|
||||
/screen-share publish requires a non-empty captureId/,
|
||||
);
|
||||
assert.throws(
|
||||
() => engine.updateScreenShareEncoding(1280, 720, undefined, 30, undefined),
|
||||
/screen-share encoding update requires a non-empty captureId/,
|
||||
);
|
||||
assert.throws(
|
||||
() => engine.updateScreenShareEncoding(1280, 720, undefined, 30, {captureId: ''}),
|
||||
/screen-share encoding update requires a non-empty captureId/,
|
||||
);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test('delegates connect options through the VoiceEngine wrapper', async () => {
|
||||
const calls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
connect(url, token, e2eeKey, options) {
|
||||
calls.push({url, token, e2eeKey, options});
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
const key = Buffer.from('secret');
|
||||
await engine.connect('ws://localhost:7880', 'token', key, {
|
||||
autoSubscribe: false,
|
||||
adaptiveStream: true,
|
||||
dynacast: true,
|
||||
});
|
||||
assert.deepEqual(calls, [
|
||||
{
|
||||
url: 'ws://localhost:7880',
|
||||
token: 'token',
|
||||
e2eeKey: key,
|
||||
options: {
|
||||
autoSubscribe: false,
|
||||
adaptiveStream: true,
|
||||
dynacast: true,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('delegates native screen frame sink handle creation through the VoiceEngine wrapper', () => {
|
||||
const calls = [];
|
||||
const handle = {native: true};
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
createScreenFrameSinkHandle(captureId) {
|
||||
calls.push(captureId);
|
||||
return handle;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.equal(engine.createScreenFrameSinkHandle('capture-1'), handle);
|
||||
assert.deepEqual(calls, ['capture-1']);
|
||||
});
|
||||
|
||||
test('treats native screen frame sink handles as unavailable for older voice bindings', () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.equal(engine.createScreenFrameSinkHandle('capture-1'), null);
|
||||
});
|
||||
|
||||
test('defaults dropped video callback metrics to zero for older native bindings', () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
inboundAudioFrames() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
inboundVideoFrames() {
|
||||
return 3;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
assert.equal(engine.droppedVideoFrameCallbacks(), 0);
|
||||
});
|
||||
|
||||
test('delegates remote track subscription updates through the VoiceEngine wrapper', async () => {
|
||||
const calls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
setRemoteTrackSubscription(participantIdentity, source, subscribed, enabled, quality) {
|
||||
calls.push({participantIdentity, source, subscribed, enabled, quality});
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
await engine.setRemoteTrackSubscription({
|
||||
participantIdentity: 'user_1_conn',
|
||||
source: 'screen_share',
|
||||
subscribed: true,
|
||||
enabled: false,
|
||||
quality: 'high',
|
||||
});
|
||||
|
||||
assert.deepEqual(calls, [
|
||||
{
|
||||
participantIdentity: 'user_1_conn',
|
||||
source: 'screen_share',
|
||||
subscribed: true,
|
||||
enabled: false,
|
||||
quality: 'high',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('ignores remote track subscription updates for older native bindings', async () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
await engine.setRemoteTrackSubscription({
|
||||
participantIdentity: 'user_1_conn',
|
||||
source: 'camera',
|
||||
subscribed: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('delegates data packets through the VoiceEngine wrapper', async () => {
|
||||
const calls = [];
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishData(payload, reliable, topic, destinationIdentities) {
|
||||
calls.push({payload, reliable, topic, destinationIdentities});
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
await engine.publishData(new Uint8Array([1, 2, 3]), {
|
||||
reliable: true,
|
||||
topic: 'screen-share-codec',
|
||||
destinationIdentities: ['user_1_conn'],
|
||||
});
|
||||
|
||||
assert.equal(Buffer.isBuffer(calls[0].payload), true);
|
||||
assert.deepEqual([...calls[0].payload], [1, 2, 3]);
|
||||
assert.deepEqual(calls, [
|
||||
{
|
||||
payload: calls[0].payload,
|
||||
reliable: true,
|
||||
topic: 'screen-share-codec',
|
||||
destinationIdentities: ['user_1_conn'],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects invalid data packet payloads in the wrapper', async () => {
|
||||
webrtcSender.__setBindingForTests({
|
||||
VoiceEngine: class {
|
||||
publishData() {
|
||||
return Promise.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const engine = new webrtcSender.VoiceEngine();
|
||||
await assert.rejects(() => engine.publishData('not-bytes'), /payload must be/);
|
||||
});
|
||||
|
||||
test('returns an unavailable hardware encoder capability without a binding', () => {
|
||||
webrtcSender.__setBindingForTests(null);
|
||||
|
||||
assert.deepEqual(webrtcSender.getHardwareEncoderCapability(), {
|
||||
available: false,
|
||||
backend: 'none',
|
||||
compiled: false,
|
||||
runtime: false,
|
||||
codecs: [],
|
||||
zeroCopy: false,
|
||||
nativeInputs: [],
|
||||
reason: 'native_binding_unavailable',
|
||||
detail: '@fluxer/webrtc-sender binding unavailable',
|
||||
});
|
||||
});
|
||||
|
||||
test('exposes every VoiceEngine method declared in index.d.ts on the wrapper', async () => {
|
||||
const {readFileSync} = await import('node:fs');
|
||||
const dts = readFileSync(new URL('./index.d.ts', import.meta.url), 'utf8');
|
||||
const classStart = dts.indexOf('export declare class VoiceEngine {');
|
||||
assert.ok(classStart >= 0);
|
||||
const classEnd = dts.indexOf('\n}', classStart);
|
||||
assert.ok(classEnd > classStart);
|
||||
const classBody = dts.slice(classStart, classEnd);
|
||||
const declaredMethods = [...classBody.matchAll(/^\t([A-Za-z0-9_]+)\(/gm)]
|
||||
.map((match) => match[1])
|
||||
.filter((name) => name !== 'constructor');
|
||||
assert.ok(declaredMethods.length >= 30);
|
||||
const wrapperMethods = new Set(Object.getOwnPropertyNames(webrtcSender.VoiceEngine.prototype));
|
||||
const missing = declaredMethods.filter((name) => !wrapperMethods.has(name));
|
||||
assert.deepEqual(missing, []);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
# Bundled segmentation models
|
||||
|
||||
## selfie_segmenter_landscape.onnx
|
||||
|
||||
- **Source**: Google MediaPipe Selfie Segmenter (landscape), `selfie_segmenter_landscape.tflite`, downloaded from `https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite` (sha256 `490e9ea734313e0de10fa0cd9e3c6133e36ea4db2b7a49bde9ef019f72796b8e`).
|
||||
- **License**: Apache License 2.0, per the official model card ("Model Card MediaPipe Selfie Segmentation", Google, 2021; `https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Selfie%20Segmentation.pdf`). This is the Apache-licensed Selfie model, not the ToS-restricted Google Meet model (`segm_full_v679.tflite`), which must never be shipped.
|
||||
- **Conversion**: `tf2onnx` (`python -m tf2onnx.convert --tflite selfie_segmenter_landscape.tflite --output model.onnx --opset 13`), followed by graph surgery that replaces the single `TFL_Convolution2DTransposeBias` custom op with a standard `ConvTranspose` (weights transposed from `[out, kh, kw, in]` to `[in, out, kh, kw]`, strides 2x2, no padding) wrapped in NHWC/NCHW transposes, and bumps the default opset domain to 14 for `HardSwish`.
|
||||
- **Verification**: output of the converted model matches the original tflite interpreter to a max abs diff of 8.7e-8 on random input. On a portrait test image the output is person confidence (1.0 on the subject, 0.0 in background corners), despite the output tensor name `segment_back`.
|
||||
- **Signature**: input `input_1` `[1, 144, 256, 3]` f32 RGB scaled to 0..1, output `segment_back` `[1, 144, 256, 1]` f32 person confidence 0..1.
|
||||
- **sha256**: `e8224061bba6031282bfd00cf23a2563fa11a1e28baae0a9052cef6b4e7f3321`
|
||||
Binary file not shown.
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"name": "@fluxer/webrtc-sender",
|
||||
"version": "0.0.0",
|
||||
"description": "Native main-process WebRTC video sender (LiveKit/libwebrtc) for publishing game/screen capture without a renderer hop",
|
||||
"private": true,
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"os": [
|
||||
"darwin",
|
||||
"linux",
|
||||
"win32"
|
||||
],
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
],
|
||||
"main": "index.js",
|
||||
"types": "index.d.ts",
|
||||
"files": [
|
||||
"index.js",
|
||||
"index.d.ts",
|
||||
"webrtc-sender.darwin-x64.node",
|
||||
"webrtc-sender.darwin-arm64.node",
|
||||
"webrtc-sender.linux-x64-gnu.node",
|
||||
"webrtc-sender.linux-arm64-gnu.node",
|
||||
"webrtc-sender.win32-x64-msvc.node",
|
||||
"webrtc-sender.win32-arm64-msvc.node"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "cargo run --locked --quiet --manifest-path ../../../tools/ci/Cargo.toml -- build-desktop-native-addon",
|
||||
"test": "cargo run --locked --quiet --manifest-path ../../../tools/ci/Cargo.toml -- test-webrtc-sender-rust && node --test index.test.mjs scripts/livekit-harness.test.mjs",
|
||||
"test:livekit": "node scripts/livekit-harness.mjs"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,842 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {describe, test} from 'node:test';
|
||||
import {
|
||||
buildReport,
|
||||
buildScenarioConfigs,
|
||||
createLiveKitAdminToken,
|
||||
expectedCodecMime,
|
||||
jwtSubject,
|
||||
liveKitApiUrl,
|
||||
liveKitTcpTarget,
|
||||
parseCodecList,
|
||||
parseConfig,
|
||||
sanitizeConfig,
|
||||
scenarioRoomName,
|
||||
serverPublishedTrackChecks,
|
||||
statsHasOutbound,
|
||||
statsHasOutboundCodec,
|
||||
strictFailures,
|
||||
} from './livekit-harness.mjs';
|
||||
|
||||
function tokenForSubject(subject) {
|
||||
const payload = Buffer.from(JSON.stringify({sub: subject}), 'utf8').toString('base64url');
|
||||
return `header.${payload}.signature`;
|
||||
}
|
||||
|
||||
function jwtPayload(token) {
|
||||
const parts = token.split('.');
|
||||
return JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
|
||||
}
|
||||
|
||||
async function withHarnessEnv(env, callback) {
|
||||
const previous = {...process.env};
|
||||
for (const key of Object.keys(process.env)) {
|
||||
if (
|
||||
key.startsWith('LIVEKIT_') ||
|
||||
key.startsWith('FLUXER_WEBRTC_SENDER_LIVEKIT') ||
|
||||
key.startsWith('FLUXER_NATIVE_MEDIA')
|
||||
) {
|
||||
delete process.env[key];
|
||||
}
|
||||
}
|
||||
Object.assign(process.env, env);
|
||||
try {
|
||||
return await callback();
|
||||
} finally {
|
||||
for (const key of Object.keys(process.env)) {
|
||||
delete process.env[key];
|
||||
}
|
||||
Object.assign(process.env, previous);
|
||||
}
|
||||
}
|
||||
|
||||
function strictReportFor(
|
||||
config,
|
||||
{
|
||||
videoFrameTimes = null,
|
||||
videoFrameRecords = null,
|
||||
audioFrameTimes = [],
|
||||
droppedVideoFrameCallbacks = 0,
|
||||
subscriberStatsSamples = [],
|
||||
} = {},
|
||||
) {
|
||||
const strictStartedAtMs = 1_000;
|
||||
const strictEndedAtMs = 2_000;
|
||||
const effectiveVideoFrameRecords =
|
||||
videoFrameRecords ??
|
||||
(videoFrameTimes ?? []).map((atMs) => ({
|
||||
atMs,
|
||||
identity: config.publisherIdentity,
|
||||
kind: 'video',
|
||||
source: 'screen_share',
|
||||
trackName: 'screen',
|
||||
trackSid: 'TR_screen',
|
||||
width: config.expectedWidth,
|
||||
height: config.expectedHeight,
|
||||
}));
|
||||
const effectiveVideoFrameTimes = videoFrameTimes ?? effectiveVideoFrameRecords.map((record) => record.atMs);
|
||||
return buildReport({
|
||||
status: 'pass',
|
||||
config,
|
||||
context: {
|
||||
startedAtMs: strictStartedAtMs - 500,
|
||||
startedAtIso: new Date(strictStartedAtMs - 500).toISOString(),
|
||||
strictStartedAtMs,
|
||||
strictEndedAtMs,
|
||||
publisherState: {statsSamples: [], stats: null},
|
||||
subscriberState: {
|
||||
statsSamples: subscriberStatsSamples,
|
||||
stats: subscriberStatsSamples.at(-1)?.payload ?? null,
|
||||
videoCallbacks: effectiveVideoFrameTimes.length,
|
||||
videoBytes: effectiveVideoFrameTimes.length * 4,
|
||||
videoFrameTimes: effectiveVideoFrameTimes,
|
||||
videoFrameRecords: effectiveVideoFrameRecords,
|
||||
audioFrameTimes,
|
||||
lastVideoMeta: effectiveVideoFrameRecords.at(-1) ?? null,
|
||||
lastVideoFrameAtMs: effectiveVideoFrameTimes.at(-1) ?? null,
|
||||
lastAudioFrameAtMs: audioFrameTimes.at(-1) ?? null,
|
||||
},
|
||||
subscriber: {
|
||||
inboundAudioFrames: () => audioFrameTimes.length,
|
||||
inboundVideoFrames: () => effectiveVideoFrameTimes.length,
|
||||
droppedVideoFrameCallbacks: () => droppedVideoFrameCallbacks,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function findCheck(checks, name) {
|
||||
const check = checks.find((entry) => entry.name === name);
|
||||
assert.ok(check, `missing check: ${name}`);
|
||||
return check;
|
||||
}
|
||||
|
||||
function serverChecksFor(config, serverState) {
|
||||
return serverPublishedTrackChecks({
|
||||
config,
|
||||
serverState,
|
||||
publisherIdentity: config.publisherIdentity,
|
||||
secondaryPublisherIdentity: config.secondaryPublisherIdentity,
|
||||
});
|
||||
}
|
||||
|
||||
describe('livekit harness setup matrix', () => {
|
||||
test('normalizes LiveKit screenshare source spelling in outbound stats', () => {
|
||||
const stats = {
|
||||
outbound: [
|
||||
{
|
||||
trackSid: 'TR_screen',
|
||||
source: 'screenshare',
|
||||
kind: 'video',
|
||||
codec: 'video/VP8',
|
||||
bitrateKbps: 400,
|
||||
packetsLost: 0,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
assert.equal(statsHasOutbound(stats, 'video', 'screen_share'), true);
|
||||
assert.equal(statsHasOutboundCodec(stats, 'video', 'screen_share', 'video/VP8'), true);
|
||||
assert.equal(statsHasOutboundCodec(stats, 'video', 'screen_share', 'video/H264'), false);
|
||||
});
|
||||
|
||||
test('maps all supported codec spellings to expected mime types', () => {
|
||||
assert.equal(expectedCodecMime('vp8'), 'video/VP8');
|
||||
assert.equal(expectedCodecMime('VP9'), 'video/VP9');
|
||||
assert.equal(expectedCodecMime('h264'), 'video/H264');
|
||||
assert.equal(expectedCodecMime('av1'), 'video/AV1');
|
||||
assert.equal(expectedCodecMime('h265'), 'video/H265');
|
||||
assert.equal(expectedCodecMime('hevc'), 'video/H265');
|
||||
assert.equal(expectedCodecMime(''), null);
|
||||
assert.throws(() => expectedCodecMime('h266'), /unsupported/);
|
||||
});
|
||||
|
||||
test('parses codec lists with fallback and rejects empty effective lists', () => {
|
||||
assert.deepEqual(parseCodecList(' vp8, h264,, hevc ', 'av1'), ['vp8', 'h264', 'hevc']);
|
||||
assert.deepEqual(parseCodecList('', 'vp9'), ['vp9']);
|
||||
assert.throws(() => parseCodecList('', ''), /at least one/);
|
||||
assert.throws(() => parseCodecList('vp8,h266', 'vp8'), /unsupported/);
|
||||
});
|
||||
|
||||
test('default non-strict config keeps the fast local live setup small', async () => {
|
||||
await withHarnessEnv({}, () => {
|
||||
const config = parseConfig();
|
||||
assert.equal(config.url, 'ws://localhost:7880');
|
||||
assert.equal(config.serverApiUrl, 'http://localhost:7880');
|
||||
assert.deepEqual(config.screenCodecs, ['vp8']);
|
||||
assert.equal(config.expectedScreenCodec, 'video/VP8');
|
||||
assert.equal(config.secondaryPublisher, false);
|
||||
assert.equal(config.microphone, true);
|
||||
assert.equal(config.screenAudio, false);
|
||||
assert.equal(config.dataPacket, false);
|
||||
assert.equal(config.subscriptionCycle, false);
|
||||
assert.equal(config.validateServerPublishing, false);
|
||||
assert.equal(config.durationMs, 0);
|
||||
assert.equal(config.externalTokens, false);
|
||||
assert.equal(config.screenSimulcast, true);
|
||||
assert.equal(config.secondaryPublisherScreenSimulcast, true);
|
||||
assert.equal(config.adaptiveSend, true);
|
||||
assert.equal(config.minVideoFps, 15);
|
||||
assert.equal(config.minResolutionScale, 0.5);
|
||||
assert.equal(config.maxAudioBufferMs, 750);
|
||||
});
|
||||
});
|
||||
|
||||
test('strict config expands to multi-codec secondary-publisher scenarios', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_HARNESS_STRICT: '1',
|
||||
LIVEKIT_ROOM: 'matrix-room',
|
||||
LIVEKIT_SCREEN_CODECS: 'vp8,h264,hevc',
|
||||
LIVEKIT_EXPECT_SCREEN_CODECS: 'vp8,h264,h265',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
assert.equal(config.secondaryPublisher, true);
|
||||
assert.equal(config.screenAudio, true);
|
||||
assert.equal(config.dataPacket, true);
|
||||
assert.equal(config.subscriptionCycle, true);
|
||||
assert.equal(config.validateServerPublishing, true);
|
||||
assert.equal(config.durationMs, 10 * 60 * 1000);
|
||||
assert.equal(config.maxAudioFrameGapMs, 250);
|
||||
assert.equal(config.maxPacketLoss, null);
|
||||
assert.equal(config.requireStableResolution, true);
|
||||
assert.equal(config.videoPattern, 'gradient');
|
||||
|
||||
const scenarios = buildScenarioConfigs(config);
|
||||
assert.deepEqual(
|
||||
scenarios.map((scenario) => scenario.scenarioName),
|
||||
['codec-vp8-with-h264', 'codec-h264-with-hevc', 'codec-hevc-with-vp8'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
scenarios.map((scenario) => scenario.expectedScreenCodec),
|
||||
['video/VP8', 'video/H264', 'video/H265'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
scenarios.map((scenario) => scenario.expectedSecondaryPublisherCodec),
|
||||
['video/H264', 'video/H265', 'video/VP8'],
|
||||
);
|
||||
assert.equal(scenarios[0].room, 'matrix-room-codec-vp8-with-h264');
|
||||
assert.equal(scenarios[1].reportPath, null);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('stress knobs parse packet loss, resolution stability, audio gap, and video pattern', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_MAX_PACKET_LOSS: '0',
|
||||
LIVEKIT_REQUIRE_STABLE_RESOLUTION: '0',
|
||||
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '180',
|
||||
LIVEKIT_VIDEO_PATTERN: 'fast',
|
||||
LIVEKIT_VIDEO_INPUT: 'nv12',
|
||||
LIVEKIT_SUBSCRIBER_VIDEO_QUALITY: 'high',
|
||||
LIVEKIT_SCREEN_SIMULCAST: '0',
|
||||
LIVEKIT_SECOND_PUBLISHER_SCREEN_SIMULCAST: '1',
|
||||
LIVEKIT_SCREEN_FPS: '60',
|
||||
LIVEKIT_ADAPTIVE_SEND: '0',
|
||||
LIVEKIT_MIN_VIDEO_FPS: '24',
|
||||
LIVEKIT_MIN_RESOLUTION_SCALE: '0.75',
|
||||
LIVEKIT_MAX_AUDIO_BUFFER_MS: '640',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
assert.equal(config.maxPacketLoss, 0);
|
||||
assert.equal(config.requireStableResolution, false);
|
||||
assert.equal(config.maxAudioFrameGapMs, 180);
|
||||
assert.equal(config.videoPattern, 'fast');
|
||||
assert.equal(config.videoInput, 'nv12');
|
||||
assert.equal(config.subscriberVideoQuality, 'high');
|
||||
assert.equal(config.screenSimulcast, false);
|
||||
assert.equal(config.secondaryPublisherScreenSimulcast, true);
|
||||
assert.equal(config.adaptiveSend, false);
|
||||
assert.equal(config.minVideoFps, 24);
|
||||
assert.equal(config.minResolutionScale, 0.75);
|
||||
assert.equal(config.maxAudioBufferMs, 640);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('stress knobs reject impossible send pacing configurations', async () => {
|
||||
await withHarnessEnv({LIVEKIT_SCREEN_FPS: '0'}, () =>
|
||||
assert.throws(() => parseConfig(), /LIVEKIT_SCREEN_FPS must be a positive number/),
|
||||
);
|
||||
await withHarnessEnv({LIVEKIT_MIN_VIDEO_FPS: '0'}, () =>
|
||||
assert.throws(() => parseConfig(), /LIVEKIT_MIN_VIDEO_FPS must be a positive number/),
|
||||
);
|
||||
await withHarnessEnv({LIVEKIT_SCREEN_FPS: '30', LIVEKIT_MIN_VIDEO_FPS: '60'}, () =>
|
||||
assert.throws(() => parseConfig(), /MIN_VIDEO_FPS must be less than or equal/),
|
||||
);
|
||||
await withHarnessEnv({LIVEKIT_MAX_AUDIO_BUFFER_MS: '0'}, () =>
|
||||
assert.throws(() => parseConfig(), /LIVEKIT_MAX_AUDIO_BUFFER_MS must be a positive integer/),
|
||||
);
|
||||
await withHarnessEnv({LIVEKIT_MIN_RESOLUTION_SCALE: '0'}, () =>
|
||||
assert.throws(() => parseConfig(), /LIVEKIT_MIN_RESOLUTION_SCALE must be a positive number/),
|
||||
);
|
||||
await withHarnessEnv({LIVEKIT_MIN_RESOLUTION_SCALE: '1.5'}, () =>
|
||||
assert.throws(() => parseConfig(), /LIVEKIT_MIN_RESOLUTION_SCALE must be greater than 0/),
|
||||
);
|
||||
await withHarnessEnv({LIVEKIT_ENABLE_SCREEN_AUDIO: 'maybe'}, () =>
|
||||
assert.throws(() => parseConfig(), /LIVEKIT_ENABLE_SCREEN_AUDIO must be a boolean flag/),
|
||||
);
|
||||
await withHarnessEnv({LIVEKIT_SCREEN_CODECS: 'vp8,h264', LIVEKIT_EXPECT_SCREEN_CODECS: 'vp8'}, () =>
|
||||
assert.throws(() => parseConfig(), /EXPECT_SCREEN_CODECS length must match/),
|
||||
);
|
||||
});
|
||||
|
||||
test('explicit secondary codec expectations override rotating defaults', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
|
||||
LIVEKIT_SCREEN_CODECS: 'vp8,h264',
|
||||
LIVEKIT_SECOND_PUBLISHER_CODEC: 'av1',
|
||||
LIVEKIT_EXPECT_SECOND_PUBLISHER_SCREEN_CODEC: 'hevc',
|
||||
},
|
||||
() => {
|
||||
const scenarios = buildScenarioConfigs(parseConfig());
|
||||
assert.deepEqual(
|
||||
scenarios.map((scenario) => scenario.secondaryPublisherCodec),
|
||||
['av1', 'av1'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
scenarios.map((scenario) => scenario.expectedSecondaryPublisherCodec),
|
||||
['video/H265', 'video/H265'],
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('secondary screen codec aliases are accepted and conflicting aliases are rejected', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
|
||||
LIVEKIT_SCREEN_CODECS: 'h264',
|
||||
LIVEKIT_SECOND_PUBLISHER_SCREEN_CODECS: 'vp8',
|
||||
LIVEKIT_EXPECT_SECONDARY_PUBLISHER_SCREEN_CODEC: 'vp8',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
assert.equal(config.secondaryPublisherCodec, 'vp8');
|
||||
assert.equal(config.expectedSecondaryPublisherCodec, 'video/VP8');
|
||||
},
|
||||
);
|
||||
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
|
||||
LIVEKIT_SECOND_PUBLISHER_CODEC: 'h264',
|
||||
LIVEKIT_SECOND_PUBLISHER_SCREEN_CODEC: 'vp8',
|
||||
},
|
||||
() => assert.throws(() => parseConfig(), /conflicts with LIVEKIT_SECOND_PUBLISHER_CODEC/),
|
||||
);
|
||||
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
|
||||
LIVEKIT_EXPECT_SECOND_PUBLISHER_SCREEN_CODEC: 'h264',
|
||||
LIVEKIT_EXPECT_SECONDARY_PUBLISHER_SCREEN_CODEC: 'vp8',
|
||||
},
|
||||
() => assert.throws(() => parseConfig(), /conflicts with LIVEKIT_EXPECT_SECOND_PUBLISHER_SCREEN_CODEC/),
|
||||
);
|
||||
});
|
||||
|
||||
test('external token setup derives identities, requires matching tokens, and redacts reports', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_PUBLISHER_TOKEN: tokenForSubject('publisher-subject'),
|
||||
},
|
||||
() => assert.throws(() => parseConfig(), /both required/),
|
||||
);
|
||||
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_ROOM: 'shared-token-room',
|
||||
LIVEKIT_SCREEN_CODECS: 'vp8,h264',
|
||||
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
|
||||
LIVEKIT_PUBLISHER_TOKEN: tokenForSubject('publisher-subject'),
|
||||
LIVEKIT_SUBSCRIBER_TOKEN: tokenForSubject('subscriber-subject'),
|
||||
},
|
||||
() => assert.throws(() => parseConfig(), /SECONDARY_PUBLISHER_TOKEN/),
|
||||
);
|
||||
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_ROOM: 'shared-token-room',
|
||||
LIVEKIT_SCREEN_CODECS: 'vp8,h264',
|
||||
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
|
||||
LIVEKIT_PUBLISHER_TOKEN: tokenForSubject('publisher-subject'),
|
||||
LIVEKIT_SUBSCRIBER_TOKEN: tokenForSubject('subscriber-subject'),
|
||||
LIVEKIT_SECONDARY_PUBLISHER_TOKEN: tokenForSubject('secondary-subject'),
|
||||
LIVEKIT_E2EE_KEY: 'secret-key',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
assert.equal(config.externalTokens, true);
|
||||
assert.equal(config.publisherIdentity, 'publisher-subject');
|
||||
assert.equal(config.subscriberIdentity, 'subscriber-subject');
|
||||
assert.equal(config.secondaryPublisherIdentity, 'secondary-subject');
|
||||
assert.deepEqual(
|
||||
buildScenarioConfigs(config).map((scenario) => scenario.room),
|
||||
['shared-token-room', 'shared-token-room'],
|
||||
);
|
||||
|
||||
const sanitized = sanitizeConfig(config);
|
||||
assert.equal(sanitized.url, 'ws://localhost:7880');
|
||||
assert.equal(sanitized.apiSecret, '<redacted>');
|
||||
assert.equal(sanitized.e2eeKey, '<present>');
|
||||
assert.equal(sanitized.publisherToken, '<present>');
|
||||
assert.equal(sanitized.subscriberToken, '<present>');
|
||||
assert.equal(sanitized.secondaryPublisherToken, '<present>');
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('report sanitization strips URL credentials, query strings, and fragments', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_URL: 'wss://user:password@example.test:443/rtc?access_token=secret#fragment',
|
||||
},
|
||||
() => {
|
||||
const sanitized = sanitizeConfig(parseConfig());
|
||||
assert.equal(sanitized.url, 'wss://example.test/rtc');
|
||||
assert.doesNotMatch(JSON.stringify(sanitized), /user|password|access_token|secret|fragment/);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('jwt subject and TCP target parsing catch malformed live setups early', () => {
|
||||
assert.equal(jwtSubject(tokenForSubject('user_1_connection')), 'user_1_connection');
|
||||
assert.throws(() => jwtSubject('not-a-jwt'), /must be a JWT/);
|
||||
assert.throws(() => jwtSubject('header.bad-json.signature'), /failed to decode/);
|
||||
assert.throws(() => jwtSubject(tokenForSubject('')), /does not contain/);
|
||||
|
||||
assert.deepEqual(liveKitTcpTarget('ws://localhost:7880'), {host: 'localhost', port: 7880});
|
||||
assert.deepEqual(liveKitTcpTarget('wss://[::1]/rtc'), {host: '::1', port: 443});
|
||||
assert.equal(liveKitApiUrl('ws://localhost:7880/rtc?token=secret'), 'http://localhost:7880');
|
||||
assert.equal(liveKitApiUrl('wss://livekit.example.test/rtc'), 'https://livekit.example.test');
|
||||
assert.throws(() => liveKitTcpTarget('https://localhost'), /must use ws/);
|
||||
});
|
||||
|
||||
test('admin token has the room-scoped grant needed for server publishing validation', () => {
|
||||
const token = createLiveKitAdminToken({apiKey: 'devkey', apiSecret: 'secret', room: 'room-a'});
|
||||
const payload = jwtPayload(token);
|
||||
assert.equal(payload.iss, 'devkey');
|
||||
assert.equal(payload.video.room, 'room-a');
|
||||
assert.equal(payload.video.roomAdmin, true);
|
||||
assert.equal(payload.video.roomJoin, undefined);
|
||||
});
|
||||
|
||||
test('scenario room names are stable and bounded for generated-token suites', () => {
|
||||
assert.equal(scenarioRoomName('room', 'codec-vp8', 1, false), 'room');
|
||||
assert.equal(scenarioRoomName('room', 'codec:vP8 with H264', 2, false), 'room-codec-vP8-with-H264');
|
||||
assert.equal(scenarioRoomName('room', 'codec-vp8', 2, true), 'room');
|
||||
|
||||
const longRoom = scenarioRoomName('r'.repeat(120), 'codec-h264-with-hevc', 2, false);
|
||||
assert.equal(longRoom.length, 128);
|
||||
assert.ok(longRoom.startsWith('r'.repeat(120)));
|
||||
});
|
||||
|
||||
test('strict report checks fail when the strict window has too few samples to measure gaps', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_MIN_RECEIVED_FPS_RATIO: '0',
|
||||
LIVEKIT_MAX_FRAME_GAP_MS: '250',
|
||||
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '250',
|
||||
LIVEKIT_MAX_AV_DRIFT_MS: '0',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
const report = strictReportFor(config, {
|
||||
videoFrameTimes: [1_250],
|
||||
audioFrameTimes: [1_300],
|
||||
});
|
||||
assert.equal(report.metrics.strictVideoFrames, 1);
|
||||
assert.equal(report.metrics.strictAudioFrames, 1);
|
||||
assert.match(strictFailures(report, config).join('\n'), /video frame gap unavailable/);
|
||||
assert.match(strictFailures(report, config).join('\n'), /audio frame gap unavailable/);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('strict report checks use only samples inside the measured window', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_MIN_RECEIVED_FPS_RATIO: '0',
|
||||
LIVEKIT_MAX_FRAME_GAP_MS: '250',
|
||||
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '250',
|
||||
LIVEKIT_MAX_AV_DRIFT_MS: '0',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
const report = strictReportFor(config, {
|
||||
videoFrameTimes: [500, 1_050, 1_200, 2_500],
|
||||
audioFrameTimes: [600, 1_100, 1_240, 2_600],
|
||||
});
|
||||
assert.equal(report.metrics.strictVideoFrames, 2);
|
||||
assert.equal(report.metrics.strictAudioFrames, 2);
|
||||
assert.equal(report.metrics.maxVideoFrameGapMs, 150);
|
||||
assert.equal(report.metrics.maxAudioFrameGapMs, 140);
|
||||
assert.deepEqual(strictFailures(report, config), []);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('strict screen report ignores camera video callbacks for resolution and drift', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_ENABLE_CAMERA: '1',
|
||||
LIVEKIT_MIN_RECEIVED_FPS_RATIO: '0',
|
||||
LIVEKIT_MAX_FRAME_GAP_MS: '250',
|
||||
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '250',
|
||||
LIVEKIT_MAX_AV_DRIFT_MS: '80',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
const report = strictReportFor(config, {
|
||||
videoFrameRecords: [
|
||||
{
|
||||
atMs: 1_010,
|
||||
identity: config.publisherIdentity,
|
||||
kind: 'video',
|
||||
source: 'screen_share',
|
||||
trackName: 'screen',
|
||||
trackSid: 'TR_screen',
|
||||
width: 320,
|
||||
height: 180,
|
||||
},
|
||||
{
|
||||
atMs: 1_025,
|
||||
identity: config.publisherIdentity,
|
||||
kind: 'video',
|
||||
source: 'camera',
|
||||
trackName: 'camera',
|
||||
trackSid: 'TR_camera',
|
||||
width: 480,
|
||||
height: 360,
|
||||
},
|
||||
{
|
||||
atMs: 1_080,
|
||||
identity: config.publisherIdentity,
|
||||
kind: 'video',
|
||||
source: 'screen_share',
|
||||
trackName: 'screen',
|
||||
trackSid: 'TR_screen',
|
||||
width: 320,
|
||||
height: 180,
|
||||
},
|
||||
{
|
||||
atMs: 1_220,
|
||||
identity: config.publisherIdentity,
|
||||
kind: 'video',
|
||||
source: 'camera',
|
||||
trackName: 'camera',
|
||||
trackSid: 'TR_camera',
|
||||
width: 480,
|
||||
height: 360,
|
||||
},
|
||||
],
|
||||
audioFrameTimes: [1_040, 1_100],
|
||||
});
|
||||
|
||||
assert.equal(report.metrics.videoCallbacks, 4);
|
||||
assert.equal(report.metrics.strictVideoFrames, 2);
|
||||
assert.deepEqual(report.metrics.videoResolutionCounts, {'320x180': 2});
|
||||
assert.equal(report.metrics.videoResolutionMismatchCount, 0);
|
||||
assert.equal(report.metrics.avDriftMs, 20);
|
||||
assert.deepEqual(strictFailures(report, config), []);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('strict packet loss gate uses loss deltas inside the strict window', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_MIN_RECEIVED_FPS_RATIO: '0',
|
||||
LIVEKIT_MAX_FRAME_GAP_MS: '0',
|
||||
LIVEKIT_MAX_AUDIO_FRAME_GAP_MS: '0',
|
||||
LIVEKIT_MAX_AV_DRIFT_MS: '0',
|
||||
LIVEKIT_MAX_PACKET_LOSS: '0',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
const report = strictReportFor(config, {
|
||||
videoFrameTimes: [1_100, 1_200],
|
||||
subscriberStatsSamples: [
|
||||
{atMs: 900, payload: {outbound: [], inbound: [{kind: 'video', packetsLost: 4}]}},
|
||||
{atMs: 1_500, payload: {outbound: [], inbound: [{kind: 'video', packetsLost: 4}]}},
|
||||
],
|
||||
});
|
||||
assert.equal(report.metrics.maxObservedPacketLoss, 4);
|
||||
assert.equal(report.metrics.maxObservedPacketLossDelta, 0);
|
||||
assert.deepEqual(strictFailures(report, config), []);
|
||||
|
||||
const failingReport = strictReportFor(config, {
|
||||
videoFrameTimes: [1_100, 1_200],
|
||||
subscriberStatsSamples: [
|
||||
{atMs: 900, payload: {outbound: [], inbound: [{kind: 'video', packetsLost: 4}]}},
|
||||
{atMs: 1_500, payload: {outbound: [], inbound: [{kind: 'video', packetsLost: 5}]}},
|
||||
],
|
||||
});
|
||||
assert.equal(failingReport.metrics.maxObservedPacketLossDelta, 1);
|
||||
assert.match(strictFailures(failingReport, config).join('\n'), /packet loss delta 1/);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('server publishing checks cover primary and secondary screen, audio, and camera tracks', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1',
|
||||
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
|
||||
LIVEKIT_ENABLE_SCREEN_AUDIO: '1',
|
||||
LIVEKIT_ENABLE_CAMERA: '1',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
const serverState = {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [
|
||||
{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false, sid: 'TR_V1'},
|
||||
{type: 'AUDIO', source: 'MICROPHONE', mimeType: 'audio/red', muted: false, sid: 'TR_A1'},
|
||||
{type: 'AUDIO', source: 'SCREEN_SHARE_AUDIO', mimeType: 'audio/red', muted: false, sid: 'TR_A2'},
|
||||
{type: 'VIDEO', source: 'CAMERA', muted: false, sid: 'TR_V2'},
|
||||
],
|
||||
},
|
||||
{
|
||||
identity: config.secondaryPublisherIdentity,
|
||||
tracks: [
|
||||
{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false, sid: 'TR_V3'},
|
||||
{type: 'AUDIO', source: 'MICROPHONE', mimeType: 'audio/red', muted: false, sid: 'TR_A3'},
|
||||
{type: 'AUDIO', source: 'SCREEN_SHARE_AUDIO', mimeType: 'audio/red', muted: false, sid: 'TR_A4'},
|
||||
],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
};
|
||||
const checks = serverChecksFor(config, serverState);
|
||||
assert.equal(checks.length, 8);
|
||||
assert.deepEqual(
|
||||
checks.map((check) => [check.name, check.pass]),
|
||||
[
|
||||
['server publishing API participants listed', true],
|
||||
['server sees publisher screenshare publication', true],
|
||||
['server sees publisher microphone publication', true],
|
||||
['server sees publisher screen-share audio publication', true],
|
||||
['server sees publisher camera publication', true],
|
||||
['server sees secondary publisher screenshare publication', true],
|
||||
['server sees secondary publisher microphone publication', true],
|
||||
['server sees secondary publisher screen-share audio publication', true],
|
||||
],
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('server publishing checks fail closed on missing server tracks', async () => {
|
||||
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1', LIVEKIT_ENABLE_SCREEN_AUDIO: '1'}, () => {
|
||||
const config = parseConfig();
|
||||
const checks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(checks, 'server sees publisher screenshare publication').pass, true);
|
||||
assert.equal(findCheck(checks, 'server sees publisher screen-share audio publication').pass, false);
|
||||
});
|
||||
});
|
||||
|
||||
test('server publishing checks reject wrong participant identity', async () => {
|
||||
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1'}, () => {
|
||||
const config = parseConfig();
|
||||
const checks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.subscriberIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(checks, 'server publishing API participants listed').pass, true);
|
||||
assert.equal(findCheck(checks, 'server sees publisher screenshare publication').pass, false);
|
||||
});
|
||||
});
|
||||
|
||||
test('server publishing checks reject wrong kind, source, and screen mime', async () => {
|
||||
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1', LIVEKIT_EXPECT_SCREEN_CODEC: 'h264'}, () => {
|
||||
const config = parseConfig();
|
||||
const wrongKindChecks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 'AUDIO', source: 'SCREEN_SHARE', mimeType: 'video/H264', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(wrongKindChecks, 'server sees publisher screenshare publication').pass, false);
|
||||
|
||||
const wrongSourceChecks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'CAMERA', mimeType: 'video/H264', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(wrongSourceChecks, 'server sees publisher screenshare publication').pass, false);
|
||||
|
||||
const wrongMimeChecks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(wrongMimeChecks, 'server sees publisher screenshare publication').pass, false);
|
||||
|
||||
const matchingChecks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mime_type: 'video/h264', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(matchingChecks, 'server sees publisher screenshare publication').pass, true);
|
||||
});
|
||||
});
|
||||
|
||||
test('server publishing checks reject muted or missing mute state on expected tracks', async () => {
|
||||
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1'}, () => {
|
||||
const config = parseConfig();
|
||||
const mutedChecks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: true}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(mutedChecks, 'server sees publisher screenshare publication').pass, false);
|
||||
|
||||
const missingMutedChecks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8'}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(missingMutedChecks, 'server sees publisher screenshare publication').pass, false);
|
||||
});
|
||||
});
|
||||
|
||||
test('server publishing checks handle protobuf numeric enums without accepting data tracks as video', async () => {
|
||||
await withHarnessEnv({LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1'}, () => {
|
||||
const config = parseConfig();
|
||||
const dataTrackChecks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 2, source: 3, mimeType: 'video/VP8', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(dataTrackChecks, 'server sees publisher screenshare publication').pass, false);
|
||||
|
||||
const videoTrackChecks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 1, source: 3, mimeType: 'video/VP8', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(videoTrackChecks, 'server sees publisher screenshare publication').pass, true);
|
||||
});
|
||||
});
|
||||
|
||||
test('server publishing checks fail closed on missing secondary, audio, camera, and screen-audio tracks', async () => {
|
||||
await withHarnessEnv(
|
||||
{
|
||||
LIVEKIT_VALIDATE_SERVER_PUBLISHING: '1',
|
||||
LIVEKIT_ENABLE_SECOND_PUBLISHER: '1',
|
||||
LIVEKIT_ENABLE_SCREEN_AUDIO: '1',
|
||||
LIVEKIT_ENABLE_CAMERA: '1',
|
||||
},
|
||||
() => {
|
||||
const config = parseConfig();
|
||||
const checks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
|
||||
},
|
||||
{
|
||||
identity: config.secondaryPublisherIdentity,
|
||||
tracks: [{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false}],
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
});
|
||||
assert.equal(findCheck(checks, 'server sees publisher screenshare publication').pass, true);
|
||||
assert.equal(findCheck(checks, 'server sees publisher microphone publication').pass, false);
|
||||
assert.equal(findCheck(checks, 'server sees publisher screen-share audio publication').pass, false);
|
||||
assert.equal(findCheck(checks, 'server sees publisher camera publication').pass, false);
|
||||
assert.equal(findCheck(checks, 'server sees secondary publisher screenshare publication').pass, true);
|
||||
assert.equal(findCheck(checks, 'server sees secondary publisher microphone publication').pass, false);
|
||||
assert.equal(findCheck(checks, 'server sees secondary publisher screen-share audio publication').pass, false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('strict server publishing validation does not pass open on Twirp errors', async () => {
|
||||
await withHarnessEnv({LIVEKIT_HARNESS_STRICT: '1'}, () => {
|
||||
const config = parseConfig();
|
||||
const checks = serverChecksFor(config, {
|
||||
participants: [
|
||||
{
|
||||
identity: config.publisherIdentity,
|
||||
tracks: [
|
||||
{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false},
|
||||
{type: 'AUDIO', source: 'MICROPHONE', mimeType: 'audio/red', muted: false},
|
||||
{type: 'AUDIO', source: 'SCREEN_SHARE_AUDIO', mimeType: 'audio/red', muted: false},
|
||||
],
|
||||
},
|
||||
{
|
||||
identity: config.secondaryPublisherIdentity,
|
||||
tracks: [
|
||||
{type: 'VIDEO', source: 'SCREEN_SHARE', mimeType: 'video/VP8', muted: false},
|
||||
{type: 'AUDIO', source: 'MICROPHONE', mimeType: 'audio/red', muted: false},
|
||||
{type: 'AUDIO', source: 'SCREEN_SHARE_AUDIO', mimeType: 'audio/red', muted: false},
|
||||
],
|
||||
},
|
||||
],
|
||||
error: 'RoomService.ListParticipants failed with HTTP 401',
|
||||
});
|
||||
assert.equal(config.validateServerPublishing, true);
|
||||
assert.equal(findCheck(checks, 'server publishing API participants listed').pass, false);
|
||||
assert.equal(
|
||||
checks.every((check) => check.pass),
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,65 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use napi_derive::napi;
|
||||
|
||||
pub const ENGINE_BRIDGE_VERSION: u32 = 18;
|
||||
|
||||
const _: () = assert!(ENGINE_BRIDGE_VERSION > 0);
|
||||
|
||||
fn check_engine_bridge_version(version: u32) -> Result<(), String> {
|
||||
if version == ENGINE_BRIDGE_VERSION {
|
||||
return Ok(());
|
||||
}
|
||||
Err(format!(
|
||||
"voice engine bridge version mismatch: host sent {version}, native addon expects {ENGINE_BRIDGE_VERSION}"
|
||||
))
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn get_engine_bridge_version() -> u32 {
|
||||
ENGINE_BRIDGE_VERSION
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn assert_engine_bridge_version(version: u32) -> napi::Result<()> {
|
||||
check_engine_bridge_version(version).map_err(napi::Error::from_reason)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn matching_version_passes() {
|
||||
assert!(check_engine_bridge_version(ENGINE_BRIDGE_VERSION).is_ok());
|
||||
assert!(assert_engine_bridge_version(ENGINE_BRIDGE_VERSION).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mismatched_version_fails_with_both_versions_in_message() {
|
||||
let error = check_engine_bridge_version(ENGINE_BRIDGE_VERSION + 1).unwrap_err();
|
||||
assert!(error.contains("voice engine bridge version mismatch"));
|
||||
assert!(error.contains(&(ENGINE_BRIDGE_VERSION + 1).to_string()));
|
||||
assert!(error.contains(&ENGINE_BRIDGE_VERSION.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_version_fails() {
|
||||
assert!(check_engine_bridge_version(0).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mismatch_surfaces_as_napi_error() {
|
||||
let error = assert_engine_bridge_version(ENGINE_BRIDGE_VERSION - 1).unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.reason
|
||||
.contains("voice engine bridge version mismatch")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exported_getter_reports_the_constant() {
|
||||
assert_eq!(get_engine_bridge_version(), ENGINE_BRIDGE_VERSION);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,200 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PublishConfig {
|
||||
pub url: String,
|
||||
pub token: String,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub fps: u32,
|
||||
pub codec: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl PublishConfig {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if self.url.trim().is_empty() {
|
||||
return Err("livekit url is empty".into());
|
||||
}
|
||||
if !(self.url.starts_with("ws://") || self.url.starts_with("wss://")) {
|
||||
return Err("livekit url must be ws:// or wss://".into());
|
||||
}
|
||||
if self.token.trim().is_empty() {
|
||||
return Err("livekit token is empty".into());
|
||||
}
|
||||
if self.width < 2 || self.height < 2 {
|
||||
return Err("capture dimensions too small".into());
|
||||
}
|
||||
if !self.width.is_multiple_of(2) || !self.height.is_multiple_of(2) {
|
||||
return Err("capture dimensions must be even".into());
|
||||
}
|
||||
if self.width > 8192 || self.height > 8192 {
|
||||
return Err("capture dimensions too large".into());
|
||||
}
|
||||
if self.fps == 0 {
|
||||
return Err("capture fps must be positive".into());
|
||||
}
|
||||
if !self.codec.trim().is_empty() && canonical_codec_name(&self.codec).is_none() {
|
||||
return Err("unsupported video codec".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub const SUPPORTED_CODECS: &[&str] = &["vp8", "h264", "vp9", "av1", "h265"];
|
||||
|
||||
pub fn canonical_codec_name(name: &str) -> Option<&'static str> {
|
||||
let lower = name.trim().to_ascii_lowercase();
|
||||
if lower == "hevc" {
|
||||
return Some("h265");
|
||||
}
|
||||
SUPPORTED_CODECS.iter().copied().find(|&c| c == lower)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PublisherState {
|
||||
Idle,
|
||||
Connecting,
|
||||
Publishing,
|
||||
Closed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl PublisherState {
|
||||
pub fn accepts_frames(self) -> bool {
|
||||
matches!(self, PublisherState::Publishing)
|
||||
}
|
||||
|
||||
pub fn can_connect(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
PublisherState::Idle | PublisherState::Closed | PublisherState::Failed
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cfg() -> PublishConfig {
|
||||
PublishConfig {
|
||||
url: "wss://sfu.example/rtc".into(),
|
||||
token: "jwt".into(),
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
fps: 30,
|
||||
codec: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valid_config_passes() {
|
||||
assert!(cfg().validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_bad_url() {
|
||||
let mut c = cfg();
|
||||
c.url = "https://sfu".into();
|
||||
assert!(c.validate().is_err());
|
||||
c.url = String::new();
|
||||
assert!(c.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_token() {
|
||||
let mut c = cfg();
|
||||
c.token = " ".into();
|
||||
assert!(c.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_odd_or_oob_dimensions() {
|
||||
let mut c = cfg();
|
||||
c.width = 1921;
|
||||
assert!(c.validate().is_err());
|
||||
c.width = 1920;
|
||||
c.height = 0;
|
||||
assert!(c.validate().is_err());
|
||||
c.height = 16384;
|
||||
assert!(c.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_dimension_boundaries_and_rejects_zero_fps() {
|
||||
let mut c = cfg();
|
||||
c.width = 2;
|
||||
c.height = 2;
|
||||
c.fps = 1;
|
||||
assert!(c.validate().is_ok());
|
||||
|
||||
c.width = 8192;
|
||||
c.height = 8192;
|
||||
assert!(c.validate().is_ok());
|
||||
|
||||
c.fps = 0;
|
||||
assert_eq!(
|
||||
c.validate(),
|
||||
Err("capture fps must be positive".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_non_empty_codec_but_allows_empty_default() {
|
||||
let mut c = cfg();
|
||||
c.codec = String::new();
|
||||
assert!(c.validate().is_ok());
|
||||
|
||||
c.codec = " ".into();
|
||||
assert!(c.validate().is_ok());
|
||||
|
||||
c.codec = "h266".into();
|
||||
assert_eq!(c.validate(), Err("unsupported video codec".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_codec_name_accepts_all_five_case_insensitively() {
|
||||
for (input, expected) in [
|
||||
("vp8", "vp8"),
|
||||
(" vp8 ", "vp8"),
|
||||
("VP8", "vp8"),
|
||||
("h264", "h264"),
|
||||
("H264", "h264"),
|
||||
("vp9", "vp9"),
|
||||
("Vp9", "vp9"),
|
||||
("av1", "av1"),
|
||||
("AV1", "av1"),
|
||||
("h265", "h265"),
|
||||
("H265", "h265"),
|
||||
("hevc", "h265"),
|
||||
("HEVC", "h265"),
|
||||
] {
|
||||
assert_eq!(canonical_codec_name(input), Some(expected), "codec {input}");
|
||||
}
|
||||
assert_eq!(SUPPORTED_CODECS.len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_codec_name_rejects_empty_and_unknown() {
|
||||
assert_eq!(canonical_codec_name(""), None);
|
||||
assert_eq!(canonical_codec_name("h266"), None);
|
||||
assert_eq!(canonical_codec_name("rubbish"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_gates_frames_and_connect() {
|
||||
assert!(PublisherState::Publishing.accepts_frames());
|
||||
assert!(!PublisherState::Connecting.accepts_frames());
|
||||
assert!(!PublisherState::Idle.accepts_frames());
|
||||
assert!(PublisherState::Idle.can_connect());
|
||||
assert!(PublisherState::Closed.can_connect());
|
||||
assert!(PublisherState::Failed.can_connect());
|
||||
assert!(!PublisherState::Failed.accepts_frames());
|
||||
assert!(!PublisherState::Publishing.can_connect());
|
||||
assert!(!PublisherState::Connecting.can_connect());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::audio::{
|
||||
DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX, DEEP_FILTER_NOISE_REDUCTION_LEVEL_MIN,
|
||||
clamp_deep_filter_noise_reduction_level,
|
||||
};
|
||||
use df::tract::{DfParams, DfTract, RuntimeParams};
|
||||
use ndarray::Array2;
|
||||
|
||||
pub const DEEP_FILTER_SAMPLE_RATE_HZ: u32 = 48_000;
|
||||
pub const DEEP_FILTER_NUM_CHANNELS: u32 = 1;
|
||||
pub const DEEP_FILTER_FRAME_SAMPLES: usize = 480;
|
||||
|
||||
const SAMPLE_SCALE_I16_TO_F32: f32 = 1.0 / 32_768.0;
|
||||
const SAMPLE_SCALE_F32_TO_I16: f32 = 32_767.0;
|
||||
|
||||
const _: () = assert!(DEEP_FILTER_FRAME_SAMPLES == DEEP_FILTER_SAMPLE_RATE_HZ as usize / 100);
|
||||
const _: () = assert!(DEEP_FILTER_NUM_CHANNELS == 1);
|
||||
|
||||
pub struct DeepFilterProcessor {
|
||||
model: DfTract,
|
||||
input: Array2<f32>,
|
||||
output: Array2<f32>,
|
||||
}
|
||||
|
||||
impl DeepFilterProcessor {
|
||||
pub fn new(noise_reduction_level: f64) -> Result<DeepFilterProcessor, String> {
|
||||
let level_db = clamp_deep_filter_noise_reduction_level(noise_reduction_level);
|
||||
assert!(level_db >= DEEP_FILTER_NOISE_REDUCTION_LEVEL_MIN);
|
||||
assert!(level_db <= DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX);
|
||||
let params = RuntimeParams::default_with_ch(DEEP_FILTER_NUM_CHANNELS as usize)
|
||||
.with_atten_lim(level_db as f32);
|
||||
let model = DfTract::new(DfParams::default(), ¶ms)
|
||||
.map_err(|error| format!("deep filter model init: {error:#}"))?;
|
||||
if model.sr != DEEP_FILTER_SAMPLE_RATE_HZ as usize {
|
||||
return Err(format!(
|
||||
"deep filter model sample rate {} != {DEEP_FILTER_SAMPLE_RATE_HZ}",
|
||||
model.sr
|
||||
));
|
||||
}
|
||||
if model.ch != DEEP_FILTER_NUM_CHANNELS as usize {
|
||||
return Err(format!(
|
||||
"deep filter model channels {} != {DEEP_FILTER_NUM_CHANNELS}",
|
||||
model.ch
|
||||
));
|
||||
}
|
||||
if model.hop_size != DEEP_FILTER_FRAME_SAMPLES {
|
||||
return Err(format!(
|
||||
"deep filter model hop {} != {DEEP_FILTER_FRAME_SAMPLES}",
|
||||
model.hop_size
|
||||
));
|
||||
}
|
||||
Ok(DeepFilterProcessor {
|
||||
model,
|
||||
input: Array2::zeros((1, DEEP_FILTER_FRAME_SAMPLES)),
|
||||
output: Array2::zeros((1, DEEP_FILTER_FRAME_SAMPLES)),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn process_frame(&mut self, samples: &mut [i16]) -> Result<(), String> {
|
||||
assert_eq!(samples.len(), DEEP_FILTER_FRAME_SAMPLES);
|
||||
assert_eq!(self.input.len(), DEEP_FILTER_FRAME_SAMPLES);
|
||||
assert_eq!(self.output.len(), DEEP_FILTER_FRAME_SAMPLES);
|
||||
for (target, sample) in self.input.iter_mut().zip(samples.iter()) {
|
||||
*target = f32::from(*sample) * SAMPLE_SCALE_I16_TO_F32;
|
||||
}
|
||||
self.model
|
||||
.process(self.input.view(), self.output.view_mut())
|
||||
.map_err(|error| format!("deep filter process: {error:#}"))?;
|
||||
for (sample, enhanced) in samples.iter_mut().zip(self.output.iter()) {
|
||||
*sample = sample_f32_to_i16(*enhanced);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_f32_to_i16(sample: f32) -> i16 {
|
||||
if !sample.is_finite() {
|
||||
return 0;
|
||||
}
|
||||
let clamped = sample.clamp(-1.0, 1.0);
|
||||
assert!(clamped >= -1.0);
|
||||
assert!(clamped <= 1.0);
|
||||
(clamped * SAMPLE_SCALE_F32_TO_I16) as i16
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn next_noise_sample(seed: &mut u32) -> i16 {
|
||||
*seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
((*seed >> 16) as u16 as i16) / 4
|
||||
}
|
||||
|
||||
fn noise_frame(seed: &mut u32) -> [i16; DEEP_FILTER_FRAME_SAMPLES] {
|
||||
let mut frame = [0i16; DEEP_FILTER_FRAME_SAMPLES];
|
||||
for sample in frame.iter_mut() {
|
||||
*sample = next_noise_sample(seed);
|
||||
}
|
||||
frame
|
||||
}
|
||||
|
||||
fn frame_rms(samples: &[i16]) -> f64 {
|
||||
assert!(!samples.is_empty());
|
||||
let sum_squares: f64 = samples
|
||||
.iter()
|
||||
.map(|sample| {
|
||||
let normalized = f64::from(*sample) / 32_768.0;
|
||||
normalized * normalized
|
||||
})
|
||||
.sum();
|
||||
(sum_squares / samples.len() as f64).sqrt()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_conversion_holds_the_contract_range() {
|
||||
assert_eq!(sample_f32_to_i16(0.0), 0);
|
||||
assert_eq!(sample_f32_to_i16(1.0), 32_767);
|
||||
assert_eq!(sample_f32_to_i16(-1.0), -32_767);
|
||||
assert_eq!(sample_f32_to_i16(2.0), 32_767);
|
||||
assert_eq!(sample_f32_to_i16(-2.0), -32_767);
|
||||
assert_eq!(sample_f32_to_i16(0.5), 16_383);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_conversion_maps_non_finite_to_silence() {
|
||||
assert_eq!(sample_f32_to_i16(f32::NAN), 0);
|
||||
assert_eq!(sample_f32_to_i16(f32::INFINITY), 0);
|
||||
assert_eq!(sample_f32_to_i16(f32::NEG_INFINITY), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_level_passes_audio_through() {
|
||||
let mut processor = DeepFilterProcessor::new(0.0).expect("embedded model must initialize");
|
||||
let mut seed = 0x2545_f491u32;
|
||||
for _ in 0..5 {
|
||||
let original = noise_frame(&mut seed);
|
||||
let mut processed = original;
|
||||
processor
|
||||
.process_frame(&mut processed)
|
||||
.expect("processing must succeed");
|
||||
for (output, input) in processed.iter().zip(original.iter()) {
|
||||
assert!((i32::from(*output) - i32::from(*input)).abs() <= 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_level_attenuates_steady_noise() {
|
||||
let mut processor = DeepFilterProcessor::new(DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX)
|
||||
.expect("embedded model must initialize");
|
||||
let mut seed = 0x9e37_79b9u32;
|
||||
let mut input_rms = 0.0;
|
||||
let mut output_rms = 0.0;
|
||||
for frame_index in 0..30 {
|
||||
let mut frame = noise_frame(&mut seed);
|
||||
let frame_input_rms = frame_rms(&frame);
|
||||
processor
|
||||
.process_frame(&mut frame)
|
||||
.expect("processing must succeed");
|
||||
if frame_index >= 20 {
|
||||
input_rms += frame_input_rms;
|
||||
output_rms += frame_rms(&frame);
|
||||
}
|
||||
}
|
||||
assert!(input_rms > 0.0);
|
||||
assert!(output_rms < input_rms * 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_length_contract_is_enforced() {
|
||||
let mut processor = DeepFilterProcessor::new(DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX)
|
||||
.expect("embedded model must initialize");
|
||||
let mut short_frame = [0i16; DEEP_FILTER_FRAME_SAMPLES - 1];
|
||||
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
let _ = processor.process_frame(&mut short_frame);
|
||||
}));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,643 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
pub fn push_json_string(out: &mut String, value: &str) {
|
||||
out.push('"');
|
||||
for ch in value.chars() {
|
||||
match ch {
|
||||
'"' => out.push_str("\\\""),
|
||||
'\\' => out.push_str("\\\\"),
|
||||
'\n' => out.push_str("\\n"),
|
||||
'\r' => out.push_str("\\r"),
|
||||
'\t' => out.push_str("\\t"),
|
||||
'\u{08}' => out.push_str("\\b"),
|
||||
'\u{0C}' => out.push_str("\\f"),
|
||||
c if (c as u32) < 0x20 => {
|
||||
out.push_str(&format!("\\u{:04x}", c as u32));
|
||||
}
|
||||
c => out.push(c),
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
|
||||
pub enum JsonValue {
|
||||
Str(String),
|
||||
Raw(String),
|
||||
}
|
||||
|
||||
pub fn json_object(fields: &[(&str, JsonValue)]) -> String {
|
||||
let mut out = String::from("{");
|
||||
for (i, (key, value)) in fields.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_json_string(&mut out, key);
|
||||
out.push(':');
|
||||
match value {
|
||||
JsonValue::Str(s) => push_json_string(&mut out, s),
|
||||
JsonValue::Raw(r) => out.push_str(r),
|
||||
}
|
||||
}
|
||||
out.push('}');
|
||||
out
|
||||
}
|
||||
|
||||
pub fn json_string_array(items: &[String]) -> String {
|
||||
let mut out = String::from("[");
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_json_string(&mut out, item);
|
||||
}
|
||||
out.push(']');
|
||||
out
|
||||
}
|
||||
|
||||
pub fn json_u8_array(items: &[u8]) -> String {
|
||||
let mut out = String::from("[");
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
out.push_str(&item.to_string());
|
||||
}
|
||||
out.push(']');
|
||||
out
|
||||
}
|
||||
|
||||
pub fn json_raw_array(items: &[String]) -> String {
|
||||
let mut out = String::from("[");
|
||||
for (i, item) in items.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
out.push_str(item);
|
||||
}
|
||||
out.push(']');
|
||||
out
|
||||
}
|
||||
|
||||
pub fn json_string_map(items: &std::collections::HashMap<String, String>) -> String {
|
||||
let mut entries: Vec<(&String, &String)> = items.iter().collect();
|
||||
entries.sort_by(|left, right| left.0.cmp(right.0));
|
||||
let mut out = String::from("{");
|
||||
for (i, (key, value)) in entries.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_json_string(&mut out, key);
|
||||
out.push(':');
|
||||
push_json_string(&mut out, value);
|
||||
}
|
||||
out.push('}');
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(feature = "publisher")]
|
||||
mod live {
|
||||
use super::{
|
||||
JsonValue, json_object, json_raw_array, json_string_array, json_string_map, json_u8_array,
|
||||
};
|
||||
use livekit::participant::{
|
||||
ConnectionQuality, LocalParticipant, Participant, RemoteParticipant,
|
||||
};
|
||||
use livekit::publication::{
|
||||
LocalTrackPublication, RemoteTrackPublication, SubscriptionStatus, TrackPublication,
|
||||
};
|
||||
use livekit::track::{TrackKind, TrackSource};
|
||||
use livekit::{DataPacketKind, RoomEvent};
|
||||
|
||||
pub fn track_kind_str(kind: TrackKind) -> &'static str {
|
||||
match kind {
|
||||
TrackKind::Audio => "audio",
|
||||
TrackKind::Video => "video",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn track_source_str(source: TrackSource) -> &'static str {
|
||||
match source {
|
||||
TrackSource::Unknown => "unknown",
|
||||
TrackSource::Camera => "camera",
|
||||
TrackSource::Microphone => "microphone",
|
||||
TrackSource::Screenshare => "screen_share",
|
||||
TrackSource::ScreenshareAudio => "screen_share_audio",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn connection_quality_str(quality: ConnectionQuality) -> &'static str {
|
||||
match quality {
|
||||
ConnectionQuality::Excellent => "excellent",
|
||||
ConnectionQuality::Good => "good",
|
||||
ConnectionQuality::Poor => "poor",
|
||||
ConnectionQuality::Lost => "lost",
|
||||
}
|
||||
}
|
||||
|
||||
fn s(value: impl Into<String>) -> JsonValue {
|
||||
JsonValue::Str(value.into())
|
||||
}
|
||||
|
||||
fn b(value: bool) -> JsonValue {
|
||||
JsonValue::Raw(value.to_string())
|
||||
}
|
||||
|
||||
fn subscription_status_str(status: SubscriptionStatus) -> &'static str {
|
||||
match status {
|
||||
SubscriptionStatus::Desired => "desired",
|
||||
SubscriptionStatus::Subscribed => "subscribed",
|
||||
SubscriptionStatus::Unsubscribed => "unsubscribed",
|
||||
}
|
||||
}
|
||||
|
||||
fn participant_snapshot(participant: &Participant) -> String {
|
||||
json_object(&[
|
||||
("sid", s(participant.sid().to_string())),
|
||||
("identity", s(participant.identity().to_string())),
|
||||
("name", s(participant.name())),
|
||||
])
|
||||
}
|
||||
|
||||
fn push_remote_participant_fields(
|
||||
fields: &mut Vec<(&'static str, JsonValue)>,
|
||||
participant: &RemoteParticipant,
|
||||
) {
|
||||
fields.push(("participantSid", s(participant.sid().to_string())));
|
||||
fields.push(("identity", s(participant.identity().to_string())));
|
||||
fields.push(("participantName", s(participant.name())));
|
||||
}
|
||||
|
||||
fn push_local_participant_fields(
|
||||
fields: &mut Vec<(&'static str, JsonValue)>,
|
||||
participant: &LocalParticipant,
|
||||
) {
|
||||
fields.push(("participantSid", s(participant.sid().to_string())));
|
||||
fields.push(("identity", s(participant.identity().to_string())));
|
||||
fields.push(("participantName", s(participant.name())));
|
||||
}
|
||||
|
||||
fn push_participant_fields(
|
||||
fields: &mut Vec<(&'static str, JsonValue)>,
|
||||
participant: &Participant,
|
||||
) {
|
||||
fields.push(("participantSid", s(participant.sid().to_string())));
|
||||
fields.push(("identity", s(participant.identity().to_string())));
|
||||
fields.push(("participantName", s(participant.name())));
|
||||
}
|
||||
|
||||
fn push_remote_publication_fields(
|
||||
fields: &mut Vec<(&'static str, JsonValue)>,
|
||||
publication: &RemoteTrackPublication,
|
||||
) {
|
||||
fields.push(("trackSid", s(publication.sid().to_string())));
|
||||
fields.push(("trackName", s(publication.name())));
|
||||
fields.push(("kind", s(track_kind_str(publication.kind()))));
|
||||
fields.push(("source", s(track_source_str(publication.source()))));
|
||||
fields.push(("muted", b(publication.is_muted())));
|
||||
fields.push(("subscribed", b(publication.is_subscribed())));
|
||||
fields.push((
|
||||
"subscriptionStatus",
|
||||
s(subscription_status_str(publication.subscription_status())),
|
||||
));
|
||||
}
|
||||
|
||||
fn push_local_publication_fields(
|
||||
fields: &mut Vec<(&'static str, JsonValue)>,
|
||||
publication: &LocalTrackPublication,
|
||||
) {
|
||||
fields.push(("trackSid", s(publication.sid().to_string())));
|
||||
fields.push(("trackName", s(publication.name())));
|
||||
fields.push(("kind", s(track_kind_str(publication.kind()))));
|
||||
fields.push(("source", s(track_source_str(publication.source()))));
|
||||
fields.push(("muted", b(publication.is_muted())));
|
||||
}
|
||||
|
||||
fn push_publication_fields(
|
||||
fields: &mut Vec<(&'static str, JsonValue)>,
|
||||
publication: &TrackPublication,
|
||||
) {
|
||||
fields.push(("trackSid", s(publication.sid().to_string())));
|
||||
fields.push(("trackName", s(publication.name())));
|
||||
fields.push(("kind", s(track_kind_str(publication.kind()))));
|
||||
fields.push(("source", s(track_source_str(publication.source()))));
|
||||
fields.push(("muted", b(publication.is_muted())));
|
||||
}
|
||||
|
||||
fn remote_track_payload(
|
||||
participant: &RemoteParticipant,
|
||||
publication: &RemoteTrackPublication,
|
||||
) -> String {
|
||||
let mut fields = Vec::new();
|
||||
push_remote_participant_fields(&mut fields, participant);
|
||||
push_remote_publication_fields(&mut fields, publication);
|
||||
json_object(&fields)
|
||||
}
|
||||
|
||||
const CONNECTED_ROSTER_PARTICIPANTS_MAX: usize = 1024;
|
||||
const CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX: usize = 16;
|
||||
|
||||
fn connected_payload(
|
||||
participants_with_tracks: &[(RemoteParticipant, Vec<RemoteTrackPublication>)],
|
||||
) -> String {
|
||||
let participant_count = participants_with_tracks
|
||||
.len()
|
||||
.min(CONNECTED_ROSTER_PARTICIPANTS_MAX);
|
||||
let mut entries = Vec::with_capacity(participant_count);
|
||||
for (participant, publications) in participants_with_tracks
|
||||
.iter()
|
||||
.take(CONNECTED_ROSTER_PARTICIPANTS_MAX)
|
||||
{
|
||||
let track_count = publications
|
||||
.len()
|
||||
.min(CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX);
|
||||
let mut tracks = Vec::with_capacity(track_count);
|
||||
for publication in publications
|
||||
.iter()
|
||||
.take(CONNECTED_ROSTER_TRACKS_PER_PARTICIPANT_MAX)
|
||||
{
|
||||
tracks.push(remote_track_payload(participant, publication));
|
||||
}
|
||||
entries.push(json_object(&[
|
||||
("sid", s(participant.sid().to_string())),
|
||||
("identity", s(participant.identity().to_string())),
|
||||
("name", s(participant.name())),
|
||||
("tracks", JsonValue::Raw(json_raw_array(&tracks))),
|
||||
]));
|
||||
}
|
||||
assert!(entries.len() <= CONNECTED_ROSTER_PARTICIPANTS_MAX);
|
||||
json_object(&[("participants", JsonValue::Raw(json_raw_array(&entries)))])
|
||||
}
|
||||
|
||||
fn local_track_payload(
|
||||
participant: &LocalParticipant,
|
||||
publication: &LocalTrackPublication,
|
||||
) -> String {
|
||||
let mut fields = Vec::new();
|
||||
push_local_participant_fields(&mut fields, participant);
|
||||
push_local_publication_fields(&mut fields, publication);
|
||||
json_object(&fields)
|
||||
}
|
||||
|
||||
pub fn map_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
|
||||
map_participant_lifecycle_room_event(event)
|
||||
.or_else(|| map_participant_profile_room_event(event))
|
||||
.or_else(|| map_track_room_event(event))
|
||||
.or_else(|| map_local_track_room_event(event))
|
||||
.or_else(|| map_connection_room_event(event))
|
||||
}
|
||||
|
||||
fn map_participant_lifecycle_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
|
||||
match event {
|
||||
RoomEvent::ParticipantConnected(p) | RoomEvent::ParticipantActive(p) => Some((
|
||||
"participantJoined",
|
||||
json_object(&[
|
||||
("sid", s(p.sid().to_string())),
|
||||
("identity", s(p.identity().to_string())),
|
||||
("name", s(p.name())),
|
||||
]),
|
||||
)),
|
||||
RoomEvent::ParticipantDisconnected(p) => Some((
|
||||
"participantLeft",
|
||||
json_object(&[
|
||||
("sid", s(p.sid().to_string())),
|
||||
("identity", s(p.identity().to_string())),
|
||||
("name", s(p.name())),
|
||||
]),
|
||||
)),
|
||||
RoomEvent::ParticipantNameChanged {
|
||||
participant,
|
||||
old_name,
|
||||
name,
|
||||
} => Some((
|
||||
"participantNameChanged",
|
||||
json_object(&[
|
||||
("sid", s(participant.sid().to_string())),
|
||||
("identity", s(participant.identity().to_string())),
|
||||
("oldName", s(old_name.to_string())),
|
||||
("name", s(name.to_string())),
|
||||
]),
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn map_participant_profile_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
|
||||
match event {
|
||||
RoomEvent::ParticipantMetadataChanged {
|
||||
participant,
|
||||
old_metadata,
|
||||
metadata,
|
||||
} => Some((
|
||||
"participantMetadataChanged",
|
||||
json_object(&[
|
||||
("sid", s(participant.sid().to_string())),
|
||||
("identity", s(participant.identity().to_string())),
|
||||
("name", s(participant.name())),
|
||||
("oldMetadata", s(old_metadata.to_string())),
|
||||
("metadata", s(metadata.to_string())),
|
||||
(
|
||||
"attributes",
|
||||
JsonValue::Raw(json_string_map(&participant.attributes())),
|
||||
),
|
||||
]),
|
||||
)),
|
||||
RoomEvent::ParticipantAttributesChanged {
|
||||
participant,
|
||||
changed_attributes,
|
||||
} => Some((
|
||||
"participantAttributesChanged",
|
||||
json_object(&[
|
||||
("sid", s(participant.sid().to_string())),
|
||||
("identity", s(participant.identity().to_string())),
|
||||
("name", s(participant.name())),
|
||||
(
|
||||
"attributes",
|
||||
JsonValue::Raw(json_string_map(&participant.attributes())),
|
||||
),
|
||||
(
|
||||
"changedAttributes",
|
||||
JsonValue::Raw(json_string_map(changed_attributes)),
|
||||
),
|
||||
]),
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn map_track_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
|
||||
match event {
|
||||
RoomEvent::TrackSubscribed {
|
||||
publication,
|
||||
participant,
|
||||
..
|
||||
} => Some((
|
||||
"trackSubscribed",
|
||||
remote_track_payload(participant, publication),
|
||||
)),
|
||||
RoomEvent::TrackUnsubscribed {
|
||||
publication,
|
||||
participant,
|
||||
..
|
||||
} => Some((
|
||||
"trackUnsubscribed",
|
||||
remote_track_payload(participant, publication),
|
||||
)),
|
||||
RoomEvent::TrackSubscriptionFailed {
|
||||
participant,
|
||||
error,
|
||||
track_sid,
|
||||
} => {
|
||||
let mut fields = Vec::new();
|
||||
push_remote_participant_fields(&mut fields, participant);
|
||||
if let Some(publication) = participant.get_track_publication(track_sid) {
|
||||
push_remote_publication_fields(&mut fields, &publication);
|
||||
} else {
|
||||
fields.push(("trackSid", s(track_sid.to_string())));
|
||||
}
|
||||
fields.push(("error", s(format!("{error}"))));
|
||||
Some(("trackSubscriptionFailed", json_object(&fields)))
|
||||
}
|
||||
RoomEvent::TrackPublished {
|
||||
publication,
|
||||
participant,
|
||||
} => Some((
|
||||
"trackPublished",
|
||||
remote_track_payload(participant, publication),
|
||||
)),
|
||||
RoomEvent::TrackUnpublished {
|
||||
publication,
|
||||
participant,
|
||||
} => Some((
|
||||
"trackUnpublished",
|
||||
remote_track_payload(participant, publication),
|
||||
)),
|
||||
RoomEvent::TrackMuted {
|
||||
participant,
|
||||
publication,
|
||||
} => {
|
||||
let mut fields = Vec::new();
|
||||
push_participant_fields(&mut fields, participant);
|
||||
push_publication_fields(&mut fields, publication);
|
||||
Some(("trackMuted", json_object(&fields)))
|
||||
}
|
||||
RoomEvent::TrackUnmuted {
|
||||
participant,
|
||||
publication,
|
||||
} => {
|
||||
let mut fields = Vec::new();
|
||||
push_participant_fields(&mut fields, participant);
|
||||
push_publication_fields(&mut fields, publication);
|
||||
Some(("trackUnmuted", json_object(&fields)))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn map_connection_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
|
||||
match event {
|
||||
RoomEvent::ActiveSpeakersChanged { speakers } => {
|
||||
let sids: Vec<String> = speakers.iter().map(|p| p.sid().to_string()).collect();
|
||||
let participants: Vec<String> = speakers.iter().map(participant_snapshot).collect();
|
||||
Some((
|
||||
"activeSpeakers",
|
||||
json_object(&[
|
||||
("sids", JsonValue::Raw(json_string_array(&sids))),
|
||||
(
|
||||
"participants",
|
||||
JsonValue::Raw(json_raw_array(&participants)),
|
||||
),
|
||||
]),
|
||||
))
|
||||
}
|
||||
RoomEvent::ConnectionQualityChanged {
|
||||
quality,
|
||||
participant,
|
||||
} => Some((
|
||||
"connectionQuality",
|
||||
json_object(&[
|
||||
("sid", s(participant.sid().to_string())),
|
||||
("identity", s(participant.identity().to_string())),
|
||||
("name", s(participant.name())),
|
||||
("quality", s(connection_quality_str(*quality))),
|
||||
]),
|
||||
)),
|
||||
RoomEvent::DataReceived {
|
||||
payload,
|
||||
topic,
|
||||
kind,
|
||||
participant,
|
||||
} => Some((
|
||||
"dataReceived",
|
||||
data_received_payload(
|
||||
payload.as_ref().as_slice(),
|
||||
topic.as_deref(),
|
||||
kind,
|
||||
participant.as_ref(),
|
||||
),
|
||||
)),
|
||||
RoomEvent::E2eeStateChanged { participant, state } => Some((
|
||||
"e2eeState",
|
||||
json_object(&[
|
||||
("sid", s(participant.sid().to_string())),
|
||||
("identity", s(participant.identity().to_string())),
|
||||
("name", s(participant.name())),
|
||||
("state", s(format!("{state:?}").to_lowercase())),
|
||||
]),
|
||||
)),
|
||||
RoomEvent::ConnectionStateChanged(state) => Some((
|
||||
"connectionState",
|
||||
json_object(&[("state", s(format!("{state:?}").to_lowercase()))]),
|
||||
)),
|
||||
RoomEvent::Disconnected { reason } => Some((
|
||||
"disconnected",
|
||||
json_object(&[("reason", s(format!("{reason:?}").to_lowercase()))]),
|
||||
)),
|
||||
RoomEvent::Connected {
|
||||
participants_with_tracks,
|
||||
} => Some(("connected", connected_payload(participants_with_tracks))),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn data_received_payload(
|
||||
payload: &[u8],
|
||||
topic: Option<&str>,
|
||||
kind: &DataPacketKind,
|
||||
participant: Option<&RemoteParticipant>,
|
||||
) -> String {
|
||||
let mut fields = vec![
|
||||
("payloadBytes", JsonValue::Raw(json_u8_array(payload))),
|
||||
(
|
||||
"reliable",
|
||||
JsonValue::Raw(matches!(kind, DataPacketKind::Reliable).to_string()),
|
||||
),
|
||||
(
|
||||
"kind",
|
||||
s(match kind {
|
||||
DataPacketKind::Reliable => "reliable",
|
||||
DataPacketKind::Lossy => "lossy",
|
||||
}),
|
||||
),
|
||||
];
|
||||
if let Some(topic) = topic {
|
||||
fields.push(("topic", s(topic.to_string())));
|
||||
}
|
||||
if let Ok(payload_text) = std::str::from_utf8(payload) {
|
||||
fields.push(("payloadText", s(payload_text.to_string())));
|
||||
}
|
||||
if let Some(participant) = participant {
|
||||
push_remote_participant_fields(&mut fields, participant);
|
||||
}
|
||||
json_object(&fields)
|
||||
}
|
||||
|
||||
fn map_local_track_room_event(event: &RoomEvent) -> Option<(&'static str, String)> {
|
||||
match event {
|
||||
RoomEvent::LocalTrackPublished {
|
||||
publication,
|
||||
participant,
|
||||
..
|
||||
} => Some((
|
||||
"localTrackPublished",
|
||||
local_track_payload(participant, publication),
|
||||
)),
|
||||
RoomEvent::LocalTrackUnpublished {
|
||||
publication,
|
||||
participant,
|
||||
} => Some((
|
||||
"localTrackUnpublished",
|
||||
local_track_payload(participant, publication),
|
||||
)),
|
||||
RoomEvent::LocalTrackRepublished {
|
||||
previous_sid,
|
||||
publication,
|
||||
participant,
|
||||
..
|
||||
} => {
|
||||
let mut fields = Vec::new();
|
||||
push_local_participant_fields(&mut fields, participant);
|
||||
fields.push(("previousTrackSid", s(previous_sid.to_string())));
|
||||
push_local_publication_fields(&mut fields, publication);
|
||||
Some(("localTrackRepublished", json_object(&fields)))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "publisher")]
|
||||
pub use live::{map_room_event, track_kind_str, track_source_str};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn escapes_quote_backslash_and_control_chars() {
|
||||
let mut out = String::new();
|
||||
push_json_string(&mut out, "a\"b\\c\nd\te");
|
||||
assert_eq!(out, "\"a\\\"b\\\\c\\nd\\te\"");
|
||||
|
||||
let mut ctrl = String::new();
|
||||
push_json_string(&mut ctrl, "\u{01}");
|
||||
assert_eq!(ctrl, "\"\\u0001\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_plain_ascii_and_unicode_untouched() {
|
||||
let mut out = String::new();
|
||||
push_json_string(&mut out, "PA_abc123");
|
||||
assert_eq!(out, "\"PA_abc123\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_object_preserves_order_and_mixes_str_and_raw() {
|
||||
let json = json_object(&[
|
||||
("sid", JsonValue::Str("PA_1".into())),
|
||||
(
|
||||
"sids",
|
||||
JsonValue::Raw(json_string_array(&["PA_1".into(), "PA_2".into()])),
|
||||
),
|
||||
]);
|
||||
assert_eq!(json, "{\"sid\":\"PA_1\",\"sids\":[\"PA_1\",\"PA_2\"]}");
|
||||
}
|
||||
|
||||
#[cfg(feature = "publisher")]
|
||||
#[test]
|
||||
fn track_source_strings_match_livekit_js_sources() {
|
||||
use livekit::track::TrackSource;
|
||||
|
||||
assert_eq!(track_source_str(TrackSource::Camera), "camera");
|
||||
assert_eq!(track_source_str(TrackSource::Microphone), "microphone");
|
||||
assert_eq!(track_source_str(TrackSource::Screenshare), "screen_share");
|
||||
assert_eq!(
|
||||
track_source_str(TrackSource::ScreenshareAudio),
|
||||
"screen_share_audio"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_object_and_array() {
|
||||
assert_eq!(json_object(&[]), "{}");
|
||||
assert_eq!(json_string_array(&[]), "[]");
|
||||
assert_eq!(json_u8_array(&[]), "[]");
|
||||
assert_eq!(json_raw_array(&[]), "[]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_u8_array_serializes_bytes_as_numbers() {
|
||||
assert_eq!(json_u8_array(&[0, 1, 127, 255]), "[0,1,127,255]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_array_preserves_prebuilt_json_objects() {
|
||||
let items = vec![
|
||||
json_object(&[("sid", JsonValue::Str("PA_1".into()))]),
|
||||
json_object(&[("sid", JsonValue::Str("PA_2".into()))]),
|
||||
];
|
||||
assert_eq!(
|
||||
json_raw_array(&items),
|
||||
"[{\"sid\":\"PA_1\"},{\"sid\":\"PA_2\"}]"
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,415 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use futures_core::Stream;
|
||||
use napi::tokio;
|
||||
use napi::tokio::task::AbortHandle;
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::HashMap;
|
||||
use std::future::poll_fn;
|
||||
use std::pin::pin;
|
||||
|
||||
pub const INBOUND_FORWARDERS_MAX: usize = 512;
|
||||
|
||||
struct ForwarderEntry {
|
||||
participant_sid: String,
|
||||
handle: AbortHandle,
|
||||
}
|
||||
|
||||
pub struct InboundForwarderRegistry {
|
||||
entries: Mutex<HashMap<String, ForwarderEntry>>,
|
||||
}
|
||||
|
||||
impl InboundForwarderRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.lock().len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn contains(&self, track_sid: &str) -> bool {
|
||||
self.entries.lock().contains_key(track_sid)
|
||||
}
|
||||
|
||||
pub fn register(&self, track_sid: &str, participant_sid: &str, handle: AbortHandle) -> bool {
|
||||
if track_sid.is_empty() {
|
||||
handle.abort();
|
||||
return false;
|
||||
}
|
||||
let mut entries = self.entries.lock();
|
||||
if let Some(previous) = entries.remove(track_sid) {
|
||||
previous.handle.abort();
|
||||
}
|
||||
if entries.len() >= INBOUND_FORWARDERS_MAX {
|
||||
drop(entries);
|
||||
handle.abort();
|
||||
eprintln!(
|
||||
"webrtc-sender: inbound forwarder registry at cap {INBOUND_FORWARDERS_MAX}; \
|
||||
refusing forwarder for track {track_sid}"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
entries.insert(
|
||||
track_sid.to_string(),
|
||||
ForwarderEntry {
|
||||
participant_sid: participant_sid.to_string(),
|
||||
handle,
|
||||
},
|
||||
);
|
||||
assert!(entries.len() <= INBOUND_FORWARDERS_MAX);
|
||||
true
|
||||
}
|
||||
|
||||
pub fn cancel(&self, track_sid: &str) {
|
||||
let removed = self.entries.lock().remove(track_sid);
|
||||
if let Some(entry) = removed {
|
||||
entry.handle.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cancel_for_participant(&self, participant_sid: &str) {
|
||||
let aborted: Vec<ForwarderEntry> = {
|
||||
let mut entries = self.entries.lock();
|
||||
let matching: Vec<String> = entries
|
||||
.iter()
|
||||
.filter(|(_, entry)| entry.participant_sid == participant_sid)
|
||||
.map(|(track_sid, _)| track_sid.clone())
|
||||
.collect();
|
||||
matching
|
||||
.into_iter()
|
||||
.filter_map(|track_sid| entries.remove(&track_sid))
|
||||
.collect()
|
||||
};
|
||||
for entry in aborted {
|
||||
entry.handle.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(&self) {
|
||||
let drained: Vec<ForwarderEntry> = {
|
||||
let mut entries = self.entries.lock();
|
||||
entries.drain().map(|(_, entry)| entry).collect()
|
||||
};
|
||||
for entry in drained {
|
||||
entry.handle.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for InboundForwarderRegistry {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn spawn_drain_forwarder<S, F>(stream: S, mut on_item: F) -> AbortHandle
|
||||
where
|
||||
S: Stream + Send + 'static,
|
||||
S::Item: Send,
|
||||
F: FnMut(S::Item) + Send + 'static,
|
||||
{
|
||||
let task = tokio::spawn(async move {
|
||||
let mut stream = pin!(stream);
|
||||
loop {
|
||||
let item = poll_fn(|cx| stream.as_mut().poll_next(cx)).await;
|
||||
match item {
|
||||
Some(item) => on_item(item),
|
||||
None => return,
|
||||
}
|
||||
}
|
||||
});
|
||||
task.abort_handle()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::VecDeque;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::runtime::Builder;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
struct CloseSignal {
|
||||
closed: AtomicBool,
|
||||
notify: Notify,
|
||||
}
|
||||
|
||||
impl CloseSignal {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
closed: AtomicBool::new(false),
|
||||
notify: Notify::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn is_closed(&self) -> bool {
|
||||
self.closed.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
fn fire(&self) {
|
||||
self.closed.store(true, Ordering::SeqCst);
|
||||
self.notify.notify_one();
|
||||
}
|
||||
|
||||
async fn wait(&self) {
|
||||
if self.is_closed() {
|
||||
return;
|
||||
}
|
||||
self.notify.notified().await;
|
||||
assert!(self.is_closed());
|
||||
}
|
||||
}
|
||||
|
||||
struct FakeVideoStream {
|
||||
signal: Arc<CloseSignal>,
|
||||
}
|
||||
|
||||
impl Stream for FakeVideoStream {
|
||||
type Item = u64;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FakeVideoStream {
|
||||
fn drop(&mut self) {
|
||||
self.signal.fire();
|
||||
}
|
||||
}
|
||||
|
||||
struct CountedStream {
|
||||
items: VecDeque<u64>,
|
||||
count: Arc<AtomicUsize>,
|
||||
done: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl Stream for CountedStream {
|
||||
type Item = u64;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
match self.items.pop_front() {
|
||||
Some(item) => {
|
||||
self.count.fetch_add(1, Ordering::SeqCst);
|
||||
Poll::Ready(Some(item))
|
||||
}
|
||||
None => {
|
||||
self.done.notify_one();
|
||||
Poll::Ready(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn runtime() -> tokio::runtime::Runtime {
|
||||
Builder::new_multi_thread()
|
||||
.worker_threads(1)
|
||||
.build()
|
||||
.expect("tokio runtime")
|
||||
}
|
||||
|
||||
fn dispatch_track_subscribed(
|
||||
registry: &InboundForwarderRegistry,
|
||||
track_sid: &str,
|
||||
participant_sid: &str,
|
||||
) -> Arc<CloseSignal> {
|
||||
let (signal, registered) = dispatch_with_outcome(registry, track_sid, participant_sid);
|
||||
assert!(registered);
|
||||
signal
|
||||
}
|
||||
|
||||
fn dispatch_with_outcome(
|
||||
registry: &InboundForwarderRegistry,
|
||||
track_sid: &str,
|
||||
participant_sid: &str,
|
||||
) -> (Arc<CloseSignal>, bool) {
|
||||
let signal = CloseSignal::new();
|
||||
let stream = FakeVideoStream {
|
||||
signal: signal.clone(),
|
||||
};
|
||||
let handle = spawn_drain_forwarder(stream, |_frame| {});
|
||||
let registered = registry.register(track_sid, participant_sid, handle);
|
||||
(signal, registered)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn teardown_symmetry_closes_the_stream() {
|
||||
let runtime = runtime();
|
||||
runtime.block_on(async {
|
||||
let registry = InboundForwarderRegistry::new();
|
||||
let signal = dispatch_track_subscribed(®istry, "TR_x", "PA_one");
|
||||
|
||||
assert_eq!(registry.len(), 1);
|
||||
assert!(!signal.is_closed());
|
||||
|
||||
registry.cancel("TR_x");
|
||||
assert_eq!(registry.len(), 0);
|
||||
|
||||
signal.wait().await;
|
||||
assert!(signal.is_closed());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_subscribe_unsubscribe_never_accumulates() {
|
||||
let runtime = runtime();
|
||||
runtime.block_on(async {
|
||||
let registry = InboundForwarderRegistry::new();
|
||||
let cycles = 32usize;
|
||||
let mut signals = Vec::with_capacity(cycles);
|
||||
|
||||
for _ in 0..cycles {
|
||||
let signal = dispatch_track_subscribed(®istry, "TR_x", "PA_one");
|
||||
assert_eq!(registry.len(), 1);
|
||||
registry.cancel("TR_x");
|
||||
assert_eq!(registry.len(), 0);
|
||||
signal.wait().await;
|
||||
signals.push(signal);
|
||||
}
|
||||
|
||||
assert_eq!(registry.len(), 0);
|
||||
let closed_count = signals.iter().filter(|signal| signal.is_closed()).count();
|
||||
assert_eq!(closed_count, cycles);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn double_subscribe_keeps_one_live_forwarder() {
|
||||
let runtime = runtime();
|
||||
runtime.block_on(async {
|
||||
let registry = InboundForwarderRegistry::new();
|
||||
let first_signal = dispatch_track_subscribed(®istry, "TR_x", "PA_one");
|
||||
let second_signal = dispatch_track_subscribed(®istry, "TR_x", "PA_one");
|
||||
|
||||
assert_eq!(registry.len(), 1);
|
||||
first_signal.wait().await;
|
||||
assert!(first_signal.is_closed());
|
||||
assert!(!second_signal.is_closed());
|
||||
|
||||
registry.cancel("TR_x");
|
||||
assert_eq!(registry.len(), 0);
|
||||
second_signal.wait().await;
|
||||
assert!(second_signal.is_closed());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn participant_disconnect_tears_down_all_forwarders() {
|
||||
let runtime = runtime();
|
||||
runtime.block_on(async {
|
||||
let registry = InboundForwarderRegistry::new();
|
||||
let video_signal = dispatch_track_subscribed(®istry, "TR_video", "PA_one");
|
||||
let audio_signal = dispatch_track_subscribed(®istry, "TR_audio", "PA_one");
|
||||
let other_signal = dispatch_track_subscribed(®istry, "TR_other", "PA_two");
|
||||
assert_eq!(registry.len(), 3);
|
||||
|
||||
registry.cancel_for_participant("PA_one");
|
||||
assert_eq!(registry.len(), 1);
|
||||
assert!(registry.contains("TR_other"));
|
||||
|
||||
video_signal.wait().await;
|
||||
audio_signal.wait().await;
|
||||
assert!(video_signal.is_closed());
|
||||
assert!(audio_signal.is_closed());
|
||||
assert!(!other_signal.is_closed());
|
||||
|
||||
registry.clear();
|
||||
assert_eq!(registry.len(), 0);
|
||||
other_signal.wait().await;
|
||||
assert!(other_signal.is_closed());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_tears_down_every_forwarder() {
|
||||
let runtime = runtime();
|
||||
runtime.block_on(async {
|
||||
let registry = InboundForwarderRegistry::new();
|
||||
let mut signals = Vec::new();
|
||||
for index in 0..8 {
|
||||
let track_sid = format!("TR_{index}");
|
||||
signals.push(dispatch_track_subscribed(®istry, &track_sid, "PA_one"));
|
||||
}
|
||||
assert_eq!(registry.len(), 8);
|
||||
|
||||
registry.clear();
|
||||
assert_eq!(registry.len(), 0);
|
||||
|
||||
for signal in &signals {
|
||||
signal.wait().await;
|
||||
}
|
||||
assert!(signals.iter().all(|signal| signal.is_closed()));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_forwarder_invokes_callback_per_item() {
|
||||
let runtime = runtime();
|
||||
runtime.block_on(async {
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let polled = Arc::new(AtomicUsize::new(0));
|
||||
let polled_in_task = polled.clone();
|
||||
let done = Arc::new(Notify::new());
|
||||
let stream = CountedStream {
|
||||
items: VecDeque::from(vec![1u64, 2, 3, 4]),
|
||||
count: count.clone(),
|
||||
done: done.clone(),
|
||||
};
|
||||
let _handle = spawn_drain_forwarder(stream, move |_item| {
|
||||
polled_in_task.fetch_add(1, Ordering::SeqCst);
|
||||
});
|
||||
done.notified().await;
|
||||
|
||||
assert_eq!(count.load(Ordering::SeqCst), 4);
|
||||
assert_eq!(polled.load(Ordering::SeqCst), 4);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_refuses_and_closes_forwarder_at_cap() {
|
||||
let runtime = runtime();
|
||||
runtime.block_on(async {
|
||||
let registry = InboundForwarderRegistry::new();
|
||||
for index in 0..INBOUND_FORWARDERS_MAX {
|
||||
let track_sid = format!("TR_{index}");
|
||||
let (_signal, registered) = dispatch_with_outcome(®istry, &track_sid, "PA_one");
|
||||
assert!(registered);
|
||||
}
|
||||
assert_eq!(registry.len(), INBOUND_FORWARDERS_MAX);
|
||||
|
||||
let (overflow_signal, registered) =
|
||||
dispatch_with_outcome(®istry, "TR_overflow", "PA_one");
|
||||
assert!(!registered);
|
||||
assert_eq!(registry.len(), INBOUND_FORWARDERS_MAX);
|
||||
assert!(!registry.contains("TR_overflow"));
|
||||
|
||||
overflow_signal.wait().await;
|
||||
assert!(overflow_signal.is_closed());
|
||||
|
||||
registry.clear();
|
||||
assert_eq!(registry.len(), 0);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_refuses_and_closes_forwarder_for_empty_sid() {
|
||||
let runtime = runtime();
|
||||
runtime.block_on(async {
|
||||
let registry = InboundForwarderRegistry::new();
|
||||
let (signal, registered) = dispatch_with_outcome(®istry, "", "PA_one");
|
||||
assert!(!registered);
|
||||
assert_eq!(registry.len(), 0);
|
||||
|
||||
signal.wait().await;
|
||||
assert!(signal.is_closed());
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
#![allow(unsafe_op_in_unsafe_fn)]
|
||||
#![cfg_attr(
|
||||
not(all(feature = "publisher", feature = "camera-native")),
|
||||
allow(dead_code)
|
||||
)]
|
||||
|
||||
mod audio;
|
||||
mod bridge_version;
|
||||
mod camera;
|
||||
mod camera_background;
|
||||
mod config;
|
||||
mod deep_filter;
|
||||
mod events;
|
||||
mod hardware_encoder;
|
||||
mod inbound_forwarder;
|
||||
mod mask_refine;
|
||||
mod native_camera;
|
||||
mod person_segmentation;
|
||||
mod send_control;
|
||||
mod speaking;
|
||||
mod stats;
|
||||
mod texture_source;
|
||||
mod yuv;
|
||||
|
||||
#[cfg(feature = "publisher")]
|
||||
mod engine;
|
||||
|
||||
#[cfg(feature = "bench-internals")]
|
||||
pub mod bench_internals {
|
||||
pub use crate::audio::DEEP_FILTER_NOISE_REDUCTION_LEVEL_MAX;
|
||||
pub use crate::deep_filter::{DEEP_FILTER_FRAME_SAMPLES, DeepFilterProcessor};
|
||||
pub use crate::mask_refine::MaskRefiner;
|
||||
|
||||
pub struct BlurScratch(crate::camera_background::BlurScratch);
|
||||
|
||||
impl BlurScratch {
|
||||
pub fn new(width: usize, height: usize) -> Self {
|
||||
Self(crate::camera_background::BlurScratch::new(width, height))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn blur_plane_masked(
|
||||
plane: &mut [u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
mask: &[u8],
|
||||
radius_pass: usize,
|
||||
scratch: &mut BlurScratch,
|
||||
) {
|
||||
let mask = crate::camera_background::plane_mask(mask, width, 1);
|
||||
crate::camera_background::blur_plane_masked(
|
||||
plane,
|
||||
width,
|
||||
height,
|
||||
mask,
|
||||
radius_pass,
|
||||
&mut scratch.0,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn composite_masked_plane(
|
||||
plane: &mut [u8],
|
||||
background: &[u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
mask: &[u8],
|
||||
) {
|
||||
let mask = crate::camera_background::plane_mask(mask, width, 1);
|
||||
crate::camera_background::composite_masked_plane(plane, background, width, height, mask);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,626 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
const FRAME_EDGE_MAX: usize = 8192;
|
||||
const MASK_REFINE_DOWNSAMPLE: usize = 4;
|
||||
const GUIDED_FILTER_RADIUS_LOW: usize = 4;
|
||||
const GUIDED_FILTER_EPSILON: f32 = 1e-4;
|
||||
const TEMPORAL_COMBINE_RATIO: f32 = 0.7;
|
||||
const TEMPORAL_UNCERTAINTY_C1: f32 = 5.68842;
|
||||
const TEMPORAL_UNCERTAINTY_C2: f32 = -0.748699;
|
||||
const TEMPORAL_UNCERTAINTY_C3: f32 = -57.8051;
|
||||
const TEMPORAL_UNCERTAINTY_C4: f32 = 291.309;
|
||||
const TEMPORAL_UNCERTAINTY_C5: f32 = -624.717;
|
||||
const SHAPE_SMOOTHSTEP_EDGE_LOW: f32 = 0.55;
|
||||
const SHAPE_SMOOTHSTEP_EDGE_HIGH: f32 = 0.85;
|
||||
const LUT_LEN: usize = 256;
|
||||
|
||||
pub struct MaskRefiner {
|
||||
width: usize,
|
||||
height: usize,
|
||||
low_width: usize,
|
||||
low_height: usize,
|
||||
previous_mask: Vec<u8>,
|
||||
previous_mask_valid: bool,
|
||||
temporal_weight_lut: [u16; LUT_LEN],
|
||||
shape_lut: [u8; LUT_LEN],
|
||||
column_fixed: Vec<u32>,
|
||||
guide_low: Vec<f32>,
|
||||
mask_low: Vec<f32>,
|
||||
mean_guide: Vec<f32>,
|
||||
mean_mask: Vec<f32>,
|
||||
corr_guide_guide: Vec<f32>,
|
||||
corr_guide_mask: Vec<f32>,
|
||||
coeff_a: Vec<f32>,
|
||||
coeff_b: Vec<f32>,
|
||||
scratch: Vec<f32>,
|
||||
}
|
||||
|
||||
impl MaskRefiner {
|
||||
pub fn new(width: usize, height: usize) -> Self {
|
||||
assert!(width >= 2);
|
||||
assert!(height >= 2);
|
||||
assert!(width <= FRAME_EDGE_MAX);
|
||||
assert!(height <= FRAME_EDGE_MAX);
|
||||
let low_width = (width / MASK_REFINE_DOWNSAMPLE).max(1);
|
||||
let low_height = (height / MASK_REFINE_DOWNSAMPLE).max(1);
|
||||
let low_len = low_width * low_height;
|
||||
let mut column_fixed = vec![0u32; width];
|
||||
for (x, slot) in column_fixed.iter_mut().enumerate() {
|
||||
*slot = bilinear_fixed_coord(x, width, low_width);
|
||||
}
|
||||
Self {
|
||||
width,
|
||||
height,
|
||||
low_width,
|
||||
low_height,
|
||||
previous_mask: vec![0; width * height],
|
||||
previous_mask_valid: false,
|
||||
temporal_weight_lut: temporal_weight_lut(),
|
||||
shape_lut: shape_lut(),
|
||||
column_fixed,
|
||||
guide_low: vec![0.0; low_len],
|
||||
mask_low: vec![0.0; low_len],
|
||||
mean_guide: vec![0.0; low_len],
|
||||
mean_mask: vec![0.0; low_len],
|
||||
corr_guide_guide: vec![0.0; low_len],
|
||||
corr_guide_mask: vec![0.0; low_len],
|
||||
coeff_a: vec![0.0; low_len],
|
||||
coeff_b: vec![0.0; low_len],
|
||||
scratch: vec![0.0; low_len],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn refine(&mut self, luma: &[u8], mask: &mut [u8]) {
|
||||
assert!(luma.len() >= self.width * self.height);
|
||||
assert!(mask.len() >= self.width * self.height);
|
||||
self.blend_temporal(mask);
|
||||
self.downsample(luma, mask);
|
||||
self.close_mask_low();
|
||||
self.solve_guided_coefficients();
|
||||
self.apply_guided_coefficients(luma, mask);
|
||||
}
|
||||
|
||||
fn blend_temporal(&mut self, mask: &mut [u8]) {
|
||||
let len = self.width * self.height;
|
||||
assert!(mask.len() >= len);
|
||||
assert_eq!(self.previous_mask.len(), len);
|
||||
if self.previous_mask_valid {
|
||||
for (current, previous) in mask[..len].iter_mut().zip(self.previous_mask.iter()) {
|
||||
let new_value = i32::from(*current);
|
||||
let weight = i32::from(self.temporal_weight_lut[usize::from(*current)]);
|
||||
let delta = (i32::from(*previous) - new_value) * weight;
|
||||
*current = (new_value + ((delta + 128) >> 8)).clamp(0, 255) as u8;
|
||||
}
|
||||
}
|
||||
self.previous_mask.copy_from_slice(&mask[..len]);
|
||||
self.previous_mask_valid = true;
|
||||
}
|
||||
|
||||
fn downsample(&mut self, luma: &[u8], mask: &[u8]) {
|
||||
assert_eq!(self.guide_low.len(), self.low_width * self.low_height);
|
||||
assert_eq!(self.mask_low.len(), self.guide_low.len());
|
||||
for low_y in 0..self.low_height {
|
||||
let y_start = low_y * MASK_REFINE_DOWNSAMPLE;
|
||||
let y_end = (y_start + MASK_REFINE_DOWNSAMPLE).min(self.height);
|
||||
for low_x in 0..self.low_width {
|
||||
let x_start = low_x * MASK_REFINE_DOWNSAMPLE;
|
||||
let x_end = (x_start + MASK_REFINE_DOWNSAMPLE).min(self.width);
|
||||
let mut guide_sum: u32 = 0;
|
||||
let mut mask_sum: u32 = 0;
|
||||
for y in y_start..y_end {
|
||||
let row = y * self.width;
|
||||
for x in x_start..x_end {
|
||||
guide_sum += u32::from(luma[row + x]);
|
||||
mask_sum += u32::from(mask[row + x]);
|
||||
}
|
||||
}
|
||||
let count = ((y_end - y_start) * (x_end - x_start)) as f32;
|
||||
assert!(count >= 1.0);
|
||||
let low_offset = low_y * self.low_width + low_x;
|
||||
self.guide_low[low_offset] = guide_sum as f32 / (count * 255.0);
|
||||
self.mask_low[low_offset] = mask_sum as f32 / (count * 255.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn close_mask_low(&mut self) {
|
||||
morph_pass_low(
|
||||
&self.mask_low,
|
||||
&mut self.scratch,
|
||||
&mut self.coeff_a,
|
||||
self.low_width,
|
||||
self.low_height,
|
||||
f32::max,
|
||||
);
|
||||
morph_pass_low(
|
||||
&self.coeff_a,
|
||||
&mut self.scratch,
|
||||
&mut self.mask_low,
|
||||
self.low_width,
|
||||
self.low_height,
|
||||
f32::min,
|
||||
);
|
||||
}
|
||||
|
||||
fn solve_guided_coefficients(&mut self) {
|
||||
let len = self.low_width * self.low_height;
|
||||
assert_eq!(self.coeff_a.len(), len);
|
||||
assert_eq!(self.coeff_b.len(), len);
|
||||
let radius = GUIDED_FILTER_RADIUS_LOW;
|
||||
let w = self.low_width;
|
||||
let h = self.low_height;
|
||||
box_filter_low(
|
||||
&self.guide_low,
|
||||
&mut self.scratch,
|
||||
&mut self.mean_guide,
|
||||
w,
|
||||
h,
|
||||
radius,
|
||||
);
|
||||
box_filter_low(
|
||||
&self.mask_low,
|
||||
&mut self.scratch,
|
||||
&mut self.mean_mask,
|
||||
w,
|
||||
h,
|
||||
radius,
|
||||
);
|
||||
for i in 0..len {
|
||||
self.coeff_a[i] = self.guide_low[i] * self.guide_low[i];
|
||||
self.coeff_b[i] = self.guide_low[i] * self.mask_low[i];
|
||||
}
|
||||
box_filter_low(
|
||||
&self.coeff_a,
|
||||
&mut self.scratch,
|
||||
&mut self.corr_guide_guide,
|
||||
w,
|
||||
h,
|
||||
radius,
|
||||
);
|
||||
box_filter_low(
|
||||
&self.coeff_b,
|
||||
&mut self.scratch,
|
||||
&mut self.corr_guide_mask,
|
||||
w,
|
||||
h,
|
||||
radius,
|
||||
);
|
||||
for i in 0..len {
|
||||
let variance = self.corr_guide_guide[i] - self.mean_guide[i] * self.mean_guide[i];
|
||||
let covariance = self.corr_guide_mask[i] - self.mean_guide[i] * self.mean_mask[i];
|
||||
let a = covariance / (variance.max(0.0) + GUIDED_FILTER_EPSILON);
|
||||
self.coeff_a[i] = a;
|
||||
self.coeff_b[i] = self.mean_mask[i] - a * self.mean_guide[i];
|
||||
}
|
||||
box_filter_low(
|
||||
&self.coeff_a,
|
||||
&mut self.scratch,
|
||||
&mut self.mean_guide,
|
||||
w,
|
||||
h,
|
||||
radius,
|
||||
);
|
||||
box_filter_low(
|
||||
&self.coeff_b,
|
||||
&mut self.scratch,
|
||||
&mut self.mean_mask,
|
||||
w,
|
||||
h,
|
||||
radius,
|
||||
);
|
||||
}
|
||||
|
||||
fn apply_guided_coefficients(&self, luma: &[u8], mask: &mut [u8]) {
|
||||
assert!(luma.len() >= self.width * self.height);
|
||||
assert!(mask.len() >= self.width * self.height);
|
||||
let low_w = self.low_width;
|
||||
for y in 0..self.height {
|
||||
let row_fixed = bilinear_fixed_coord(y, self.height, self.low_height);
|
||||
let sy = (row_fixed / 256) as usize;
|
||||
let fy = (row_fixed % 256) as f32 / 256.0;
|
||||
let sy_next = (sy + 1).min(self.low_height - 1);
|
||||
let row = y * self.width;
|
||||
for x in 0..self.width {
|
||||
let col_fixed = self.column_fixed[x];
|
||||
let sx = (col_fixed / 256) as usize;
|
||||
let fx = (col_fixed % 256) as f32 / 256.0;
|
||||
let sx_next = (sx + 1).min(low_w - 1);
|
||||
let a = bilinear_sample(&self.mean_guide, low_w, sx, sx_next, sy, sy_next, fx, fy);
|
||||
let b = bilinear_sample(&self.mean_mask, low_w, sx, sx_next, sy, sy_next, fx, fy);
|
||||
let q = a * (f32::from(luma[row + x]) / 255.0) + b;
|
||||
let shaped = (q * 255.0 + 0.5).clamp(0.0, 255.0) as usize;
|
||||
mask[row + x] = self.shape_lut[shaped.min(LUT_LEN - 1)];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn bilinear_fixed_coord(index: usize, full_len: usize, low_len: usize) -> u32 {
|
||||
assert!(full_len >= 1);
|
||||
assert!(low_len >= 1);
|
||||
if full_len == 1 {
|
||||
return 0;
|
||||
}
|
||||
(index * (low_len - 1) * 256 / (full_len - 1)) as u32
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
pub(crate) fn bilinear_sample(
|
||||
plane: &[f32],
|
||||
width: usize,
|
||||
sx: usize,
|
||||
sx_next: usize,
|
||||
sy: usize,
|
||||
sy_next: usize,
|
||||
fx: f32,
|
||||
fy: f32,
|
||||
) -> f32 {
|
||||
assert!(sy * width + sx_next < plane.len());
|
||||
assert!(sy_next * width + sx_next < plane.len());
|
||||
let top = plane[sy * width + sx] * (1.0 - fx) + plane[sy * width + sx_next] * fx;
|
||||
let bottom = plane[sy_next * width + sx] * (1.0 - fx) + plane[sy_next * width + sx_next] * fx;
|
||||
top * (1.0 - fy) + bottom * fy
|
||||
}
|
||||
|
||||
fn temporal_uncertainty(probability: f32) -> f32 {
|
||||
assert!(probability >= 0.0);
|
||||
assert!(probability <= 1.0);
|
||||
let x = (probability - 0.5) * (probability - 0.5);
|
||||
let polynomial = x
|
||||
* (TEMPORAL_UNCERTAINTY_C1
|
||||
+ x * (TEMPORAL_UNCERTAINTY_C2
|
||||
+ x * (TEMPORAL_UNCERTAINTY_C3
|
||||
+ x * (TEMPORAL_UNCERTAINTY_C4 + x * TEMPORAL_UNCERTAINTY_C5))));
|
||||
1.0 - polynomial.min(1.0)
|
||||
}
|
||||
|
||||
fn temporal_weight_lut() -> [u16; LUT_LEN] {
|
||||
let mut lut = [0u16; LUT_LEN];
|
||||
for (value, slot) in lut.iter_mut().enumerate() {
|
||||
let probability = value as f32 / 255.0;
|
||||
let weight = temporal_uncertainty(probability) * TEMPORAL_COMBINE_RATIO;
|
||||
assert!(weight >= 0.0);
|
||||
assert!(weight <= 1.0);
|
||||
*slot = (weight * 256.0 + 0.5) as u16;
|
||||
}
|
||||
lut
|
||||
}
|
||||
|
||||
fn shape_lut() -> [u8; LUT_LEN] {
|
||||
let span = SHAPE_SMOOTHSTEP_EDGE_HIGH - SHAPE_SMOOTHSTEP_EDGE_LOW;
|
||||
assert!(span > 0.0);
|
||||
let mut lut = [0u8; LUT_LEN];
|
||||
for (value, slot) in lut.iter_mut().enumerate() {
|
||||
let probability = value as f32 / 255.0;
|
||||
let t = ((probability - SHAPE_SMOOTHSTEP_EDGE_LOW) / span).clamp(0.0, 1.0);
|
||||
let smooth = t * t * (3.0 - 2.0 * t);
|
||||
*slot = (smooth * 255.0 + 0.5) as u8;
|
||||
}
|
||||
assert_eq!(lut[0], 0);
|
||||
assert_eq!(lut[LUT_LEN - 1], 255);
|
||||
lut
|
||||
}
|
||||
|
||||
pub(crate) fn box_filter_low(
|
||||
src: &[f32],
|
||||
scratch: &mut [f32],
|
||||
dst: &mut [f32],
|
||||
width: usize,
|
||||
height: usize,
|
||||
radius: usize,
|
||||
) {
|
||||
assert!(width >= 1);
|
||||
assert!(height >= 1);
|
||||
assert!(src.len() >= width * height);
|
||||
assert!(scratch.len() >= width * height);
|
||||
assert!(dst.len() >= width * height);
|
||||
box_filter_rows_low(src, scratch, width, height, radius);
|
||||
box_filter_columns_low(scratch, dst, width, height, radius);
|
||||
}
|
||||
|
||||
fn box_filter_rows_low(src: &[f32], dst: &mut [f32], width: usize, height: usize, radius: usize) {
|
||||
assert!(width >= 1);
|
||||
assert!(src.len() >= width * height);
|
||||
for y in 0..height {
|
||||
let row = y * width;
|
||||
let mut start = 0usize;
|
||||
let mut end = radius.min(width - 1);
|
||||
let mut sum: f32 = src[row..=row + end].iter().sum();
|
||||
for x in 0..width {
|
||||
dst[row + x] = sum / ((end - start + 1) as f32);
|
||||
let next_end = (x + 1 + radius).min(width - 1);
|
||||
if next_end > end {
|
||||
sum += src[row + next_end];
|
||||
end = next_end;
|
||||
}
|
||||
let next_start = (x + 1).saturating_sub(radius);
|
||||
if next_start > start {
|
||||
sum -= src[row + start];
|
||||
start = next_start;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn box_filter_columns_low(
|
||||
src: &[f32],
|
||||
dst: &mut [f32],
|
||||
width: usize,
|
||||
height: usize,
|
||||
radius: usize,
|
||||
) {
|
||||
assert!(width >= 1);
|
||||
assert!(width <= FRAME_EDGE_MAX);
|
||||
assert!(height >= 1);
|
||||
let mut sums = [0.0f32; FRAME_EDGE_MAX];
|
||||
let mut start = 0usize;
|
||||
let mut end = radius.min(height - 1);
|
||||
for y in 0..=end {
|
||||
let row = y * width;
|
||||
for x in 0..width {
|
||||
sums[x] += src[row + x];
|
||||
}
|
||||
}
|
||||
for y in 0..height {
|
||||
let scale = 1.0 / ((end - start + 1) as f32);
|
||||
let row = y * width;
|
||||
for x in 0..width {
|
||||
dst[row + x] = sums[x] * scale;
|
||||
}
|
||||
let next_end = (y + 1 + radius).min(height - 1);
|
||||
if next_end > end {
|
||||
let next_row = next_end * width;
|
||||
for x in 0..width {
|
||||
sums[x] += src[next_row + x];
|
||||
}
|
||||
end = next_end;
|
||||
}
|
||||
let next_start = (y + 1).saturating_sub(radius);
|
||||
if next_start > start {
|
||||
let previous_row = start * width;
|
||||
for x in 0..width {
|
||||
sums[x] -= src[previous_row + x];
|
||||
}
|
||||
start = next_start;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn morph_pass_low(
|
||||
src: &[f32],
|
||||
scratch: &mut [f32],
|
||||
dst: &mut [f32],
|
||||
width: usize,
|
||||
height: usize,
|
||||
select: fn(f32, f32) -> f32,
|
||||
) {
|
||||
assert!(width >= 1);
|
||||
assert!(height >= 1);
|
||||
assert!(src.len() >= width * height);
|
||||
assert!(scratch.len() >= width * height);
|
||||
assert!(dst.len() >= width * height);
|
||||
for y in 0..height {
|
||||
let row = y * width;
|
||||
for x in 0..width {
|
||||
let left = src[row + x.saturating_sub(1)];
|
||||
let right = src[row + (x + 1).min(width - 1)];
|
||||
scratch[row + x] = select(select(left, src[row + x]), right);
|
||||
}
|
||||
}
|
||||
for y in 0..height {
|
||||
let above = y.saturating_sub(1) * width;
|
||||
let below = (y + 1).min(height - 1) * width;
|
||||
let row = y * width;
|
||||
for x in 0..width {
|
||||
dst[row + x] = select(
|
||||
select(scratch[above + x], scratch[row + x]),
|
||||
scratch[below + x],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn gradient_luma(width: usize, height: usize) -> Vec<u8> {
|
||||
let mut luma = vec![0u8; width * height];
|
||||
for (index, value) in luma.iter_mut().enumerate() {
|
||||
*value = ((index % width) * 255 / (width - 1).max(1)) as u8;
|
||||
}
|
||||
luma
|
||||
}
|
||||
|
||||
fn left_half_mask(width: usize, height: usize) -> Vec<u8> {
|
||||
let mut mask = vec![0u8; width * height];
|
||||
for y in 0..height {
|
||||
for x in 0..width / 2 {
|
||||
mask[y * width + x] = 255;
|
||||
}
|
||||
}
|
||||
mask
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn temporal_weight_lut_smooths_uncertain_values_and_trusts_confident_ones() {
|
||||
let lut = temporal_weight_lut();
|
||||
|
||||
assert!(lut[128] >= 170);
|
||||
assert!(lut[128] <= 182);
|
||||
assert!(lut[0] <= 8);
|
||||
assert!(lut[255] <= 8);
|
||||
assert!(lut[64] > lut[16]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn temporal_blend_pulls_uncertain_pixels_toward_previous_mask() {
|
||||
let mut refiner = MaskRefiner::new(8, 8);
|
||||
let mut first = vec![128u8; 64];
|
||||
refiner.blend_temporal(&mut first);
|
||||
let mut second = vec![128u8; 64];
|
||||
second[0] = 255;
|
||||
second[1] = 130;
|
||||
|
||||
refiner.blend_temporal(&mut second);
|
||||
|
||||
assert_eq!(second[0], 255);
|
||||
assert!(second[1] < 130);
|
||||
assert_eq!(second[63], 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn temporal_blend_first_frame_passes_mask_through_unchanged() {
|
||||
let mut refiner = MaskRefiner::new(8, 8);
|
||||
let mut mask = vec![37u8; 64];
|
||||
|
||||
refiner.blend_temporal(&mut mask);
|
||||
|
||||
assert!(mask.iter().all(|value| *value == 37));
|
||||
assert!(refiner.previous_mask_valid);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shape_lut_is_monotonic_and_saturates_at_both_ends() {
|
||||
let lut = shape_lut();
|
||||
|
||||
for value in 1..LUT_LEN {
|
||||
assert!(lut[value] >= lut[value - 1]);
|
||||
}
|
||||
assert_eq!(lut[(255.0 * SHAPE_SMOOTHSTEP_EDGE_LOW) as usize - 4], 0);
|
||||
assert_eq!(lut[(255.0 * SHAPE_SMOOTHSTEP_EDGE_HIGH) as usize + 4], 255);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refine_keeps_solid_person_and_background_regions_saturated() {
|
||||
let width = 64usize;
|
||||
let height = 48usize;
|
||||
let mut refiner = MaskRefiner::new(width, height);
|
||||
let luma = {
|
||||
let mut luma = vec![32u8; width * height];
|
||||
for y in 0..height {
|
||||
for x in width / 2..width {
|
||||
luma[y * width + x] = 224;
|
||||
}
|
||||
}
|
||||
luma
|
||||
};
|
||||
let mut mask = left_half_mask(width, height);
|
||||
|
||||
refiner.refine(&luma, &mut mask);
|
||||
|
||||
assert_eq!(mask[24 * width], 255);
|
||||
assert_eq!(mask[24 * width + 4], 255);
|
||||
assert_eq!(mask[24 * width + width - 1], 0);
|
||||
assert_eq!(mask[24 * width + width - 5], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refine_snaps_mask_transition_to_the_luma_edge() {
|
||||
let width = 64usize;
|
||||
let height = 48usize;
|
||||
let mut refiner = MaskRefiner::new(width, height);
|
||||
let mut luma = vec![16u8; width * height];
|
||||
for y in 0..height {
|
||||
for x in 0..width / 2 {
|
||||
luma[y * width + x] = 240;
|
||||
}
|
||||
}
|
||||
let mut blurry_mask = vec![0u8; width * height];
|
||||
for y in 0..height {
|
||||
for x in 0..width {
|
||||
let distance = (width as i32 / 2 - x as i32).clamp(-12, 12);
|
||||
blurry_mask[y * width + x] = (127 + distance * 10).clamp(0, 255) as u8;
|
||||
}
|
||||
}
|
||||
|
||||
refiner.refine(&luma, &mut blurry_mask);
|
||||
|
||||
let row = 24 * width;
|
||||
assert!(blurry_mask[row + width / 2 - 8] > 220);
|
||||
assert!(blurry_mask[row + width / 2 + 8] < 35);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refine_fills_small_holes_inside_the_person() {
|
||||
let width = 64usize;
|
||||
let height = 48usize;
|
||||
let mut refiner = MaskRefiner::new(width, height);
|
||||
let luma = vec![128u8; width * height];
|
||||
let mut mask = vec![255u8; width * height];
|
||||
mask[24 * width + 32] = 0;
|
||||
|
||||
refiner.refine(&luma, &mut mask);
|
||||
|
||||
assert!(mask[24 * width + 32] > 200);
|
||||
assert_eq!(mask[0], 255);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refine_handles_minimum_dimensions_without_panicking() {
|
||||
let mut refiner = MaskRefiner::new(2, 2);
|
||||
let luma = vec![128u8; 4];
|
||||
let mut mask = vec![255u8; 4];
|
||||
|
||||
refiner.refine(&luma, &mut mask);
|
||||
|
||||
assert_eq!(mask.len(), 4);
|
||||
assert!(mask.iter().all(|value| *value == 255));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn box_filter_low_preserves_constant_planes_exactly() {
|
||||
let width = 9usize;
|
||||
let height = 7usize;
|
||||
let src = vec![0.625f32; width * height];
|
||||
let mut scratch = vec![0.0f32; width * height];
|
||||
let mut dst = vec![0.0f32; width * height];
|
||||
|
||||
box_filter_low(&src, &mut scratch, &mut dst, width, height, 4);
|
||||
|
||||
for value in dst {
|
||||
assert!((value - 0.625).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn morph_close_removes_single_pixel_pits_and_keeps_plateaus() {
|
||||
let width = 8usize;
|
||||
let height = 8usize;
|
||||
let mut src = vec![1.0f32; width * height];
|
||||
src[3 * width + 3] = 0.0;
|
||||
let mut scratch = vec![0.0f32; width * height];
|
||||
let mut maxed = vec![0.0f32; width * height];
|
||||
let mut closed = vec![0.0f32; width * height];
|
||||
|
||||
morph_pass_low(&src, &mut scratch, &mut maxed, width, height, f32::max);
|
||||
morph_pass_low(&maxed, &mut scratch, &mut closed, width, height, f32::min);
|
||||
|
||||
assert!(closed[3 * width + 3] > 0.99);
|
||||
assert!(closed[0] > 0.99);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refine_converges_to_stable_mask_over_repeated_identical_frames() {
|
||||
let width = 32usize;
|
||||
let height = 24usize;
|
||||
let mut refiner = MaskRefiner::new(width, height);
|
||||
let luma = gradient_luma(width, height);
|
||||
let raw = left_half_mask(width, height);
|
||||
let mut previous_output = vec![0u8; width * height];
|
||||
for iteration in 0..8 {
|
||||
let mut mask = raw.clone();
|
||||
refiner.refine(&luma, &mut mask);
|
||||
if iteration == 7 {
|
||||
let drift: i32 = mask
|
||||
.iter()
|
||||
.zip(previous_output.iter())
|
||||
.map(|(a, b)| (i32::from(*a) - i32::from(*b)).abs())
|
||||
.sum();
|
||||
assert!(drift <= (width * height) as i32);
|
||||
}
|
||||
previous_output.copy_from_slice(&mask);
|
||||
}
|
||||
assert_eq!(previous_output[12 * width], 255);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
#![allow(dead_code)]
|
||||
|
||||
#[cfg(feature = "publisher")]
|
||||
use livekit::webrtc::video_frame::{VideoBuffer, VideoFrame, VideoRotation, native::NativeBuffer};
|
||||
#[cfg(feature = "publisher")]
|
||||
use livekit::webrtc::video_source::native::NativeVideoSource;
|
||||
use napi_derive::napi;
|
||||
|
||||
pub const NATIVE_CAMERA_FRAME_QUEUE_CAPACITY: usize = 3;
|
||||
const MIN_NATIVE_CAMERA_EDGE: u32 = 2;
|
||||
const MAX_NATIVE_CAMERA_EDGE: u32 = 8192;
|
||||
const TRANSPORT_CV_PIXEL_BUFFER: &str = "cvPixelBuffer";
|
||||
const TRANSPORT_D3D11_TEXTURE: &str = "d3d11Texture";
|
||||
const TRANSPORT_DMABUF: &str = "dmabuf";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum NativeCameraTransport {
|
||||
CvPixelBuffer,
|
||||
D3d11Texture,
|
||||
Dmabuf,
|
||||
}
|
||||
|
||||
impl NativeCameraTransport {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::CvPixelBuffer => TRANSPORT_CV_PIXEL_BUFFER,
|
||||
Self::D3d11Texture => TRANSPORT_D3D11_TEXTURE,
|
||||
Self::Dmabuf => TRANSPORT_DMABUF,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn required_transports() -> &'static [NativeCameraTransport] {
|
||||
const TRANSPORTS: &[NativeCameraTransport] = &[
|
||||
NativeCameraTransport::CvPixelBuffer,
|
||||
NativeCameraTransport::D3d11Texture,
|
||||
NativeCameraTransport::Dmabuf,
|
||||
];
|
||||
assert_eq!(TRANSPORTS.len(), 3);
|
||||
TRANSPORTS
|
||||
}
|
||||
|
||||
pub fn required_transport_names() -> [&'static str; 3] {
|
||||
let transports = required_transports();
|
||||
[
|
||||
transports[0].as_str(),
|
||||
transports[1].as_str(),
|
||||
transports[2].as_str(),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn platform_native_backgrounds_available() -> bool {
|
||||
platform_unavailable_reason().is_none()
|
||||
}
|
||||
|
||||
pub fn camera_backgrounds_available() -> bool {
|
||||
cfg!(feature = "camera-native")
|
||||
}
|
||||
|
||||
#[napi]
|
||||
pub fn has_native_camera_backgrounds() -> bool {
|
||||
camera_backgrounds_available()
|
||||
}
|
||||
|
||||
pub fn platform_unavailable_reason() -> Option<&'static str> {
|
||||
Some(match std::env::consts::OS {
|
||||
"macos" => "macOS AVFoundation CVPixelBuffer camera backend is not compiled",
|
||||
"windows" => "Windows Media Foundation D3D11 camera backend is not compiled",
|
||||
"linux" => "Linux PipeWire/V4L2 dmabuf camera backend is not compiled",
|
||||
_ => "native platform-buffer camera backend is unsupported on this platform",
|
||||
})
|
||||
}
|
||||
|
||||
pub fn validate_native_frame_dimensions(width: u32, height: u32) -> bool {
|
||||
if width < MIN_NATIVE_CAMERA_EDGE {
|
||||
return false;
|
||||
}
|
||||
if height < MIN_NATIVE_CAMERA_EDGE {
|
||||
return false;
|
||||
}
|
||||
if !width.is_multiple_of(2) {
|
||||
return false;
|
||||
}
|
||||
if !height.is_multiple_of(2) {
|
||||
return false;
|
||||
}
|
||||
width <= MAX_NATIVE_CAMERA_EDGE && height <= MAX_NATIVE_CAMERA_EDGE
|
||||
}
|
||||
|
||||
pub fn unavailable_error() -> String {
|
||||
let reason = platform_unavailable_reason().unwrap_or("native camera backend unavailable");
|
||||
format!(
|
||||
"native camera backgrounds require platform camera buffers ({}, {}, {}): {reason}",
|
||||
TRANSPORT_CV_PIXEL_BUFFER, TRANSPORT_D3D11_TEXTURE, TRANSPORT_DMABUF
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(feature = "publisher")]
|
||||
pub fn publish_native_buffer(
|
||||
source: &NativeVideoSource,
|
||||
buffer: NativeBuffer,
|
||||
timestamp_us: i64,
|
||||
) -> bool {
|
||||
assert!(timestamp_us >= 0);
|
||||
let width = buffer.width();
|
||||
let height = buffer.height();
|
||||
if !validate_native_frame_dimensions(width, height) {
|
||||
return false;
|
||||
}
|
||||
source.capture_frame(&VideoFrame {
|
||||
rotation: VideoRotation::VideoRotation0,
|
||||
timestamp_us,
|
||||
frame_metadata: None,
|
||||
buffer,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn required_transports_are_stable() {
|
||||
assert_eq!(
|
||||
required_transport_names(),
|
||||
["cvPixelBuffer", "d3d11Texture", "dmabuf"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_capacity_stays_bounded_for_realtime_capture() {
|
||||
assert_eq!(NATIVE_CAMERA_FRAME_QUEUE_CAPACITY, 3);
|
||||
assert!(NATIVE_CAMERA_FRAME_QUEUE_CAPACITY < 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn camera_background_capability_tracks_native_camera_feature() {
|
||||
assert_eq!(
|
||||
camera_backgrounds_available(),
|
||||
cfg!(feature = "camera-native")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_frame_dimensions_require_even_reasonable_sizes() {
|
||||
assert!(validate_native_frame_dimensions(1280, 720));
|
||||
assert!(!validate_native_frame_dimensions(0, 720));
|
||||
assert!(!validate_native_frame_dimensions(1280, 1));
|
||||
assert!(!validate_native_frame_dimensions(1279, 720));
|
||||
assert!(!validate_native_frame_dimensions(1280, 721));
|
||||
assert!(!validate_native_frame_dimensions(16_384, 720));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_error_names_all_required_native_transports() {
|
||||
let error = unavailable_error();
|
||||
assert!(error.contains("cvPixelBuffer"));
|
||||
assert!(error.contains("d3d11Texture"));
|
||||
assert!(error.contains("dmabuf"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,681 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
pub const PERSON_MASK_BACKGROUND: u8 = 0;
|
||||
pub const PERSON_MASK_PERSON: u8 = 255;
|
||||
pub const SEGMENTATION_FRAME_BUDGET_MS: u64 = 12;
|
||||
pub const SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX: u32 = 30;
|
||||
|
||||
pub trait PersonMaskSource {
|
||||
fn mask_into(&mut self, frame: &crate::yuv::I420, mask: &mut [u8]) -> bool;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SegmentationQualityGovernor {
|
||||
consecutive_slow_frames: u32,
|
||||
downgraded: bool,
|
||||
}
|
||||
|
||||
impl SegmentationQualityGovernor {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn record_frame_duration_ms(&mut self, duration_ms: u64) -> bool {
|
||||
assert!(self.consecutive_slow_frames < SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX);
|
||||
if self.downgraded {
|
||||
return false;
|
||||
}
|
||||
if duration_ms <= SEGMENTATION_FRAME_BUDGET_MS {
|
||||
self.consecutive_slow_frames = 0;
|
||||
return false;
|
||||
}
|
||||
self.consecutive_slow_frames += 1;
|
||||
assert!(self.consecutive_slow_frames <= SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX);
|
||||
if self.consecutive_slow_frames < SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX {
|
||||
return false;
|
||||
}
|
||||
self.consecutive_slow_frames = 0;
|
||||
self.downgraded = true;
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_person_mask_source(width: u32, height: u32) -> Option<Box<dyn PersonMaskSource>> {
|
||||
assert!(width >= 2);
|
||||
assert!(height >= 2);
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
if let Some(source) = vision::VisionPersonMaskSource::new(width, height) {
|
||||
return Some(Box::new(source) as Box<dyn PersonMaskSource>);
|
||||
}
|
||||
}
|
||||
selfie::SelfieMaskSource::new(width, height)
|
||||
.map(|source| Box::new(source) as Box<dyn PersonMaskSource>)
|
||||
}
|
||||
|
||||
pub fn resize_mask_bilinear(
|
||||
src: &[u8],
|
||||
src_width: usize,
|
||||
src_height: usize,
|
||||
src_stride: usize,
|
||||
dst: &mut [u8],
|
||||
dst_width: usize,
|
||||
dst_height: usize,
|
||||
) {
|
||||
assert!(src_width >= 1);
|
||||
assert!(src_height >= 1);
|
||||
assert!(src_stride >= src_width);
|
||||
assert!(dst_width >= 1);
|
||||
assert!(dst_height >= 1);
|
||||
assert!(src.len() >= src_stride * (src_height - 1) + src_width);
|
||||
assert!(dst.len() >= dst_width * dst_height);
|
||||
|
||||
for y in 0..dst_height {
|
||||
let sy_fixed = if dst_height == 1 {
|
||||
0
|
||||
} else {
|
||||
y * (src_height - 1) * 256 / (dst_height - 1)
|
||||
};
|
||||
let sy = sy_fixed / 256;
|
||||
let fy = (sy_fixed % 256) as u32;
|
||||
let sy_next = (sy + 1).min(src_height - 1);
|
||||
for x in 0..dst_width {
|
||||
let sx_fixed = if dst_width == 1 {
|
||||
0
|
||||
} else {
|
||||
x * (src_width - 1) * 256 / (dst_width - 1)
|
||||
};
|
||||
let sx = sx_fixed / 256;
|
||||
let fx = (sx_fixed % 256) as u32;
|
||||
let sx_next = (sx + 1).min(src_width - 1);
|
||||
let top = u32::from(src[sy * src_stride + sx]) * (256 - fx)
|
||||
+ u32::from(src[sy * src_stride + sx_next]) * fx;
|
||||
let bottom = u32::from(src[sy_next * src_stride + sx]) * (256 - fx)
|
||||
+ u32::from(src[sy_next * src_stride + sx_next]) * fx;
|
||||
dst[y * dst_width + x] = ((top * (256 - fy) + bottom * fy) >> 16) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod selfie {
|
||||
use super::PersonMaskSource;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tract_onnx::prelude::*;
|
||||
|
||||
const MODEL_BYTES: &[u8] = include_bytes!("../models/selfie_segmenter_landscape.onnx");
|
||||
const MODEL_INPUT_WIDTH: usize = 256;
|
||||
const MODEL_INPUT_HEIGHT: usize = 144;
|
||||
const MODEL_INPUT_CHANNELS: usize = 3;
|
||||
const MODEL_INPUT_LEN: usize = MODEL_INPUT_WIDTH * MODEL_INPUT_HEIGHT * MODEL_INPUT_CHANNELS;
|
||||
const MODEL_CHROMA_WIDTH: usize = MODEL_INPUT_WIDTH / 2;
|
||||
const MODEL_CHROMA_HEIGHT: usize = MODEL_INPUT_HEIGHT / 2;
|
||||
const INFERENCE_FRAME_INTERVAL_FULL: u32 = 1;
|
||||
const INFERENCE_FRAME_INTERVAL_DOWNGRADED: u32 = 2;
|
||||
|
||||
type SelfiePlan = TypedRunnableModel;
|
||||
|
||||
fn shared_plan() -> Option<Arc<SelfiePlan>> {
|
||||
static PLAN: OnceLock<Option<Arc<SelfiePlan>>> = OnceLock::new();
|
||||
PLAN.get_or_init(|| match load_plan() {
|
||||
Ok(plan) => Some(plan),
|
||||
Err(error) => {
|
||||
eprintln!(
|
||||
"webrtc-sender: selfie segmentation model failed to load; camera \
|
||||
background effects fall back to the portrait ellipse: {error}"
|
||||
);
|
||||
None
|
||||
}
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn load_plan() -> TractResult<Arc<SelfiePlan>> {
|
||||
let mut reader = std::io::Cursor::new(MODEL_BYTES);
|
||||
tract_onnx::onnx()
|
||||
.model_for_read(&mut reader)?
|
||||
.with_input_fact(
|
||||
0,
|
||||
f32::fact([
|
||||
1,
|
||||
MODEL_INPUT_HEIGHT,
|
||||
MODEL_INPUT_WIDTH,
|
||||
MODEL_INPUT_CHANNELS,
|
||||
])
|
||||
.into(),
|
||||
)?
|
||||
.into_optimized()?
|
||||
.into_runnable()
|
||||
}
|
||||
|
||||
pub struct SelfieMaskSource {
|
||||
width: u32,
|
||||
height: u32,
|
||||
plan: Arc<SelfiePlan>,
|
||||
luma_low: Vec<u8>,
|
||||
chroma_u_low: Vec<u8>,
|
||||
chroma_v_low: Vec<u8>,
|
||||
input_rgb: Vec<f32>,
|
||||
raw_mask_low: Vec<u8>,
|
||||
raw_mask_valid: bool,
|
||||
frame_counter: u32,
|
||||
inference_interval: u32,
|
||||
inference_error_logged: bool,
|
||||
governor: super::SegmentationQualityGovernor,
|
||||
}
|
||||
|
||||
impl SelfieMaskSource {
|
||||
pub fn new(width: u32, height: u32) -> Option<Self> {
|
||||
assert!(width >= 2);
|
||||
assert!(height >= 2);
|
||||
let plan = shared_plan()?;
|
||||
Some(Self {
|
||||
width,
|
||||
height,
|
||||
plan,
|
||||
luma_low: vec![0; MODEL_INPUT_WIDTH * MODEL_INPUT_HEIGHT],
|
||||
chroma_u_low: vec![128; MODEL_CHROMA_WIDTH * MODEL_CHROMA_HEIGHT],
|
||||
chroma_v_low: vec![128; MODEL_CHROMA_WIDTH * MODEL_CHROMA_HEIGHT],
|
||||
input_rgb: vec![0.0; MODEL_INPUT_LEN],
|
||||
raw_mask_low: vec![0; MODEL_INPUT_WIDTH * MODEL_INPUT_HEIGHT],
|
||||
raw_mask_valid: false,
|
||||
frame_counter: 0,
|
||||
inference_interval: INFERENCE_FRAME_INTERVAL_FULL,
|
||||
inference_error_logged: false,
|
||||
governor: super::SegmentationQualityGovernor::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn fill_model_input(&mut self, frame: &crate::yuv::I420) {
|
||||
assert_eq!(frame.width, self.width);
|
||||
assert_eq!(frame.height, self.height);
|
||||
let width = self.width as usize;
|
||||
let height = self.height as usize;
|
||||
super::resize_mask_bilinear(
|
||||
&frame.y,
|
||||
width,
|
||||
height,
|
||||
width,
|
||||
&mut self.luma_low,
|
||||
MODEL_INPUT_WIDTH,
|
||||
MODEL_INPUT_HEIGHT,
|
||||
);
|
||||
super::resize_mask_bilinear(
|
||||
&frame.u,
|
||||
width / 2,
|
||||
height / 2,
|
||||
width / 2,
|
||||
&mut self.chroma_u_low,
|
||||
MODEL_CHROMA_WIDTH,
|
||||
MODEL_CHROMA_HEIGHT,
|
||||
);
|
||||
super::resize_mask_bilinear(
|
||||
&frame.v,
|
||||
width / 2,
|
||||
height / 2,
|
||||
width / 2,
|
||||
&mut self.chroma_v_low,
|
||||
MODEL_CHROMA_WIDTH,
|
||||
MODEL_CHROMA_HEIGHT,
|
||||
);
|
||||
for y in 0..MODEL_INPUT_HEIGHT {
|
||||
let row = y * MODEL_INPUT_WIDTH;
|
||||
let chroma_row = (y / 2) * MODEL_CHROMA_WIDTH;
|
||||
for x in 0..MODEL_INPUT_WIDTH {
|
||||
let luma = i32::from(self.luma_low[row + x]) - 16;
|
||||
let cb = i32::from(self.chroma_u_low[chroma_row + x / 2]) - 128;
|
||||
let cr = i32::from(self.chroma_v_low[chroma_row + x / 2]) - 128;
|
||||
let r = ((298 * luma + 409 * cr + 128) >> 8).clamp(0, 255);
|
||||
let g = ((298 * luma - 100 * cb - 208 * cr + 128) >> 8).clamp(0, 255);
|
||||
let b = ((298 * luma + 516 * cb + 128) >> 8).clamp(0, 255);
|
||||
let offset = (row + x) * MODEL_INPUT_CHANNELS;
|
||||
self.input_rgb[offset] = r as f32 / 255.0;
|
||||
self.input_rgb[offset + 1] = g as f32 / 255.0;
|
||||
self.input_rgb[offset + 2] = b as f32 / 255.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_inference(&mut self, frame: &crate::yuv::I420) -> bool {
|
||||
self.fill_model_input(frame);
|
||||
let produced = self.run_model();
|
||||
if !produced && !self.inference_error_logged {
|
||||
self.inference_error_logged = true;
|
||||
eprintln!(
|
||||
"webrtc-sender: selfie segmentation inference failed; reusing the \
|
||||
previous person mask"
|
||||
);
|
||||
}
|
||||
produced
|
||||
}
|
||||
|
||||
fn run_model(&mut self) -> bool {
|
||||
assert_eq!(self.input_rgb.len(), MODEL_INPUT_LEN);
|
||||
let Ok(tensor) = Tensor::from_shape(
|
||||
&[
|
||||
1,
|
||||
MODEL_INPUT_HEIGHT,
|
||||
MODEL_INPUT_WIDTH,
|
||||
MODEL_INPUT_CHANNELS,
|
||||
],
|
||||
&self.input_rgb,
|
||||
) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(result) = self.plan.run(tvec!(tensor.into())) else {
|
||||
return false;
|
||||
};
|
||||
let Some(output) = result.first() else {
|
||||
return false;
|
||||
};
|
||||
let Ok(alphas) = output.to_plain_array_view::<f32>() else {
|
||||
return false;
|
||||
};
|
||||
if alphas.len() != self.raw_mask_low.len() {
|
||||
return false;
|
||||
}
|
||||
for (slot, alpha) in self.raw_mask_low.iter_mut().zip(alphas.iter()) {
|
||||
*slot = (alpha * 255.0 + 0.5).clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn record_inference_duration(&mut self, duration_ms: u64) {
|
||||
if self.governor.record_frame_duration_ms(duration_ms) {
|
||||
self.inference_interval = INFERENCE_FRAME_INTERVAL_DOWNGRADED;
|
||||
eprintln!(
|
||||
"webrtc-sender: selfie segmentation exceeded the {}ms frame budget for {} \
|
||||
consecutive frames; downgrading to inference every {} frames",
|
||||
super::SEGMENTATION_FRAME_BUDGET_MS,
|
||||
super::SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX,
|
||||
INFERENCE_FRAME_INTERVAL_DOWNGRADED
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PersonMaskSource for SelfieMaskSource {
|
||||
fn mask_into(&mut self, frame: &crate::yuv::I420, mask: &mut [u8]) -> bool {
|
||||
let width = self.width as usize;
|
||||
let height = self.height as usize;
|
||||
assert!(mask.len() >= width * height);
|
||||
if frame.width != self.width || frame.height != self.height {
|
||||
return false;
|
||||
}
|
||||
assert!(self.inference_interval >= 1);
|
||||
let due = self.frame_counter.is_multiple_of(self.inference_interval);
|
||||
self.frame_counter = self.frame_counter.wrapping_add(1);
|
||||
if due || !self.raw_mask_valid {
|
||||
let started = std::time::Instant::now();
|
||||
if self.run_inference(frame) {
|
||||
self.raw_mask_valid = true;
|
||||
let duration_ms = started.elapsed().as_millis() as u64;
|
||||
self.record_inference_duration(duration_ms);
|
||||
}
|
||||
}
|
||||
if !self.raw_mask_valid {
|
||||
return false;
|
||||
}
|
||||
super::resize_mask_bilinear(
|
||||
&self.raw_mask_low,
|
||||
MODEL_INPUT_WIDTH,
|
||||
MODEL_INPUT_HEIGHT,
|
||||
MODEL_INPUT_WIDTH,
|
||||
mask,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn synthetic_frame(width: u32, height: u32) -> crate::yuv::I420 {
|
||||
let mut frame = crate::yuv::I420::new(width, height).unwrap();
|
||||
for (index, value) in frame.y.iter_mut().enumerate() {
|
||||
*value = ((index * 31 + 17) % 220) as u8 + 16;
|
||||
}
|
||||
frame.u.fill(128);
|
||||
frame.v.fill(128);
|
||||
frame
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selfie_source_produces_full_range_mask_for_synthetic_frames() {
|
||||
let mut source = SelfieMaskSource::new(128, 96).expect("bundled model loads");
|
||||
let frame = synthetic_frame(128, 96);
|
||||
let mut mask = vec![0u8; 128 * 96];
|
||||
|
||||
assert!(source.mask_into(&frame, &mut mask));
|
||||
|
||||
assert_eq!(mask.len(), 128 * 96);
|
||||
assert!(source.raw_mask_valid);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selfie_source_rejects_mismatched_frame_dimensions() {
|
||||
let mut source = SelfieMaskSource::new(128, 96).expect("bundled model loads");
|
||||
let frame = synthetic_frame(64, 48);
|
||||
let mut mask = vec![0u8; 128 * 96];
|
||||
|
||||
assert!(!source.mask_into(&frame, &mut mask));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selfie_source_reuses_cached_mask_between_inference_frames() {
|
||||
let mut source = SelfieMaskSource::new(64, 48).expect("bundled model loads");
|
||||
source.inference_interval = INFERENCE_FRAME_INTERVAL_DOWNGRADED;
|
||||
let frame = synthetic_frame(64, 48);
|
||||
let mut first = vec![0u8; 64 * 48];
|
||||
let mut second = vec![0u8; 64 * 48];
|
||||
|
||||
assert!(source.mask_into(&frame, &mut first));
|
||||
assert!(source.mask_into(&frame, &mut second));
|
||||
|
||||
assert_eq!(first, second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod vision {
|
||||
use super::PersonMaskSource;
|
||||
use core::ptr::NonNull;
|
||||
use objc2::rc::Retained;
|
||||
use objc2_core_foundation::CFRetained;
|
||||
use objc2_core_video::{
|
||||
CVPixelBuffer, CVPixelBufferGetBaseAddress, CVPixelBufferGetBaseAddressOfPlane,
|
||||
CVPixelBufferGetBytesPerRow, CVPixelBufferGetBytesPerRowOfPlane, CVPixelBufferGetHeight,
|
||||
CVPixelBufferGetPixelFormatType, CVPixelBufferGetWidth, CVPixelBufferLockBaseAddress,
|
||||
CVPixelBufferLockFlags, CVPixelBufferUnlockBaseAddress,
|
||||
};
|
||||
use objc2_vision::{
|
||||
VNGeneratePersonSegmentationRequest, VNGeneratePersonSegmentationRequestQualityLevel,
|
||||
VNRequest, VNSequenceRequestHandler,
|
||||
};
|
||||
|
||||
const PIXEL_FORMAT_NV12_FULL_RANGE: u32 = u32::from_be_bytes(*b"420f");
|
||||
const PIXEL_FORMAT_ONE_COMPONENT_8: u32 = u32::from_be_bytes(*b"L008");
|
||||
const MASK_PIXELS_MAX: usize = 8192 * 8192;
|
||||
|
||||
pub struct VisionPersonMaskSource {
|
||||
width: u32,
|
||||
height: u32,
|
||||
pixel_buffer: CFRetained<CVPixelBuffer>,
|
||||
request: Retained<VNGeneratePersonSegmentationRequest>,
|
||||
requests: Retained<objc2_foundation::NSArray<VNRequest>>,
|
||||
handler: Retained<VNSequenceRequestHandler>,
|
||||
quality_governor: super::SegmentationQualityGovernor,
|
||||
}
|
||||
|
||||
impl VisionPersonMaskSource {
|
||||
pub fn new(width: u32, height: u32) -> Option<Self> {
|
||||
assert!(width >= 2);
|
||||
assert!(height >= 2);
|
||||
assert!(width.is_multiple_of(2));
|
||||
assert!(height.is_multiple_of(2));
|
||||
let mut pixel_buffer_out: *mut CVPixelBuffer = core::ptr::null_mut();
|
||||
let status = unsafe {
|
||||
objc2_core_video::CVPixelBufferCreate(
|
||||
None,
|
||||
width as usize,
|
||||
height as usize,
|
||||
PIXEL_FORMAT_NV12_FULL_RANGE,
|
||||
None,
|
||||
NonNull::new(&mut pixel_buffer_out)?,
|
||||
)
|
||||
};
|
||||
if status != 0 {
|
||||
return None;
|
||||
}
|
||||
let pixel_buffer = unsafe { CFRetained::from_raw(NonNull::new(pixel_buffer_out)?) };
|
||||
let request = unsafe { VNGeneratePersonSegmentationRequest::new() };
|
||||
unsafe {
|
||||
request.setQualityLevel(VNGeneratePersonSegmentationRequestQualityLevel::Balanced);
|
||||
request.setOutputPixelFormat(PIXEL_FORMAT_ONE_COMPONENT_8);
|
||||
}
|
||||
let request_as_base: Retained<VNRequest> =
|
||||
Retained::into_super(Retained::into_super(Retained::into_super(request.clone())));
|
||||
let requests = objc2_foundation::NSArray::from_retained_slice(&[request_as_base]);
|
||||
assert_eq!(requests.len(), 1);
|
||||
let handler = unsafe { VNSequenceRequestHandler::new() };
|
||||
Some(Self {
|
||||
width,
|
||||
height,
|
||||
pixel_buffer,
|
||||
request,
|
||||
requests,
|
||||
handler,
|
||||
quality_governor: super::SegmentationQualityGovernor::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn downgrade_to_fast_quality(&self) {
|
||||
unsafe {
|
||||
self.request
|
||||
.setQualityLevel(VNGeneratePersonSegmentationRequestQualityLevel::Fast);
|
||||
}
|
||||
eprintln!(
|
||||
"webrtc-sender: person segmentation exceeded the {}ms frame budget for {} \
|
||||
consecutive frames; downgrading Vision quality from balanced to fast",
|
||||
super::SEGMENTATION_FRAME_BUDGET_MS,
|
||||
super::SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX
|
||||
);
|
||||
}
|
||||
|
||||
fn fill_pixel_buffer(&self, frame: &crate::yuv::I420) -> bool {
|
||||
assert_eq!(frame.width, self.width);
|
||||
assert_eq!(frame.height, self.height);
|
||||
let width = self.width as usize;
|
||||
let height = self.height as usize;
|
||||
let lock_flags = CVPixelBufferLockFlags(0);
|
||||
let lock_status =
|
||||
unsafe { CVPixelBufferLockBaseAddress(&self.pixel_buffer, lock_flags) };
|
||||
if lock_status != 0 {
|
||||
return false;
|
||||
}
|
||||
let y_base = CVPixelBufferGetBaseAddressOfPlane(&self.pixel_buffer, 0);
|
||||
let y_stride = CVPixelBufferGetBytesPerRowOfPlane(&self.pixel_buffer, 0);
|
||||
let uv_base = CVPixelBufferGetBaseAddressOfPlane(&self.pixel_buffer, 1);
|
||||
let uv_stride = CVPixelBufferGetBytesPerRowOfPlane(&self.pixel_buffer, 1);
|
||||
if y_base.is_null() || uv_base.is_null() || y_stride < width || uv_stride < width {
|
||||
let _ = unsafe { CVPixelBufferUnlockBaseAddress(&self.pixel_buffer, lock_flags) };
|
||||
return false;
|
||||
}
|
||||
let chroma_width = width / 2;
|
||||
let chroma_height = height / 2;
|
||||
unsafe {
|
||||
let y_base = y_base as *mut u8;
|
||||
for row in 0..height {
|
||||
let src = &frame.y[row * width..row * width + width];
|
||||
core::ptr::copy_nonoverlapping(src.as_ptr(), y_base.add(row * y_stride), width);
|
||||
}
|
||||
let uv_base = uv_base as *mut u8;
|
||||
for row in 0..chroma_height {
|
||||
let dst_row = uv_base.add(row * uv_stride);
|
||||
for col in 0..chroma_width {
|
||||
let chroma_index = row * chroma_width + col;
|
||||
dst_row.add(col * 2).write(frame.u[chroma_index]);
|
||||
dst_row.add(col * 2 + 1).write(frame.v[chroma_index]);
|
||||
}
|
||||
}
|
||||
}
|
||||
let unlock_status =
|
||||
unsafe { CVPixelBufferUnlockBaseAddress(&self.pixel_buffer, lock_flags) };
|
||||
unlock_status == 0
|
||||
}
|
||||
|
||||
fn copy_observation_mask(
|
||||
mask_buffer: &CVPixelBuffer,
|
||||
mask: &mut [u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
) -> bool {
|
||||
if CVPixelBufferGetPixelFormatType(mask_buffer) != PIXEL_FORMAT_ONE_COMPONENT_8 {
|
||||
return false;
|
||||
}
|
||||
let lock_flags = CVPixelBufferLockFlags::ReadOnly;
|
||||
if unsafe { CVPixelBufferLockBaseAddress(mask_buffer, lock_flags) } != 0 {
|
||||
return false;
|
||||
}
|
||||
let src_width = CVPixelBufferGetWidth(mask_buffer);
|
||||
let src_height = CVPixelBufferGetHeight(mask_buffer);
|
||||
let src_stride = CVPixelBufferGetBytesPerRow(mask_buffer);
|
||||
let base = CVPixelBufferGetBaseAddress(mask_buffer);
|
||||
let valid = !base.is_null()
|
||||
&& src_width >= 1
|
||||
&& src_height >= 1
|
||||
&& src_stride >= src_width
|
||||
&& src_width * src_height <= MASK_PIXELS_MAX;
|
||||
if valid {
|
||||
let src = unsafe {
|
||||
core::slice::from_raw_parts(
|
||||
base as *const u8,
|
||||
src_stride * (src_height - 1) + src_width,
|
||||
)
|
||||
};
|
||||
super::resize_mask_bilinear(
|
||||
src, src_width, src_height, src_stride, mask, width, height,
|
||||
);
|
||||
}
|
||||
let _ = unsafe { CVPixelBufferUnlockBaseAddress(mask_buffer, lock_flags) };
|
||||
valid
|
||||
}
|
||||
}
|
||||
|
||||
impl PersonMaskSource for VisionPersonMaskSource {
|
||||
fn mask_into(&mut self, frame: &crate::yuv::I420, mask: &mut [u8]) -> bool {
|
||||
let started = std::time::Instant::now();
|
||||
let produced = self.mask_into_timed(frame, mask);
|
||||
let duration_ms = started.elapsed().as_millis() as u64;
|
||||
if self.quality_governor.record_frame_duration_ms(duration_ms) {
|
||||
self.downgrade_to_fast_quality();
|
||||
}
|
||||
produced
|
||||
}
|
||||
}
|
||||
|
||||
impl VisionPersonMaskSource {
|
||||
fn mask_into_timed(&mut self, frame: &crate::yuv::I420, mask: &mut [u8]) -> bool {
|
||||
let width = self.width as usize;
|
||||
let height = self.height as usize;
|
||||
assert!(mask.len() >= width * height);
|
||||
if frame.width != self.width || frame.height != self.height {
|
||||
return false;
|
||||
}
|
||||
if !self.fill_pixel_buffer(frame) {
|
||||
return false;
|
||||
}
|
||||
let performed = unsafe {
|
||||
self.handler
|
||||
.performRequests_onCVPixelBuffer_error(&self.requests, &self.pixel_buffer)
|
||||
};
|
||||
if performed.is_err() {
|
||||
return false;
|
||||
}
|
||||
let Some(results) = (unsafe { self.request.results() }) else {
|
||||
return false;
|
||||
};
|
||||
let Some(observation) = results.firstObject() else {
|
||||
return false;
|
||||
};
|
||||
let mask_buffer = unsafe { observation.pixelBuffer() };
|
||||
Self::copy_observation_mask(&mask_buffer, mask, width, height)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn resize_identity_returns_same_values() {
|
||||
let src = vec![0u8, 64, 128, 255];
|
||||
let mut dst = vec![0u8; 4];
|
||||
|
||||
resize_mask_bilinear(&src, 2, 2, 2, &mut dst, 2, 2);
|
||||
|
||||
assert_eq!(dst, src);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resize_upscales_with_interpolated_midpoints() {
|
||||
let src = vec![0u8, 255, 0, 255];
|
||||
let mut dst = vec![0u8; 9];
|
||||
|
||||
resize_mask_bilinear(&src, 2, 2, 2, &mut dst, 3, 3);
|
||||
|
||||
assert_eq!(dst[0], 0);
|
||||
assert_eq!(dst[2], 255);
|
||||
assert!(dst[1] > 100);
|
||||
assert!(dst[1] < 156);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resize_honours_source_stride_padding() {
|
||||
let src = vec![10u8, 20, 99, 99, 30, 40, 99, 99];
|
||||
let mut dst = vec![0u8; 4];
|
||||
|
||||
resize_mask_bilinear(&src, 2, 2, 4, &mut dst, 2, 2);
|
||||
|
||||
assert_eq!(dst, vec![10, 20, 30, 40]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resize_collapses_to_single_pixel_average_free() {
|
||||
let src = vec![200u8; 16];
|
||||
let mut dst = vec![0u8; 1];
|
||||
|
||||
resize_mask_bilinear(&src, 4, 4, 4, &mut dst, 1, 1);
|
||||
|
||||
assert_eq!(dst, vec![200]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_constants_span_full_alpha_range() {
|
||||
assert_eq!(PERSON_MASK_BACKGROUND, 0);
|
||||
assert_eq!(PERSON_MASK_PERSON, 255);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segmentation_governor_downgrades_after_consecutive_slow_frames() {
|
||||
let mut governor = SegmentationQualityGovernor::new();
|
||||
|
||||
for _ in 1..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX {
|
||||
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
|
||||
}
|
||||
|
||||
assert!(governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segmentation_governor_resets_count_after_a_frame_within_budget() {
|
||||
let mut governor = SegmentationQualityGovernor::new();
|
||||
|
||||
for _ in 1..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX {
|
||||
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
|
||||
}
|
||||
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS));
|
||||
for _ in 1..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX {
|
||||
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
|
||||
}
|
||||
|
||||
assert!(governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segmentation_governor_downgrades_only_once() {
|
||||
let mut governor = SegmentationQualityGovernor::new();
|
||||
for _ in 0..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX - 1 {
|
||||
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
|
||||
}
|
||||
assert!(governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 1));
|
||||
|
||||
for _ in 0..SEGMENTATION_SLOW_FRAMES_CONSECUTIVE_MAX * 2 {
|
||||
assert!(!governor.record_frame_duration_ms(SEGMENTATION_FRAME_BUDGET_MS + 100));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,974 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
pub const DEFAULT_AUDIO_BUFFER_TARGET_MS: u32 = 200;
|
||||
pub const DEFAULT_AUDIO_BUFFER_MAX_MS: u32 = 750;
|
||||
pub const DEFAULT_MIN_VIDEO_FPS: f64 = 15.0;
|
||||
|
||||
const PRESSURE_WINDOW_MS: u64 = 5_000;
|
||||
const RECOVERY_WINDOW_COUNT: u32 = 12;
|
||||
const AUDIO_REBUFFER_GAP_MS: u64 = 120;
|
||||
const AUDIO_STABLE_GAP_MS: u64 = 60;
|
||||
const AUDIO_BUFFER_STEP_MS: u32 = 100;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SendHealthSnapshot {
|
||||
pub outgoing_video_queue_depth: u64,
|
||||
pub outgoing_video_queue_capacity: u64,
|
||||
pub outgoing_video_max_queue_depth: u64,
|
||||
pub outgoing_video_frames_produced: u64,
|
||||
pub outgoing_video_frames_accepted: u64,
|
||||
pub outgoing_video_frames_dropped: u64,
|
||||
pub outgoing_video_frames_coalesced: u64,
|
||||
pub outgoing_video_frames_captured: u64,
|
||||
pub outgoing_video_capture_failures: u64,
|
||||
pub outgoing_video_effective_fps: f64,
|
||||
pub outgoing_video_target_fps: f64,
|
||||
pub outgoing_video_pacing_target_fps: f64,
|
||||
pub outgoing_video_max_queue_age_ms: u64,
|
||||
pub outgoing_video_max_push_latency_ms: u64,
|
||||
pub outgoing_video_pacing_mode: String,
|
||||
pub outgoing_video_bus_active: bool,
|
||||
pub outgoing_audio_buffer_target_ms: u32,
|
||||
pub outgoing_audio_buffer_max_ms: u32,
|
||||
pub outgoing_audio_underruns: u64,
|
||||
pub outgoing_audio_rebuffers: u64,
|
||||
pub outgoing_audio_max_frame_gap_ms: u64,
|
||||
pub adaptive_send_tier: String,
|
||||
pub adaptive_send_reason: String,
|
||||
}
|
||||
|
||||
impl SendHealthSnapshot {
|
||||
pub fn idle(audio: &AdaptiveAudioStats) -> Self {
|
||||
Self {
|
||||
outgoing_video_queue_depth: 0,
|
||||
outgoing_video_queue_capacity: 0,
|
||||
outgoing_video_max_queue_depth: 0,
|
||||
outgoing_video_frames_produced: 0,
|
||||
outgoing_video_frames_accepted: 0,
|
||||
outgoing_video_frames_dropped: 0,
|
||||
outgoing_video_frames_coalesced: 0,
|
||||
outgoing_video_frames_captured: 0,
|
||||
outgoing_video_capture_failures: 0,
|
||||
outgoing_video_effective_fps: 0.0,
|
||||
outgoing_video_target_fps: 0.0,
|
||||
outgoing_video_pacing_target_fps: 0.0,
|
||||
outgoing_video_max_queue_age_ms: 0,
|
||||
outgoing_video_max_push_latency_ms: 0,
|
||||
outgoing_video_pacing_mode: "idle".to_string(),
|
||||
outgoing_video_bus_active: false,
|
||||
outgoing_audio_buffer_target_ms: audio.target_buffer_ms(),
|
||||
outgoing_audio_buffer_max_ms: audio.max_buffer_ms(),
|
||||
outgoing_audio_underruns: audio.underruns.load(Ordering::Relaxed),
|
||||
outgoing_audio_rebuffers: audio.rebuffers.load(Ordering::Relaxed),
|
||||
outgoing_audio_max_frame_gap_ms: audio.max_frame_gap_ms.load(Ordering::Relaxed),
|
||||
adaptive_send_tier: "idle".to_string(),
|
||||
adaptive_send_reason: "notPublishing".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AdaptiveVideoController {
|
||||
requested_fps: f64,
|
||||
min_fps: f64,
|
||||
adaptive: bool,
|
||||
state: Mutex<AdaptiveVideoState>,
|
||||
}
|
||||
|
||||
struct AdaptiveVideoState {
|
||||
current_fps: f64,
|
||||
tier: String,
|
||||
reason: String,
|
||||
window_started_ms: u64,
|
||||
window_produced: u64,
|
||||
window_coalesced: u64,
|
||||
window_dropped: u64,
|
||||
window_max_queue_age_ms: u64,
|
||||
window_max_push_latency_ms: u64,
|
||||
window_egress_fps_sum: f64,
|
||||
window_egress_fps_samples: u32,
|
||||
stable_windows: u32,
|
||||
}
|
||||
|
||||
impl AdaptiveVideoController {
|
||||
pub fn new(requested_fps: f64, min_fps: f64, adaptive: bool, now_ms: u64) -> Self {
|
||||
let requested_fps = sanitize_fps(requested_fps, 30.0);
|
||||
let min_fps = sanitize_fps(min_fps, DEFAULT_MIN_VIDEO_FPS).min(requested_fps);
|
||||
Self {
|
||||
requested_fps,
|
||||
min_fps,
|
||||
adaptive,
|
||||
state: Mutex::new(AdaptiveVideoState {
|
||||
current_fps: requested_fps,
|
||||
tier: "full".to_string(),
|
||||
reason: "stable".to_string(),
|
||||
window_started_ms: now_ms,
|
||||
window_produced: 0,
|
||||
window_coalesced: 0,
|
||||
window_dropped: 0,
|
||||
window_max_queue_age_ms: 0,
|
||||
window_max_push_latency_ms: 0,
|
||||
window_egress_fps_sum: 0.0,
|
||||
window_egress_fps_samples: 0,
|
||||
stable_windows: 0,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn current_fps(&self) -> f64 {
|
||||
self.state.lock().current_fps
|
||||
}
|
||||
|
||||
pub fn tier_and_reason(&self) -> (String, String) {
|
||||
let state = self.state.lock();
|
||||
(state.tier.clone(), state.reason.clone())
|
||||
}
|
||||
|
||||
pub fn record_enqueue(&self, now_ms: u64, coalesced: bool) {
|
||||
let mut state = self.state.lock();
|
||||
self.rotate_window(&mut state, now_ms);
|
||||
state.window_produced += 1;
|
||||
if coalesced {
|
||||
state.window_coalesced += 1;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn record_drop(&self, now_ms: u64) {
|
||||
let mut state = self.state.lock();
|
||||
self.rotate_window(&mut state, now_ms);
|
||||
state.window_dropped += 1;
|
||||
}
|
||||
|
||||
pub fn record_capture(&self, now_ms: u64, queue_age_ms: u64, push_latency_ms: u64) {
|
||||
let mut state = self.state.lock();
|
||||
self.rotate_window(&mut state, now_ms);
|
||||
state.window_max_queue_age_ms = state.window_max_queue_age_ms.max(queue_age_ms);
|
||||
state.window_max_push_latency_ms = state.window_max_push_latency_ms.max(push_latency_ms);
|
||||
}
|
||||
|
||||
pub fn record_egress_fps(&self, now_ms: u64, fps: f64) {
|
||||
if !fps.is_finite() || fps < 0.0 {
|
||||
return;
|
||||
}
|
||||
let mut state = self.state.lock();
|
||||
self.rotate_window(&mut state, now_ms);
|
||||
state.window_egress_fps_sum += fps;
|
||||
state.window_egress_fps_samples += 1;
|
||||
}
|
||||
|
||||
fn rotate_window(&self, state: &mut AdaptiveVideoState, now_ms: u64) {
|
||||
if now_ms.saturating_sub(state.window_started_ms) < PRESSURE_WINDOW_MS {
|
||||
return;
|
||||
}
|
||||
self.apply_window(state);
|
||||
state.window_started_ms = now_ms;
|
||||
state.window_produced = 0;
|
||||
state.window_coalesced = 0;
|
||||
state.window_dropped = 0;
|
||||
state.window_max_queue_age_ms = 0;
|
||||
state.window_max_push_latency_ms = 0;
|
||||
state.window_egress_fps_sum = 0.0;
|
||||
state.window_egress_fps_samples = 0;
|
||||
}
|
||||
|
||||
fn apply_window(&self, state: &mut AdaptiveVideoState) {
|
||||
if !self.adaptive {
|
||||
state.current_fps = self.requested_fps;
|
||||
state.tier = "full".to_string();
|
||||
state.reason = "adaptiveDisabled".to_string();
|
||||
return;
|
||||
}
|
||||
|
||||
let frame_interval_ms = (1000.0 / state.current_fps.max(1.0)).ceil() as u64;
|
||||
let latency_pressure = state.window_max_queue_age_ms > frame_interval_ms * 2
|
||||
|| state.window_max_push_latency_ms > frame_interval_ms * 2;
|
||||
let drop_ratio = state.window_dropped as f64 / state.window_produced.max(1) as f64;
|
||||
let encoder_drop_pressure = state.window_produced >= 10 && drop_ratio > 0.05;
|
||||
let average_egress_fps = if state.window_egress_fps_samples == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(state.window_egress_fps_sum / state.window_egress_fps_samples as f64)
|
||||
};
|
||||
let egress_pressure = average_egress_fps.is_some_and(|fps| {
|
||||
state.window_egress_fps_samples >= 2
|
||||
&& state.window_produced >= 10
|
||||
&& fps < state.current_fps * 0.75
|
||||
});
|
||||
let pressure = latency_pressure || encoder_drop_pressure || egress_pressure;
|
||||
|
||||
if pressure {
|
||||
let next = if state.current_fps > 30.0 {
|
||||
30.0
|
||||
} else if state.current_fps > self.min_fps {
|
||||
self.min_fps
|
||||
} else {
|
||||
state.current_fps
|
||||
};
|
||||
if next < state.current_fps {
|
||||
state.current_fps = next;
|
||||
state.tier = tier_for_fps(self.requested_fps, state.current_fps);
|
||||
}
|
||||
state.reason = if latency_pressure {
|
||||
"sendLatencyPressure".to_string()
|
||||
} else if encoder_drop_pressure {
|
||||
"encoderDropPressure".to_string()
|
||||
} else {
|
||||
"encoderEgressPressure".to_string()
|
||||
};
|
||||
state.stable_windows = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
state.stable_windows += 1;
|
||||
if state.current_fps >= self.requested_fps {
|
||||
state.reason = "stable".to_string();
|
||||
}
|
||||
if state.stable_windows >= RECOVERY_WINDOW_COUNT && state.current_fps < self.requested_fps {
|
||||
state.current_fps = (state.current_fps * 2.0).min(self.requested_fps);
|
||||
state.tier = tier_for_fps(self.requested_fps, state.current_fps);
|
||||
state.stable_windows = 0;
|
||||
if state.current_fps >= self.requested_fps {
|
||||
state.reason = "stable".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AdaptiveVideoStats {
|
||||
produced: AtomicU64,
|
||||
accepted: AtomicU64,
|
||||
dropped: AtomicU64,
|
||||
coalesced: AtomicU64,
|
||||
captured: AtomicU64,
|
||||
capture_failures: AtomicU64,
|
||||
queue_depth: AtomicU64,
|
||||
max_queue_depth: AtomicU64,
|
||||
max_queue_age_ms: AtomicU64,
|
||||
max_push_latency_ms: AtomicU64,
|
||||
first_capture_ms: AtomicU64,
|
||||
last_capture_ms: AtomicU64,
|
||||
controller: AdaptiveVideoController,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct VideoTelemetryExtras {
|
||||
pub pacing_mode: String,
|
||||
pub pacing_target_fps: f64,
|
||||
pub queue_capacity: u64,
|
||||
pub bus_active: bool,
|
||||
}
|
||||
|
||||
impl Default for VideoTelemetryExtras {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pacing_mode: "unknown".to_string(),
|
||||
pacing_target_fps: 0.0,
|
||||
queue_capacity: 0,
|
||||
bus_active: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdaptiveVideoStats {
|
||||
pub fn new(requested_fps: f64, min_fps: f64, adaptive: bool, now_ms: u64) -> Self {
|
||||
Self {
|
||||
produced: AtomicU64::new(0),
|
||||
accepted: AtomicU64::new(0),
|
||||
dropped: AtomicU64::new(0),
|
||||
coalesced: AtomicU64::new(0),
|
||||
captured: AtomicU64::new(0),
|
||||
capture_failures: AtomicU64::new(0),
|
||||
queue_depth: AtomicU64::new(0),
|
||||
max_queue_depth: AtomicU64::new(0),
|
||||
max_queue_age_ms: AtomicU64::new(0),
|
||||
max_push_latency_ms: AtomicU64::new(0),
|
||||
first_capture_ms: AtomicU64::new(0),
|
||||
last_capture_ms: AtomicU64::new(0),
|
||||
controller: AdaptiveVideoController::new(requested_fps, min_fps, adaptive, now_ms),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn record_enqueue(&self, now_ms: u64, replaced_pending: bool) {
|
||||
self.record_enqueue_with_depth(now_ms, replaced_pending, 1);
|
||||
}
|
||||
|
||||
pub fn record_enqueue_with_depth(&self, now_ms: u64, replaced_pending: bool, queue_depth: u64) {
|
||||
self.produced.fetch_add(1, Ordering::Relaxed);
|
||||
self.accepted.fetch_add(1, Ordering::Relaxed);
|
||||
self.queue_depth.store(queue_depth, Ordering::Relaxed);
|
||||
update_max(&self.max_queue_depth, queue_depth);
|
||||
if replaced_pending {
|
||||
self.coalesced.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.controller.record_enqueue(now_ms, replaced_pending);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn record_drop(&self, now_ms: u64) {
|
||||
self.dropped.fetch_add(1, Ordering::Relaxed);
|
||||
self.controller.record_drop(now_ms);
|
||||
}
|
||||
|
||||
pub fn record_reject(&self) {
|
||||
self.dropped.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_capture(&self, now_ms: u64, queue_age_ms: u64, push_latency_ms: u64) {
|
||||
update_min_nonzero(&self.first_capture_ms, now_ms);
|
||||
update_max(&self.last_capture_ms, now_ms);
|
||||
self.captured.fetch_add(1, Ordering::Relaxed);
|
||||
self.queue_depth.store(0, Ordering::Relaxed);
|
||||
update_max(&self.max_queue_age_ms, queue_age_ms);
|
||||
update_max(&self.max_push_latency_ms, push_latency_ms);
|
||||
self.controller
|
||||
.record_capture(now_ms, queue_age_ms, push_latency_ms);
|
||||
}
|
||||
|
||||
pub fn record_egress_fps(&self, now_ms: u64, fps: f64) {
|
||||
self.controller.record_egress_fps(now_ms, fps);
|
||||
}
|
||||
|
||||
pub fn record_capture_failure(&self) {
|
||||
self.capture_failures.fetch_add(1, Ordering::Relaxed);
|
||||
self.queue_depth.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_queue_cleared(&self) {
|
||||
self.queue_depth.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn current_fps(&self) -> f64 {
|
||||
self.controller.current_fps()
|
||||
}
|
||||
|
||||
fn effective_fps(&self) -> f64 {
|
||||
let first = self.first_capture_ms.load(Ordering::Relaxed);
|
||||
let last = self.last_capture_ms.load(Ordering::Relaxed);
|
||||
let captured = self.captured.load(Ordering::Relaxed);
|
||||
if first == 0 || last <= first || captured <= 1 {
|
||||
return 0.0;
|
||||
}
|
||||
let elapsed_s = (last - first) as f64 / 1000.0;
|
||||
((captured - 1) as f64 / elapsed_s * 100.0).round() / 100.0
|
||||
}
|
||||
|
||||
pub fn snapshot(
|
||||
&self,
|
||||
audio: &AdaptiveAudioStats,
|
||||
extras: VideoTelemetryExtras,
|
||||
) -> SendHealthSnapshot {
|
||||
let (tier, reason) = self.controller.tier_and_reason();
|
||||
SendHealthSnapshot {
|
||||
outgoing_video_queue_depth: self.queue_depth.load(Ordering::Relaxed),
|
||||
outgoing_video_queue_capacity: extras.queue_capacity,
|
||||
outgoing_video_max_queue_depth: self.max_queue_depth.load(Ordering::Relaxed),
|
||||
outgoing_video_frames_produced: self.produced.load(Ordering::Relaxed),
|
||||
outgoing_video_frames_accepted: self.accepted.load(Ordering::Relaxed),
|
||||
outgoing_video_frames_dropped: self.dropped.load(Ordering::Relaxed),
|
||||
outgoing_video_frames_coalesced: self.coalesced.load(Ordering::Relaxed),
|
||||
outgoing_video_frames_captured: self.captured.load(Ordering::Relaxed),
|
||||
outgoing_video_capture_failures: self.capture_failures.load(Ordering::Relaxed),
|
||||
outgoing_video_effective_fps: self.effective_fps(),
|
||||
outgoing_video_target_fps: (self.current_fps() * 100.0).round() / 100.0,
|
||||
outgoing_video_pacing_target_fps: (extras.pacing_target_fps * 100.0).round() / 100.0,
|
||||
outgoing_video_max_queue_age_ms: self.max_queue_age_ms.load(Ordering::Relaxed),
|
||||
outgoing_video_max_push_latency_ms: self.max_push_latency_ms.load(Ordering::Relaxed),
|
||||
outgoing_video_pacing_mode: extras.pacing_mode,
|
||||
outgoing_video_bus_active: extras.bus_active,
|
||||
outgoing_audio_buffer_target_ms: audio.target_buffer_ms(),
|
||||
outgoing_audio_buffer_max_ms: audio.max_buffer_ms(),
|
||||
outgoing_audio_underruns: audio.underruns.load(Ordering::Relaxed),
|
||||
outgoing_audio_rebuffers: audio.rebuffers.load(Ordering::Relaxed),
|
||||
outgoing_audio_max_frame_gap_ms: audio.max_frame_gap_ms.load(Ordering::Relaxed),
|
||||
adaptive_send_tier: tier,
|
||||
adaptive_send_reason: reason,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AdaptiveAudioStats {
|
||||
max_buffer_ms: AtomicU64,
|
||||
target_buffer_ms: AtomicU64,
|
||||
underruns: AtomicU64,
|
||||
rebuffers: AtomicU64,
|
||||
max_frame_gap_ms: AtomicU64,
|
||||
last_push_ms: AtomicU64,
|
||||
stable_started_ms: AtomicU64,
|
||||
}
|
||||
|
||||
impl AdaptiveAudioStats {
|
||||
pub fn new(max_buffer_ms: u32, now_ms: u64) -> Self {
|
||||
let max_buffer_ms = clamp_audio_buffer_ms(max_buffer_ms);
|
||||
Self {
|
||||
max_buffer_ms: AtomicU64::new(max_buffer_ms as u64),
|
||||
target_buffer_ms: AtomicU64::new(
|
||||
DEFAULT_AUDIO_BUFFER_TARGET_MS.min(max_buffer_ms) as u64
|
||||
),
|
||||
underruns: AtomicU64::new(0),
|
||||
rebuffers: AtomicU64::new(0),
|
||||
max_frame_gap_ms: AtomicU64::new(0),
|
||||
last_push_ms: AtomicU64::new(0),
|
||||
stable_started_ms: AtomicU64::new(now_ms),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reset(&self, max_buffer_ms: u32, now_ms: u64) {
|
||||
let max_buffer_ms = clamp_audio_buffer_ms(max_buffer_ms);
|
||||
self.max_buffer_ms
|
||||
.store(max_buffer_ms as u64, Ordering::Relaxed);
|
||||
self.target_buffer_ms.store(
|
||||
DEFAULT_AUDIO_BUFFER_TARGET_MS.min(max_buffer_ms) as u64,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
self.underruns.store(0, Ordering::Relaxed);
|
||||
self.rebuffers.store(0, Ordering::Relaxed);
|
||||
self.max_frame_gap_ms.store(0, Ordering::Relaxed);
|
||||
self.last_push_ms.store(0, Ordering::Relaxed);
|
||||
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_push(&self, now_ms: u64) {
|
||||
let Some(previous) = advance_monotonic(&self.last_push_ms, now_ms) else {
|
||||
return;
|
||||
};
|
||||
if previous == 0 {
|
||||
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
let gap = now_ms - previous;
|
||||
update_max(&self.max_frame_gap_ms, gap);
|
||||
if gap > AUDIO_REBUFFER_GAP_MS {
|
||||
self.rebuffers.fetch_add(1, Ordering::Relaxed);
|
||||
if gap > self.target_buffer_ms() as u64 {
|
||||
self.underruns.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
let next = (self.target_buffer_ms() + AUDIO_BUFFER_STEP_MS).min(self.max_buffer_ms());
|
||||
self.target_buffer_ms.store(next as u64, Ordering::Relaxed);
|
||||
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
if gap <= AUDIO_STABLE_GAP_MS {
|
||||
let stable_started = self.stable_started_ms.load(Ordering::Relaxed);
|
||||
if now_ms.saturating_sub(stable_started) >= 30_000 {
|
||||
let current = self.target_buffer_ms();
|
||||
let next = current
|
||||
.saturating_sub(AUDIO_BUFFER_STEP_MS)
|
||||
.max(DEFAULT_AUDIO_BUFFER_TARGET_MS.min(self.max_buffer_ms()));
|
||||
self.target_buffer_ms.store(next as u64, Ordering::Relaxed);
|
||||
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
|
||||
}
|
||||
} else {
|
||||
self.stable_started_ms.store(now_ms, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn max_buffer_ms(&self) -> u32 {
|
||||
self.max_buffer_ms.load(Ordering::Relaxed) as u32
|
||||
}
|
||||
|
||||
pub fn target_buffer_ms(&self) -> u32 {
|
||||
self.target_buffer_ms.load(Ordering::Relaxed) as u32
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clamp_audio_buffer_ms(value: u32) -> u32 {
|
||||
let clamped = value.clamp(DEFAULT_AUDIO_BUFFER_TARGET_MS, DEFAULT_AUDIO_BUFFER_MAX_MS);
|
||||
clamped - (clamped % 10)
|
||||
}
|
||||
|
||||
fn sanitize_fps(value: f64, fallback: f64) -> f64 {
|
||||
if value.is_finite() && value > 0.0 {
|
||||
value
|
||||
} else {
|
||||
fallback
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_for_fps(requested_fps: f64, current_fps: f64) -> String {
|
||||
if current_fps >= requested_fps {
|
||||
"full".to_string()
|
||||
} else if current_fps >= 30.0 {
|
||||
"fps30".to_string()
|
||||
} else {
|
||||
"fps15".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn update_max(slot: &AtomicU64, value: u64) {
|
||||
let mut current = slot.load(Ordering::Relaxed);
|
||||
while value > current {
|
||||
match slot.compare_exchange(current, value, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => break,
|
||||
Err(next) => current = next,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn update_min_nonzero(slot: &AtomicU64, value: u64) {
|
||||
if value == 0 {
|
||||
return;
|
||||
}
|
||||
let mut current = slot.load(Ordering::Relaxed);
|
||||
loop {
|
||||
if current != 0 && current <= value {
|
||||
return;
|
||||
}
|
||||
match slot.compare_exchange(current, value, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return,
|
||||
Err(next) => current = next,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_monotonic(slot: &AtomicU64, value: u64) -> Option<u64> {
|
||||
let mut current = slot.load(Ordering::Relaxed);
|
||||
loop {
|
||||
if current != 0 && value < current {
|
||||
return None;
|
||||
}
|
||||
if value == current {
|
||||
return Some(current);
|
||||
}
|
||||
match slot.compare_exchange(current, value, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return Some(current),
|
||||
Err(next) => current = next,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[test]
|
||||
fn video_controller_ignores_pure_coalescing_jitter() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
|
||||
for _ in 0..200 {
|
||||
controller.record_enqueue(1_000, true);
|
||||
}
|
||||
controller.record_enqueue(5_001, false);
|
||||
|
||||
assert_eq!(controller.current_fps(), 60.0);
|
||||
assert_eq!(controller.tier_and_reason().0, "full");
|
||||
assert_eq!(controller.tier_and_reason().1, "stable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_ignores_sustained_coalescing_jitter_across_windows() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
|
||||
for _ in 0..200 {
|
||||
controller.record_enqueue(1_000, true);
|
||||
}
|
||||
controller.record_enqueue(5_001, false);
|
||||
assert_eq!(controller.current_fps(), 60.0);
|
||||
|
||||
for _ in 0..200 {
|
||||
controller.record_enqueue(6_000, true);
|
||||
}
|
||||
controller.record_enqueue(10_002, false);
|
||||
assert_eq!(controller.current_fps(), 60.0);
|
||||
assert_eq!(controller.tier_and_reason().1, "stable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_degrades_on_encoder_drop_pressure() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
|
||||
for _ in 0..100 {
|
||||
controller.record_enqueue(1_000, false);
|
||||
}
|
||||
for _ in 0..20 {
|
||||
controller.record_drop(1_000);
|
||||
}
|
||||
controller.record_enqueue(5_001, false);
|
||||
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
assert_eq!(controller.tier_and_reason().1, "encoderDropPressure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_continues_degrading_under_sustained_latency_pressure() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
|
||||
controller.record_capture(1_000, 80, 1);
|
||||
controller.record_capture(5_001, 1, 1);
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
|
||||
controller.record_capture(6_000, 80, 1);
|
||||
controller.record_capture(10_002, 1, 1);
|
||||
|
||||
assert_eq!(controller.current_fps(), 15.0);
|
||||
assert_eq!(controller.tier_and_reason().0, "fps15");
|
||||
assert_eq!(controller.tier_and_reason().1, "sendLatencyPressure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_degrades_on_latency_pressure_without_coalescing() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
|
||||
controller.record_capture(1_000, 80, 5);
|
||||
controller.record_capture(5_001, 1, 1);
|
||||
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
assert_eq!(controller.tier_and_reason().1, "sendLatencyPressure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_degrades_on_encoder_egress_pressure() {
|
||||
let controller = AdaptiveVideoController::new(30.0, 15.0, true, 0);
|
||||
for _ in 0..60 {
|
||||
controller.record_enqueue(1_000, false);
|
||||
}
|
||||
controller.record_egress_fps(1_000, 7.0);
|
||||
controller.record_egress_fps(2_000, 8.0);
|
||||
controller.record_egress_fps(4_000, 7.0);
|
||||
|
||||
controller.record_egress_fps(5_001, 7.0);
|
||||
|
||||
assert_eq!(controller.current_fps(), 15.0);
|
||||
assert_eq!(controller.tier_and_reason().0, "fps15");
|
||||
assert_eq!(controller.tier_and_reason().1, "encoderEgressPressure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_preserves_pressure_reason_while_degraded_but_stable() {
|
||||
let controller = AdaptiveVideoController::new(30.0, 15.0, true, 0);
|
||||
for _ in 0..60 {
|
||||
controller.record_enqueue(1_000, false);
|
||||
}
|
||||
controller.record_egress_fps(1_000, 7.0);
|
||||
controller.record_egress_fps(2_000, 8.0);
|
||||
controller.record_egress_fps(4_000, 7.0);
|
||||
controller.record_egress_fps(5_001, 7.0);
|
||||
|
||||
assert_eq!(controller.current_fps(), 15.0);
|
||||
assert_eq!(controller.tier_and_reason().1, "encoderEgressPressure");
|
||||
|
||||
controller.record_capture(10_002, 1, 1);
|
||||
|
||||
assert_eq!(controller.current_fps(), 15.0);
|
||||
assert_eq!(controller.tier_and_reason().0, "fps15");
|
||||
assert_eq!(controller.tier_and_reason().1, "encoderEgressPressure");
|
||||
|
||||
for index in 2..=12 {
|
||||
controller.record_capture(10_002 + index * 5_001, 1, 1);
|
||||
}
|
||||
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
assert_eq!(
|
||||
controller.tier_and_reason(),
|
||||
("full".to_string(), "stable".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_ignores_single_encoder_egress_sample() {
|
||||
let controller = AdaptiveVideoController::new(30.0, 15.0, true, 0);
|
||||
for _ in 0..60 {
|
||||
controller.record_enqueue(1_000, false);
|
||||
}
|
||||
controller.record_egress_fps(1_000, 7.0);
|
||||
|
||||
controller.record_egress_fps(5_001, 7.0);
|
||||
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
assert_eq!(controller.tier_and_reason().1, "stable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_keeps_requested_fps_when_adaptive_send_is_disabled() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, false, 0);
|
||||
for _ in 0..100 {
|
||||
controller.record_enqueue(1_000, false);
|
||||
}
|
||||
controller.record_capture(5_001, 200, 200);
|
||||
|
||||
assert_eq!(controller.current_fps(), 60.0);
|
||||
assert_eq!(
|
||||
controller.tier_and_reason(),
|
||||
("full".to_string(), "adaptiveDisabled".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_recovers_after_stable_windows() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
|
||||
controller.record_capture(1_000, 80, 1);
|
||||
controller.record_capture(5_001, 1, 1);
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
|
||||
for index in 1..=12 {
|
||||
controller.record_capture(5_001 + index * 5_001, 1, 1);
|
||||
}
|
||||
assert_eq!(controller.current_fps(), 60.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_recovers_from_minimum_in_two_stable_steps() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
|
||||
controller.record_capture(1_000, 80, 1);
|
||||
controller.record_capture(5_001, 1, 1);
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
|
||||
controller.record_capture(6_000, 70, 1);
|
||||
controller.record_capture(10_002, 1, 1);
|
||||
assert_eq!(controller.current_fps(), 15.0);
|
||||
|
||||
for index in 1..=12 {
|
||||
controller.record_capture(10_002 + index * 5_001, 1, 1);
|
||||
}
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
|
||||
for index in 13..=24 {
|
||||
controller.record_capture(10_002 + index * 5_001, 1, 1);
|
||||
}
|
||||
assert_eq!(controller.current_fps(), 60.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_requires_a_full_stable_recovery_window() {
|
||||
let controller = AdaptiveVideoController::new(60.0, 15.0, true, 0);
|
||||
controller.record_capture(1_000, 80, 1);
|
||||
controller.record_capture(5_001, 1, 1);
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
|
||||
for index in 1..12 {
|
||||
controller.record_capture(5_001 + index * 5_001, 1, 1);
|
||||
}
|
||||
assert_eq!(controller.current_fps(), 30.0);
|
||||
controller.record_capture(5_001 + 12 * 5_001, 1, 1);
|
||||
assert_eq!(controller.current_fps(), 60.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_stats_snapshot_counts_coalescing_drops_failures_and_effective_fps() {
|
||||
let audio = AdaptiveAudioStats::new(750, 0);
|
||||
let stats = AdaptiveVideoStats::new(60.0, 15.0, true, 0);
|
||||
|
||||
stats.record_enqueue(1_000, false);
|
||||
stats.record_enqueue(1_010, true);
|
||||
stats.record_drop(1_011);
|
||||
stats.record_capture(1_020, 20, 24);
|
||||
stats.record_enqueue(2_000, false);
|
||||
stats.record_capture(2_020, 20, 26);
|
||||
stats.record_capture_failure();
|
||||
|
||||
let snapshot = stats.snapshot(&audio, VideoTelemetryExtras::default());
|
||||
assert_eq!(snapshot.outgoing_video_frames_produced, 3);
|
||||
assert_eq!(snapshot.outgoing_video_frames_accepted, 3);
|
||||
assert_eq!(snapshot.outgoing_video_frames_dropped, 1);
|
||||
assert_eq!(snapshot.outgoing_video_frames_coalesced, 1);
|
||||
assert_eq!(snapshot.outgoing_video_frames_captured, 2);
|
||||
assert_eq!(snapshot.outgoing_video_capture_failures, 1);
|
||||
assert_eq!(snapshot.outgoing_video_effective_fps, 1.0);
|
||||
assert_eq!(snapshot.outgoing_video_max_queue_age_ms, 20);
|
||||
assert_eq!(snapshot.outgoing_video_max_push_latency_ms, 26);
|
||||
assert_eq!(snapshot.outgoing_video_queue_depth, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_stats_effective_fps_uses_capture_time_bounds_for_out_of_order_records() {
|
||||
let audio = AdaptiveAudioStats::new(750, 0);
|
||||
let stats = AdaptiveVideoStats::new(60.0, 15.0, true, 0);
|
||||
|
||||
stats.record_capture(2_000, 5, 6);
|
||||
stats.record_capture(1_000, 7, 8);
|
||||
stats.record_capture(3_000, 9, 10);
|
||||
stats.record_capture(2_500, 11, 12);
|
||||
|
||||
let snapshot = stats.snapshot(&audio, VideoTelemetryExtras::default());
|
||||
assert_eq!(snapshot.outgoing_video_frames_captured, 4);
|
||||
assert_eq!(snapshot.outgoing_video_effective_fps, 1.5);
|
||||
assert_eq!(snapshot.outgoing_video_max_queue_age_ms, 11);
|
||||
assert_eq!(snapshot.outgoing_video_max_push_latency_ms, 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_stats_handles_concurrent_recording_without_lost_counts() {
|
||||
let audio = AdaptiveAudioStats::new(750, 0);
|
||||
let stats = Arc::new(AdaptiveVideoStats::new(60.0, 15.0, true, 0));
|
||||
let mut workers = Vec::new();
|
||||
for worker in 0..8 {
|
||||
let stats = stats.clone();
|
||||
workers.push(std::thread::spawn(move || {
|
||||
for index in 0..250 {
|
||||
let now_ms = 1_000 + worker * 1_000 + index;
|
||||
stats.record_enqueue(now_ms, index % 3 == 0);
|
||||
if index % 5 == 0 {
|
||||
stats.record_drop(now_ms);
|
||||
}
|
||||
if index % 7 == 0 {
|
||||
stats.record_capture(now_ms + 1, 1, 2);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
for worker in workers {
|
||||
worker.join().expect("worker should not panic");
|
||||
}
|
||||
|
||||
let snapshot = stats.snapshot(&audio, VideoTelemetryExtras::default());
|
||||
assert_eq!(snapshot.outgoing_video_frames_produced, 2_000);
|
||||
assert_eq!(snapshot.outgoing_video_frames_accepted, 2_000);
|
||||
assert_eq!(snapshot.outgoing_video_frames_coalesced, 672);
|
||||
assert_eq!(snapshot.outgoing_video_frames_dropped, 400);
|
||||
assert_eq!(snapshot.outgoing_video_frames_captured, 288);
|
||||
assert_eq!(snapshot.outgoing_video_max_queue_age_ms, 1);
|
||||
assert_eq!(snapshot.outgoing_video_max_push_latency_ms, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idle_snapshot_reflects_audio_pressure_without_video_state() {
|
||||
let audio = AdaptiveAudioStats::new(750, 0);
|
||||
audio.record_push(1_000);
|
||||
audio.record_push(1_300);
|
||||
|
||||
let snapshot = SendHealthSnapshot::idle(&audio);
|
||||
assert_eq!(snapshot.outgoing_video_frames_produced, 0);
|
||||
assert_eq!(snapshot.outgoing_audio_buffer_target_ms, 300);
|
||||
assert_eq!(snapshot.outgoing_audio_buffer_max_ms, 750);
|
||||
assert_eq!(snapshot.outgoing_audio_rebuffers, 1);
|
||||
assert_eq!(snapshot.outgoing_audio_underruns, 1);
|
||||
assert_eq!(snapshot.outgoing_audio_max_frame_gap_ms, 300);
|
||||
assert_eq!(snapshot.adaptive_send_tier, "idle");
|
||||
assert_eq!(snapshot.adaptive_send_reason, "notPublishing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_controller_stress_keeps_target_inside_configured_bounds() {
|
||||
let controller = AdaptiveVideoController::new(144.0, 24.0, true, 0);
|
||||
for window in 0..240 {
|
||||
let base = window * 5_001;
|
||||
match window % 4 {
|
||||
0 => controller.record_capture(base + 1_000, 200, 1),
|
||||
1 => {
|
||||
for _ in 0..30 {
|
||||
controller.record_enqueue(base + 1_000, true);
|
||||
}
|
||||
controller.record_capture(base + 1_500, 1, 1);
|
||||
}
|
||||
2 => {
|
||||
for _ in 0..60 {
|
||||
controller.record_enqueue(base + 1_000, false);
|
||||
}
|
||||
controller.record_egress_fps(base + 1_500, 30.0);
|
||||
controller.record_egress_fps(base + 2_500, 30.0);
|
||||
}
|
||||
_ => controller.record_capture(base + 1_000, 1, 1),
|
||||
}
|
||||
controller.record_capture(base + 5_001, 1, 1);
|
||||
let fps = controller.current_fps();
|
||||
assert!(
|
||||
(24.0..=144.0).contains(&fps),
|
||||
"fps target escaped configured bounds: {fps}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_buffer_expands_on_gaps_and_shrinks_after_stability() {
|
||||
let audio = AdaptiveAudioStats::new(750, 0);
|
||||
audio.record_push(10);
|
||||
audio.record_push(200);
|
||||
assert_eq!(audio.target_buffer_ms(), 300);
|
||||
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 1);
|
||||
|
||||
for now_ms in (220..=30_240).step_by(20) {
|
||||
audio.record_push(now_ms);
|
||||
}
|
||||
assert_eq!(audio.target_buffer_ms(), 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_buffer_ignores_clock_regression_without_false_rebuffer() {
|
||||
let audio = AdaptiveAudioStats::new(750, 0);
|
||||
|
||||
audio.record_push(1_000);
|
||||
audio.record_push(1_020);
|
||||
audio.record_push(900);
|
||||
audio.record_push(1_040);
|
||||
|
||||
assert_eq!(audio.target_buffer_ms(), 200);
|
||||
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(audio.underruns.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(audio.max_frame_gap_ms.load(Ordering::Relaxed), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_buffer_ignores_concurrent_stale_pushes_without_moving_last_push_backwards() {
|
||||
let audio = Arc::new(AdaptiveAudioStats::new(750, 0));
|
||||
audio.record_push(10_000);
|
||||
|
||||
let mut workers = Vec::new();
|
||||
for worker in 0..8 {
|
||||
let audio = audio.clone();
|
||||
workers.push(std::thread::spawn(move || {
|
||||
for index in 0..100 {
|
||||
audio.record_push(1_000 + worker * 100 + index);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for worker in workers {
|
||||
worker.join().expect("worker should not panic");
|
||||
}
|
||||
|
||||
audio.record_push(10_020);
|
||||
|
||||
assert_eq!(audio.target_buffer_ms(), 200);
|
||||
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(audio.underruns.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(audio.max_frame_gap_ms.load(Ordering::Relaxed), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_buffer_growth_caps_at_configured_max_and_reset_clears_pressure() {
|
||||
let audio = AdaptiveAudioStats::new(350, 0);
|
||||
audio.record_push(10);
|
||||
for index in 1..=10 {
|
||||
audio.record_push(10 + index * 500);
|
||||
}
|
||||
assert_eq!(audio.target_buffer_ms(), 350);
|
||||
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 10);
|
||||
assert_eq!(audio.underruns.load(Ordering::Relaxed), 10);
|
||||
|
||||
audio.reset(250, 10_000);
|
||||
assert_eq!(audio.target_buffer_ms(), 200);
|
||||
assert_eq!(audio.max_buffer_ms(), 250);
|
||||
assert_eq!(audio.rebuffers.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(audio.underruns.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(audio.max_frame_gap_ms.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_buffer_stress_stays_within_realtime_bounds_under_jitter() {
|
||||
let audio = AdaptiveAudioStats::new(620, 0);
|
||||
let mut now_ms = 10;
|
||||
audio.record_push(now_ms);
|
||||
|
||||
for index in 0..5_000 {
|
||||
now_ms += match index % 11 {
|
||||
0 => 180,
|
||||
1 | 2 => 80,
|
||||
_ => 20,
|
||||
};
|
||||
audio.record_push(now_ms);
|
||||
assert!(
|
||||
(DEFAULT_AUDIO_BUFFER_TARGET_MS..=620).contains(&audio.target_buffer_ms()),
|
||||
"audio target escaped configured bounds"
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(audio.max_buffer_ms(), 620);
|
||||
assert!(audio.rebuffers.load(Ordering::Relaxed) > 0);
|
||||
assert!(audio.max_frame_gap_ms.load(Ordering::Relaxed) >= 180);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_buffer_max_is_clamped_to_real_time_bounds() {
|
||||
assert_eq!(clamp_audio_buffer_ms(50), 200);
|
||||
assert_eq!(clamp_audio_buffer_ms(777), 750);
|
||||
assert_eq!(clamp_audio_buffer_ms(333), 330);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
pub const SPEAKING_ATTACK_MS: u64 = 30;
|
||||
pub const SPEAKING_RELEASE_MS_LOCAL: u64 = 180;
|
||||
pub const SPEAKING_RELEASE_MS_REMOTE: u64 = 220;
|
||||
pub const SPEAKING_HEARTBEAT_INTERVAL_MS: u64 = 1_000;
|
||||
pub const SPEAKING_FRAME_TIMEOUT_MS: u64 = 250;
|
||||
pub const SPEAKING_THRESHOLD_RMS_LOCAL_DEFAULT: f64 = 0.008;
|
||||
pub const SPEAKING_THRESHOLD_RMS_REMOTE_DEFAULT: f64 = 0.006;
|
||||
pub const SPEAKING_THRESHOLD_RMS_MIN: f64 = 0.000_1;
|
||||
pub const SPEAKING_THRESHOLD_RMS_MAX: f64 = 0.5;
|
||||
pub const SPEAKING_FRAME_SAMPLES_MAX: usize = 1 << 20;
|
||||
|
||||
const _: () = assert!(SPEAKING_ATTACK_MS < SPEAKING_RELEASE_MS_LOCAL);
|
||||
const _: () = assert!(SPEAKING_ATTACK_MS < SPEAKING_RELEASE_MS_REMOTE);
|
||||
const _: () = assert!(SPEAKING_RELEASE_MS_REMOTE < SPEAKING_HEARTBEAT_INTERVAL_MS);
|
||||
const _: () = assert!(SPEAKING_FRAME_TIMEOUT_MS < SPEAKING_HEARTBEAT_INTERVAL_MS);
|
||||
|
||||
pub fn clamp_speaking_threshold_rms(threshold_rms: f64) -> f64 {
|
||||
if !threshold_rms.is_finite() {
|
||||
return SPEAKING_THRESHOLD_RMS_MIN;
|
||||
}
|
||||
threshold_rms.clamp(SPEAKING_THRESHOLD_RMS_MIN, SPEAKING_THRESHOLD_RMS_MAX)
|
||||
}
|
||||
|
||||
pub struct SpeakingThresholds {
|
||||
local_rms_bits: AtomicU64,
|
||||
remote_rms_bits: AtomicU64,
|
||||
}
|
||||
|
||||
impl SpeakingThresholds {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
local_rms_bits: AtomicU64::new(SPEAKING_THRESHOLD_RMS_LOCAL_DEFAULT.to_bits()),
|
||||
remote_rms_bits: AtomicU64::new(SPEAKING_THRESHOLD_RMS_REMOTE_DEFAULT.to_bits()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set(&self, local_rms: f64, remote_rms: f64) {
|
||||
let local = clamp_speaking_threshold_rms(local_rms);
|
||||
let remote = clamp_speaking_threshold_rms(remote_rms);
|
||||
assert!(local >= SPEAKING_THRESHOLD_RMS_MIN);
|
||||
assert!(remote >= SPEAKING_THRESHOLD_RMS_MIN);
|
||||
self.local_rms_bits
|
||||
.store(local.to_bits(), Ordering::Release);
|
||||
self.remote_rms_bits
|
||||
.store(remote.to_bits(), Ordering::Release);
|
||||
}
|
||||
|
||||
pub fn local_rms(&self) -> f64 {
|
||||
let value = f64::from_bits(self.local_rms_bits.load(Ordering::Acquire));
|
||||
assert!(value.is_finite());
|
||||
value
|
||||
}
|
||||
|
||||
pub fn remote_rms(&self) -> f64 {
|
||||
let value = f64::from_bits(self.remote_rms_bits.load(Ordering::Acquire));
|
||||
assert!(value.is_finite());
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SpeakingThresholds {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn frame_rms_i16(samples: &[i16]) -> f64 {
|
||||
assert!(!samples.is_empty());
|
||||
assert!(samples.len() <= SPEAKING_FRAME_SAMPLES_MAX);
|
||||
let mut sum_squares: f64 = 0.0;
|
||||
for sample in samples {
|
||||
let normalized = f64::from(*sample) / 32_768.0;
|
||||
sum_squares += normalized * normalized;
|
||||
}
|
||||
let rms = (sum_squares / samples.len() as f64).sqrt();
|
||||
assert!(rms.is_finite());
|
||||
assert!(rms >= 0.0);
|
||||
rms.min(1.0)
|
||||
}
|
||||
|
||||
pub struct SpeakingGate {
|
||||
attack_ms: u64,
|
||||
release_ms: u64,
|
||||
speaking: bool,
|
||||
above_since_ms: Option<u64>,
|
||||
below_since_ms: Option<u64>,
|
||||
last_now_ms: u64,
|
||||
}
|
||||
|
||||
impl SpeakingGate {
|
||||
pub fn new(attack_ms: u64, release_ms: u64) -> Self {
|
||||
assert!(attack_ms < release_ms);
|
||||
assert!(release_ms <= SPEAKING_HEARTBEAT_INTERVAL_MS);
|
||||
Self {
|
||||
attack_ms,
|
||||
release_ms,
|
||||
speaking: false,
|
||||
above_since_ms: None,
|
||||
below_since_ms: None,
|
||||
last_now_ms: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn speaking(&self) -> bool {
|
||||
self.speaking
|
||||
}
|
||||
|
||||
pub fn update(&mut self, rms: f64, threshold_rms: f64, now_ms: u64) -> Option<bool> {
|
||||
assert!(rms.is_finite());
|
||||
assert!(rms >= 0.0);
|
||||
assert!(threshold_rms >= SPEAKING_THRESHOLD_RMS_MIN);
|
||||
assert!(threshold_rms <= SPEAKING_THRESHOLD_RMS_MAX);
|
||||
assert!(now_ms >= self.last_now_ms);
|
||||
self.last_now_ms = now_ms;
|
||||
if rms >= threshold_rms {
|
||||
self.below_since_ms = None;
|
||||
let above_since_ms = *self.above_since_ms.get_or_insert(now_ms);
|
||||
if self.speaking {
|
||||
return None;
|
||||
}
|
||||
if now_ms - above_since_ms < self.attack_ms {
|
||||
return None;
|
||||
}
|
||||
self.speaking = true;
|
||||
return Some(true);
|
||||
}
|
||||
self.above_since_ms = None;
|
||||
let below_since_ms = *self.below_since_ms.get_or_insert(now_ms);
|
||||
if !self.speaking {
|
||||
return None;
|
||||
}
|
||||
if now_ms - below_since_ms < self.release_ms {
|
||||
return None;
|
||||
}
|
||||
self.speaking = false;
|
||||
Some(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const THRESHOLD: f64 = 0.01;
|
||||
|
||||
fn gate() -> SpeakingGate {
|
||||
SpeakingGate::new(SPEAKING_ATTACK_MS, SPEAKING_RELEASE_MS_LOCAL)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stays_quiet_below_threshold() {
|
||||
let mut gate = gate();
|
||||
for tick in 0..100u64 {
|
||||
assert_eq!(gate.update(0.001, THRESHOLD, tick * 10), None);
|
||||
}
|
||||
assert!(!gate.speaking());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attack_requires_sustained_signal() {
|
||||
let mut gate = gate();
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 0), None);
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 10), None);
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 20), None);
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 30), Some(true));
|
||||
assert!(gate.speaking());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_frame_blip_does_not_trigger() {
|
||||
let mut gate = gate();
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 0), None);
|
||||
assert_eq!(gate.update(0.001, THRESHOLD, 10), None);
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 20), None);
|
||||
assert_eq!(gate.update(0.001, THRESHOLD, 30), None);
|
||||
assert!(!gate.speaking());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_bridges_inter_word_gaps() {
|
||||
let mut gate = gate();
|
||||
for tick in 0..=3u64 {
|
||||
gate.update(0.5, THRESHOLD, tick * 10);
|
||||
}
|
||||
assert!(gate.speaking());
|
||||
for tick in 4..=20u64 {
|
||||
assert_eq!(gate.update(0.001, THRESHOLD, tick * 10), None);
|
||||
}
|
||||
assert!(gate.speaking());
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 210), None);
|
||||
assert!(gate.speaking());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_fires_after_sustained_silence() {
|
||||
let mut gate = gate();
|
||||
for tick in 0..=3u64 {
|
||||
gate.update(0.5, THRESHOLD, tick * 10);
|
||||
}
|
||||
assert!(gate.speaking());
|
||||
assert_eq!(gate.update(0.001, THRESHOLD, 40), None);
|
||||
assert_eq!(gate.update(0.001, THRESHOLD, 219), None);
|
||||
assert_eq!(gate.update(0.001, THRESHOLD, 220), Some(false));
|
||||
assert!(!gate.speaking());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retrigger_after_release_needs_full_attack() {
|
||||
let mut gate = gate();
|
||||
for tick in 0..=3u64 {
|
||||
gate.update(0.5, THRESHOLD, tick * 10);
|
||||
}
|
||||
gate.update(0.001, THRESHOLD, 40);
|
||||
assert_eq!(gate.update(0.001, THRESHOLD, 220), Some(false));
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 230), None);
|
||||
assert_eq!(gate.update(0.5, THRESHOLD, 260), Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_rms_of_silence_is_zero() {
|
||||
let samples = [0i16; 480];
|
||||
assert_eq!(frame_rms_i16(&samples), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_rms_of_full_scale_square_wave_is_one() {
|
||||
let mut samples = [i16::MIN; 480];
|
||||
for (index, sample) in samples.iter_mut().enumerate() {
|
||||
if index % 2 == 0 {
|
||||
*sample = i16::MAX;
|
||||
}
|
||||
}
|
||||
let rms = frame_rms_i16(&samples);
|
||||
assert!(rms > 0.999);
|
||||
assert!(rms <= 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_rms_scales_with_amplitude() {
|
||||
let loud = [8_192i16; 480];
|
||||
let quiet = [1_024i16; 480];
|
||||
assert!(frame_rms_i16(&loud) > frame_rms_i16(&quiet));
|
||||
assert!((frame_rms_i16(&loud) - 0.25).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thresholds_default_and_clamp() {
|
||||
let thresholds = SpeakingThresholds::new();
|
||||
assert_eq!(thresholds.local_rms(), SPEAKING_THRESHOLD_RMS_LOCAL_DEFAULT);
|
||||
assert_eq!(
|
||||
thresholds.remote_rms(),
|
||||
SPEAKING_THRESHOLD_RMS_REMOTE_DEFAULT
|
||||
);
|
||||
thresholds.set(-1.0, f64::NAN);
|
||||
assert_eq!(thresholds.local_rms(), SPEAKING_THRESHOLD_RMS_MIN);
|
||||
assert_eq!(thresholds.remote_rms(), SPEAKING_THRESHOLD_RMS_MIN);
|
||||
thresholds.set(9.0, 0.02);
|
||||
assert_eq!(thresholds.local_rms(), SPEAKING_THRESHOLD_RMS_MAX);
|
||||
assert_eq!(thresholds.remote_rms(), 0.02);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_update_is_monotonic_in_time() {
|
||||
let mut gate = gate();
|
||||
gate.update(0.5, THRESHOLD, 100);
|
||||
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
gate.update(0.5, THRESHOLD, 50);
|
||||
}));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use crate::send_control::SendHealthSnapshot;
|
||||
use fluxer_desktop_native::voice::stats as core_stats;
|
||||
|
||||
pub type ByteRateSample = core_stats::ByteRateSample;
|
||||
pub type OutboundEntry = core_stats::OutboundStatsEntry;
|
||||
pub type InboundEntry = core_stats::InboundStatsEntry;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub struct ConnectionStats {
|
||||
pub rtt_ms: Option<f64>,
|
||||
pub outbound: Vec<OutboundEntry>,
|
||||
pub inbound: Vec<InboundEntry>,
|
||||
pub send: Option<SendHealthSnapshot>,
|
||||
}
|
||||
|
||||
pub fn bitrate_kbps(prev: Option<ByteRateSample>, cur: ByteRateSample) -> f64 {
|
||||
core_stats::bitrate_kbps(prev, cur)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn sanitize_kbps(kbps: f64) -> f64 {
|
||||
core_stats::sanitize_kbps(kbps)
|
||||
}
|
||||
|
||||
pub fn jitter_seconds_to_ms(jitter_s: f64) -> Option<f64> {
|
||||
core_stats::jitter_seconds_to_ms(jitter_s)
|
||||
}
|
||||
|
||||
pub fn rtt_seconds_to_ms(rtt_s: f64) -> Option<f64> {
|
||||
core_stats::rtt_seconds_to_ms(rtt_s)
|
||||
}
|
||||
|
||||
pub fn sanitize_audio_level(level: f64) -> Option<f64> {
|
||||
core_stats::sanitize_audio_level(level)
|
||||
}
|
||||
|
||||
pub fn stats_to_json(stats: &ConnectionStats) -> String {
|
||||
core_stats::stats_to_json(&core_stats::ConnectionStats {
|
||||
rtt_ms: stats.rtt_ms,
|
||||
outbound: stats.outbound.clone(),
|
||||
inbound: stats.inbound.clone(),
|
||||
send: stats.send.as_ref().map(send_health_to_core),
|
||||
})
|
||||
}
|
||||
|
||||
fn send_health_to_core(send: &SendHealthSnapshot) -> core_stats::SendHealthStats {
|
||||
core_stats::SendHealthStats {
|
||||
outgoing_video_queue_depth: send.outgoing_video_queue_depth,
|
||||
outgoing_video_queue_capacity: send.outgoing_video_queue_capacity,
|
||||
outgoing_video_max_queue_depth: send.outgoing_video_max_queue_depth,
|
||||
outgoing_video_frames_produced: send.outgoing_video_frames_produced,
|
||||
outgoing_video_frames_accepted: send.outgoing_video_frames_accepted,
|
||||
outgoing_video_frames_dropped: send.outgoing_video_frames_dropped,
|
||||
outgoing_video_frames_coalesced: send.outgoing_video_frames_coalesced,
|
||||
outgoing_video_frames_captured: send.outgoing_video_frames_captured,
|
||||
outgoing_video_capture_failures: send.outgoing_video_capture_failures,
|
||||
outgoing_video_effective_fps: send.outgoing_video_effective_fps,
|
||||
outgoing_video_target_fps: send.outgoing_video_target_fps,
|
||||
outgoing_video_pacing_target_fps: send.outgoing_video_pacing_target_fps,
|
||||
outgoing_video_max_queue_age_ms: send.outgoing_video_max_queue_age_ms,
|
||||
outgoing_video_max_push_latency_ms: send.outgoing_video_max_push_latency_ms,
|
||||
outgoing_video_pacing_mode: send.outgoing_video_pacing_mode.clone(),
|
||||
outgoing_video_bus_active: send.outgoing_video_bus_active,
|
||||
outgoing_audio_buffer_target_ms: send.outgoing_audio_buffer_target_ms,
|
||||
outgoing_audio_buffer_max_ms: send.outgoing_audio_buffer_max_ms,
|
||||
outgoing_audio_underruns: send.outgoing_audio_underruns,
|
||||
outgoing_audio_rebuffers: send.outgoing_audio_rebuffers,
|
||||
outgoing_audio_max_frame_gap_ms: send.outgoing_audio_max_frame_gap_ms,
|
||||
adaptive_send_tier: send.adaptive_send_tier.clone(),
|
||||
adaptive_send_reason: send.adaptive_send_reason.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn bitrate_first_sample_is_zero() {
|
||||
let cur = ByteRateSample {
|
||||
bytes: 1000,
|
||||
timestamp_us: 1_000_000,
|
||||
};
|
||||
assert_eq!(bitrate_kbps(None, cur), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrate_computes_kbps_from_byte_delta() {
|
||||
let prev = ByteRateSample {
|
||||
bytes: 0,
|
||||
timestamp_us: 0,
|
||||
};
|
||||
let cur = ByteRateSample {
|
||||
bytes: 12_500,
|
||||
timestamp_us: 1_000_000,
|
||||
};
|
||||
assert_eq!(bitrate_kbps(Some(prev), cur), 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrate_half_second_doubles_rate() {
|
||||
let prev = ByteRateSample {
|
||||
bytes: 1000,
|
||||
timestamp_us: 1_000_000,
|
||||
};
|
||||
let cur = ByteRateSample {
|
||||
bytes: 13_500,
|
||||
timestamp_us: 1_500_000,
|
||||
};
|
||||
assert_eq!(bitrate_kbps(Some(prev), cur), 200.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrate_rejects_backwards_time_and_bytes() {
|
||||
let prev = ByteRateSample {
|
||||
bytes: 5000,
|
||||
timestamp_us: 2_000_000,
|
||||
};
|
||||
assert_eq!(
|
||||
bitrate_kbps(
|
||||
Some(prev),
|
||||
ByteRateSample {
|
||||
bytes: 6000,
|
||||
timestamp_us: 1_000_000
|
||||
}
|
||||
),
|
||||
0.0
|
||||
);
|
||||
assert_eq!(
|
||||
bitrate_kbps(
|
||||
Some(prev),
|
||||
ByteRateSample {
|
||||
bytes: 6000,
|
||||
timestamp_us: 2_000_000
|
||||
}
|
||||
),
|
||||
0.0
|
||||
);
|
||||
assert_eq!(
|
||||
bitrate_kbps(
|
||||
Some(prev),
|
||||
ByteRateSample {
|
||||
bytes: 100,
|
||||
timestamp_us: 3_000_000
|
||||
}
|
||||
),
|
||||
0.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_kbps_drops_nan_inf_negative_and_rounds() {
|
||||
assert_eq!(sanitize_kbps(f64::NAN), 0.0);
|
||||
assert_eq!(sanitize_kbps(f64::INFINITY), 0.0);
|
||||
assert_eq!(sanitize_kbps(-5.0), 0.0);
|
||||
assert_eq!(sanitize_kbps(123.456), 123.5);
|
||||
assert_eq!(sanitize_kbps(100.0), 100.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unit_conversions_seconds_to_ms() {
|
||||
assert_eq!(jitter_seconds_to_ms(0.012), Some(12.0));
|
||||
assert_eq!(jitter_seconds_to_ms(-1.0), None);
|
||||
assert_eq!(jitter_seconds_to_ms(f64::NAN), None);
|
||||
assert_eq!(rtt_seconds_to_ms(0.045), Some(45.0));
|
||||
assert_eq!(rtt_seconds_to_ms(0.0), None);
|
||||
assert_eq!(rtt_seconds_to_ms(f64::INFINITY), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_audio_level_clamps() {
|
||||
assert_eq!(sanitize_audio_level(0.5), Some(0.5));
|
||||
assert_eq!(sanitize_audio_level(2.0), Some(1.0));
|
||||
assert_eq!(sanitize_audio_level(-0.1), Some(0.0));
|
||||
assert_eq!(sanitize_audio_level(f64::NAN), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_stats_serialise_to_null_rtt_and_empty_arrays() {
|
||||
let stats = ConnectionStats::default();
|
||||
assert_eq!(
|
||||
stats_to_json(&stats),
|
||||
"{\"rttMs\":null,\"outbound\":[],\"inbound\":[],\"send\":null}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_stats_serialise_to_exact_contract_shape() {
|
||||
let stats = ConnectionStats {
|
||||
rtt_ms: Some(42.0),
|
||||
outbound: vec![
|
||||
OutboundEntry {
|
||||
track_sid: "TR_mic1".into(),
|
||||
source: "microphone".into(),
|
||||
kind: "audio".into(),
|
||||
codec: Some("audio/opus".into()),
|
||||
bitrate_kbps: 32.0,
|
||||
packets_lost: 0,
|
||||
fps: None,
|
||||
audio_level: Some(0.62),
|
||||
..Default::default()
|
||||
},
|
||||
OutboundEntry {
|
||||
track_sid: "TR_screen1".into(),
|
||||
source: "screen_share".into(),
|
||||
kind: "video".into(),
|
||||
codec: Some("video/H265".into()),
|
||||
bitrate_kbps: 2500.5,
|
||||
packets_lost: 3,
|
||||
packets_sent: 9000,
|
||||
fps: Some(30.0),
|
||||
audio_level: None,
|
||||
width: Some(2176),
|
||||
height: Some(1200),
|
||||
source_width: Some(2176),
|
||||
source_height: Some(1200),
|
||||
target_bitrate_kbps: Some(50_000.0),
|
||||
configured_fps: Some(60.0),
|
||||
target_fps: Some(30.0),
|
||||
effective_fps: Some(29.8),
|
||||
frames_produced: Some(120),
|
||||
frames_accepted: Some(118),
|
||||
frames_dropped: Some(1),
|
||||
frames_coalesced: Some(2),
|
||||
frames_captured: Some(117),
|
||||
capture_failures: Some(0),
|
||||
max_queue_age_ms: Some(18),
|
||||
max_push_latency_ms: Some(12),
|
||||
adaptive_send_tier: Some("fps30".into()),
|
||||
adaptive_send_reason: Some("encoderEgressPressure".into()),
|
||||
},
|
||||
],
|
||||
inbound: vec![InboundEntry {
|
||||
participant_sid: "PA_remote1".into(),
|
||||
participant_identity: Some("user_2_connection_2".into()),
|
||||
track_sid: "TR_remoteAudio".into(),
|
||||
source: Some("microphone".into()),
|
||||
kind: "audio".into(),
|
||||
codec: Some("audio/opus".into()),
|
||||
bitrate_kbps: 28.0,
|
||||
packets_lost: 1,
|
||||
packets_received: 990,
|
||||
jitter_ms: Some(5.0),
|
||||
audio_level: Some(0.75),
|
||||
fps: None,
|
||||
width: None,
|
||||
height: None,
|
||||
source_width: None,
|
||||
source_height: None,
|
||||
}],
|
||||
send: None,
|
||||
};
|
||||
let json = stats_to_json(&stats);
|
||||
assert_eq!(
|
||||
json,
|
||||
"{\"rttMs\":42,\"outbound\":[\
|
||||
{\"trackSid\":\"TR_mic1\",\"source\":\"microphone\",\"kind\":\"audio\",\"bitrateKbps\":32,\"packetsLost\":0,\"packetsSent\":0,\"audioLevel\":0.62,\"codec\":\"audio/opus\"},\
|
||||
{\"trackSid\":\"TR_screen1\",\"source\":\"screen_share\",\"kind\":\"video\",\"bitrateKbps\":2500.5,\"packetsLost\":3,\"packetsSent\":9000,\"fps\":30,\"width\":2176,\"height\":1200,\"sourceWidth\":2176,\"sourceHeight\":1200,\"targetBitrateKbps\":50000,\"configuredFps\":60,\"targetFps\":30,\"effectiveFps\":29.8,\"framesProduced\":120,\"framesAccepted\":118,\"framesDropped\":1,\"framesCoalesced\":2,\"framesCaptured\":117,\"captureFailures\":0,\"maxQueueAgeMs\":18,\"maxPushLatencyMs\":12,\"adaptiveSendTier\":\"fps30\",\"adaptiveSendReason\":\"encoderEgressPressure\",\"codec\":\"video/H265\"}\
|
||||
],\"inbound\":[\
|
||||
{\"participantSid\":\"PA_remote1\",\"trackSid\":\"TR_remoteAudio\",\"kind\":\"audio\",\"bitrateKbps\":28,\"packetsLost\":1,\"packetsReceived\":990,\"participantIdentity\":\"user_2_connection_2\",\"source\":\"microphone\",\"jitterMs\":5,\"audioLevel\":0.75,\"codec\":\"audio/opus\"}\
|
||||
],\"send\":null}"
|
||||
);
|
||||
let _ = json;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_health_serialises_to_exact_contract_shape() {
|
||||
let stats = ConnectionStats {
|
||||
rtt_ms: None,
|
||||
outbound: vec![],
|
||||
inbound: vec![],
|
||||
send: Some(SendHealthSnapshot {
|
||||
outgoing_video_queue_depth: 1,
|
||||
outgoing_video_queue_capacity: 8,
|
||||
outgoing_video_max_queue_depth: 4,
|
||||
outgoing_video_frames_produced: 2,
|
||||
outgoing_video_frames_accepted: 3,
|
||||
outgoing_video_frames_dropped: 4,
|
||||
outgoing_video_frames_coalesced: 5,
|
||||
outgoing_video_frames_captured: 6,
|
||||
outgoing_video_capture_failures: 7,
|
||||
outgoing_video_effective_fps: 59.94,
|
||||
outgoing_video_target_fps: 30.0,
|
||||
outgoing_video_pacing_target_fps: 60.0,
|
||||
outgoing_video_max_queue_age_ms: 8,
|
||||
outgoing_video_max_push_latency_ms: 9,
|
||||
outgoing_video_pacing_mode: "source".to_string(),
|
||||
outgoing_video_bus_active: true,
|
||||
outgoing_audio_buffer_target_ms: 300,
|
||||
outgoing_audio_buffer_max_ms: 750,
|
||||
outgoing_audio_underruns: 10,
|
||||
outgoing_audio_rebuffers: 11,
|
||||
outgoing_audio_max_frame_gap_ms: 120,
|
||||
adaptive_send_tier: "fps30".to_string(),
|
||||
adaptive_send_reason: "sendLatencyPressure".to_string(),
|
||||
}),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
stats_to_json(&stats),
|
||||
"{\"rttMs\":null,\"outbound\":[],\"inbound\":[],\"send\":{\
|
||||
\"outgoingVideoQueueDepth\":1,\
|
||||
\"outgoingVideoQueueCapacity\":8,\
|
||||
\"outgoingVideoMaxQueueDepth\":4,\
|
||||
\"outgoingVideoFramesProduced\":2,\
|
||||
\"outgoingVideoFramesAccepted\":3,\
|
||||
\"outgoingVideoFramesDropped\":4,\
|
||||
\"outgoingVideoFramesCoalesced\":5,\
|
||||
\"outgoingVideoFramesCaptured\":6,\
|
||||
\"outgoingVideoCaptureFailures\":7,\
|
||||
\"outgoingVideoEffectiveFps\":59.94,\
|
||||
\"outgoingVideoTargetFps\":30,\
|
||||
\"outgoingVideoPacingTargetFps\":60,\
|
||||
\"outgoingVideoMaxQueueAgeMs\":8,\
|
||||
\"outgoingVideoMaxPushLatencyMs\":9,\
|
||||
\"outgoingVideoPacingMode\":\"source\",\
|
||||
\"outgoingVideoBusActive\":true,\
|
||||
\"outgoingAudioBufferTargetMs\":300,\
|
||||
\"outgoingAudioBufferMaxMs\":750,\
|
||||
\"outgoingAudioUnderruns\":10,\
|
||||
\"outgoingAudioRebuffers\":11,\
|
||||
\"outgoingAudioMaxFrameGapMs\":120,\
|
||||
\"adaptiveSendTier\":\"fps30\",\
|
||||
\"adaptiveSendReason\":\"sendLatencyPressure\"\
|
||||
}}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_inbound_omits_audio_level_audio_omits_fps() {
|
||||
let stats = ConnectionStats {
|
||||
rtt_ms: None,
|
||||
outbound: vec![],
|
||||
inbound: vec![InboundEntry {
|
||||
participant_sid: "PA_x".into(),
|
||||
participant_identity: None,
|
||||
track_sid: "TR_v".into(),
|
||||
source: Some("screen_share".into()),
|
||||
kind: "video".into(),
|
||||
codec: None,
|
||||
bitrate_kbps: 1000.0,
|
||||
packets_lost: 0,
|
||||
packets_received: 5000,
|
||||
jitter_ms: None,
|
||||
audio_level: None,
|
||||
fps: Some(29.94),
|
||||
width: Some(3840),
|
||||
height: Some(2160),
|
||||
source_width: Some(3840),
|
||||
source_height: Some(2160),
|
||||
}],
|
||||
send: None,
|
||||
};
|
||||
let json = stats_to_json(&stats);
|
||||
assert!(!json.contains("audioLevel"));
|
||||
assert!(!json.contains("jitterMs"));
|
||||
assert!(json.contains("\"source\":\"screen_share\""));
|
||||
assert!(json.contains("\"fps\":29.9"));
|
||||
assert!(json.contains("\"width\":3840"));
|
||||
assert!(json.contains("\"height\":2160"));
|
||||
assert!(json.contains("\"rttMs\":null"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,747 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct TextureFrameDesc {
|
||||
pub handle: u64,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub dxgi_format: u32,
|
||||
pub timestamp_us: i64,
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct DmabufFrameDesc {
|
||||
pub plane_count: u8,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub drm_format: u32,
|
||||
pub modifier: u64,
|
||||
pub strides: [u32; 4],
|
||||
pub offsets: [u32; 4],
|
||||
pub device_uuid: [u8; 16],
|
||||
pub timestamp_us: i64,
|
||||
}
|
||||
|
||||
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum TextureEncodeError {
|
||||
NoTexture,
|
||||
InvalidDimensions,
|
||||
UnsupportedFormat,
|
||||
InvalidPlanes,
|
||||
UnsupportedCodec,
|
||||
NoHardwareEncoder,
|
||||
SdkNativeTextureUnsupported,
|
||||
}
|
||||
|
||||
impl TextureEncodeError {
|
||||
#[allow(dead_code)]
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
TextureEncodeError::NoTexture => "noTexture",
|
||||
TextureEncodeError::InvalidDimensions => "invalidDimensions",
|
||||
TextureEncodeError::UnsupportedFormat => "unsupportedFormat",
|
||||
TextureEncodeError::InvalidPlanes => "invalidPlanes",
|
||||
TextureEncodeError::UnsupportedCodec => "unsupportedCodec",
|
||||
TextureEncodeError::NoHardwareEncoder => "noHardwareEncoder",
|
||||
TextureEncodeError::SdkNativeTextureUnsupported => "sdkNativeTextureUnsupported",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
const DXGI_FORMAT_R8G8B8A8_UNORM: u32 = 28;
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
const DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: u32 = 29;
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
const DXGI_FORMAT_B8G8R8A8_UNORM: u32 = 87;
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
const DXGI_FORMAT_B8G8R8A8_UNORM_SRGB: u32 = 91;
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
const DXGI_FORMAT_NV12: u32 = 103;
|
||||
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
pub fn dxgi_format_supported(dxgi_format: u32) -> bool {
|
||||
matches!(
|
||||
dxgi_format,
|
||||
DXGI_FORMAT_R8G8B8A8_UNORM
|
||||
| DXGI_FORMAT_R8G8B8A8_UNORM_SRGB
|
||||
| DXGI_FORMAT_B8G8R8A8_UNORM
|
||||
| DXGI_FORMAT_B8G8R8A8_UNORM_SRGB
|
||||
| DXGI_FORMAT_NV12
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const fn fourcc(bytes: [u8; 4]) -> u32 {
|
||||
u32::from_le_bytes(bytes)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_XRGB8888: u32 = fourcc(*b"XR24");
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_ARGB8888: u32 = fourcc(*b"AR24");
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_XBGR8888: u32 = fourcc(*b"XB24");
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_ABGR8888: u32 = fourcc(*b"AB24");
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_XRGB2101010: u32 = fourcc(*b"XR30");
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_ARGB2101010: u32 = fourcc(*b"AR30");
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_XBGR2101010: u32 = fourcc(*b"XB30");
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_ABGR2101010: u32 = fourcc(*b"AB30");
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const DRM_FORMAT_NV12: u32 = fourcc(*b"NV12");
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
pub fn drm_format_supported(drm_format: u32) -> bool {
|
||||
matches!(
|
||||
drm_format,
|
||||
DRM_FORMAT_XRGB8888
|
||||
| DRM_FORMAT_ARGB8888
|
||||
| DRM_FORMAT_XBGR8888
|
||||
| DRM_FORMAT_ABGR8888
|
||||
| DRM_FORMAT_XRGB2101010
|
||||
| DRM_FORMAT_ARGB2101010
|
||||
| DRM_FORMAT_XBGR2101010
|
||||
| DRM_FORMAT_ABGR2101010
|
||||
| DRM_FORMAT_NV12
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "windows", test))]
|
||||
const MAX_TEXTURE_EDGE: u32 = 8192;
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "windows", test))]
|
||||
fn validate_dimensions(width: u32, height: u32) -> Result<(), TextureEncodeError> {
|
||||
if width < 2
|
||||
|| height < 2
|
||||
|| !width.is_multiple_of(2)
|
||||
|| !height.is_multiple_of(2)
|
||||
|| width > MAX_TEXTURE_EDGE
|
||||
|| height > MAX_TEXTURE_EDGE
|
||||
{
|
||||
return Err(TextureEncodeError::InvalidDimensions);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
pub fn validate_texture_desc(desc: &TextureFrameDesc) -> Result<(), TextureEncodeError> {
|
||||
if desc.handle == 0 {
|
||||
return Err(TextureEncodeError::NoTexture);
|
||||
}
|
||||
validate_dimensions(desc.width, desc.height)?;
|
||||
if !dxgi_format_supported(desc.dxgi_format) {
|
||||
return Err(TextureEncodeError::UnsupportedFormat);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
pub fn validate_dmabuf_desc(desc: &DmabufFrameDesc) -> Result<(), TextureEncodeError> {
|
||||
validate_dimensions(desc.width, desc.height)?;
|
||||
if !drm_format_supported(desc.drm_format) {
|
||||
return Err(TextureEncodeError::UnsupportedFormat);
|
||||
}
|
||||
let plane_count = desc.plane_count as usize;
|
||||
if !(1..=4).contains(&plane_count) {
|
||||
return Err(TextureEncodeError::InvalidPlanes);
|
||||
}
|
||||
for plane in 0..plane_count {
|
||||
if desc.strides[plane] == 0 {
|
||||
return Err(TextureEncodeError::InvalidPlanes);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[cfg(test)]
|
||||
pub fn dmabuf_desc_from_parts(
|
||||
fds: &[i32],
|
||||
plane_count: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
drm_format: u32,
|
||||
modifier: u64,
|
||||
strides: &[u32],
|
||||
offsets: &[u32],
|
||||
device_uuid: &[u8],
|
||||
timestamp_us: f64,
|
||||
) -> Option<(DmabufFrameDesc, [i32; 4])> {
|
||||
let plane_count_u8 = u8::try_from(plane_count).ok()?;
|
||||
let planes = plane_count as usize;
|
||||
if !(1..=4).contains(&planes)
|
||||
|| fds.len() < planes
|
||||
|| strides.len() < planes
|
||||
|| offsets.len() < planes
|
||||
|| device_uuid.len() != 16
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if fds.iter().take(planes).any(|fd| *fd < 0) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut fd_array = [-1; 4];
|
||||
let mut stride_array = [0; 4];
|
||||
let mut offset_array = [0; 4];
|
||||
fd_array[..planes].copy_from_slice(&fds[..planes]);
|
||||
stride_array[..planes].copy_from_slice(&strides[..planes]);
|
||||
offset_array[..planes].copy_from_slice(&offsets[..planes]);
|
||||
let mut uuid = [0u8; 16];
|
||||
uuid.copy_from_slice(device_uuid);
|
||||
|
||||
Some((
|
||||
DmabufFrameDesc {
|
||||
plane_count: plane_count_u8,
|
||||
width,
|
||||
height,
|
||||
drm_format,
|
||||
modifier,
|
||||
strides: stride_array,
|
||||
offsets: offset_array,
|
||||
device_uuid: uuid,
|
||||
timestamp_us: timestamp_us as i64,
|
||||
},
|
||||
fd_array,
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct TextureCapability {
|
||||
pub available: bool,
|
||||
pub reason: TextureEncodeError,
|
||||
}
|
||||
|
||||
impl TextureCapability {
|
||||
pub fn unavailable(reason: TextureEncodeError) -> Self {
|
||||
Self {
|
||||
available: false,
|
||||
reason,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn for_screen_codec(codec: &str, has_hardware_encoder: bool) -> Self {
|
||||
if !codec_allows_native_gpu(codec) {
|
||||
return Self::unavailable(TextureEncodeError::UnsupportedCodec);
|
||||
}
|
||||
if !has_hardware_encoder {
|
||||
return Self {
|
||||
available: false,
|
||||
reason: TextureEncodeError::NoHardwareEncoder,
|
||||
};
|
||||
}
|
||||
Self {
|
||||
available: true,
|
||||
reason: TextureEncodeError::NoTexture,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn codec_allows_native_gpu(codec: &str) -> bool {
|
||||
matches!(
|
||||
codec.trim().to_ascii_lowercase().as_str(),
|
||||
"h264" | "h265" | "hevc"
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
fn sdk_accepts_d3d11_texture_buffers() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn sdk_accepts_dmabuf_texture_buffers() -> bool {
|
||||
cfg!(target_os = "linux")
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
pub fn should_attempt_texture_encode(
|
||||
capability: &TextureCapability,
|
||||
desc: &TextureFrameDesc,
|
||||
) -> Result<(), TextureEncodeError> {
|
||||
if !capability.available {
|
||||
return Err(capability.reason);
|
||||
}
|
||||
if !sdk_accepts_d3d11_texture_buffers() {
|
||||
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
|
||||
}
|
||||
validate_texture_desc(desc)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn should_attempt_dmabuf_encode(
|
||||
capability: &TextureCapability,
|
||||
desc: &DmabufFrameDesc,
|
||||
) -> Result<(), TextureEncodeError> {
|
||||
if !capability.available {
|
||||
return Err(capability.reason);
|
||||
}
|
||||
if !sdk_accepts_dmabuf_texture_buffers() {
|
||||
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
|
||||
}
|
||||
validate_dmabuf_desc(desc)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn should_attempt_texture_encode_for_tests(
|
||||
capability: &TextureCapability,
|
||||
desc: &TextureFrameDesc,
|
||||
sdk_accepts_d3d11: bool,
|
||||
) -> Result<(), TextureEncodeError> {
|
||||
if !capability.available {
|
||||
return Err(capability.reason);
|
||||
}
|
||||
if !sdk_accepts_d3d11 {
|
||||
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
|
||||
}
|
||||
validate_texture_desc(desc)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn should_attempt_dmabuf_encode_for_tests(
|
||||
capability: &TextureCapability,
|
||||
desc: &DmabufFrameDesc,
|
||||
sdk_accepts_dmabuf: bool,
|
||||
) -> Result<(), TextureEncodeError> {
|
||||
if !capability.available {
|
||||
return Err(capability.reason);
|
||||
}
|
||||
if !sdk_accepts_dmabuf {
|
||||
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
|
||||
}
|
||||
validate_dmabuf_desc(desc)
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "publisher", any(target_os = "linux", target_os = "windows")))]
|
||||
pub mod bridge {
|
||||
#[cfg(target_os = "linux")]
|
||||
use super::{DmabufFrameDesc, should_attempt_dmabuf_encode};
|
||||
use super::{TextureCapability, TextureEncodeError};
|
||||
#[cfg(target_os = "windows")]
|
||||
use super::{TextureFrameDesc, should_attempt_texture_encode};
|
||||
use livekit::webrtc::video_frame::{VideoFrame, VideoRotation, native::NativeBuffer};
|
||||
use livekit::webrtc::video_source::native::NativeVideoSource;
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn try_publish_texture(
|
||||
source: &NativeVideoSource,
|
||||
capability: &TextureCapability,
|
||||
desc: &TextureFrameDesc,
|
||||
) -> Result<(), TextureEncodeError> {
|
||||
should_attempt_texture_encode(capability, desc)?;
|
||||
|
||||
let Some(buffer) = NativeBuffer::from_fluxer_d3d11_texture(
|
||||
desc.handle,
|
||||
desc.width,
|
||||
desc.height,
|
||||
desc.dxgi_format,
|
||||
) else {
|
||||
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
|
||||
};
|
||||
publish_native_buffer(source, buffer, desc.timestamp_us);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn try_publish_dmabuf(
|
||||
source: &NativeVideoSource,
|
||||
capability: &TextureCapability,
|
||||
desc: &DmabufFrameDesc,
|
||||
fds: [i32; 4],
|
||||
) -> Result<(), TextureEncodeError> {
|
||||
should_attempt_dmabuf_encode(capability, desc)?;
|
||||
let uuid_hi = u64::from_be_bytes(desc.device_uuid[0..8].try_into().unwrap_or([0; 8]));
|
||||
let uuid_lo = u64::from_be_bytes(desc.device_uuid[8..16].try_into().unwrap_or([0; 8]));
|
||||
let Some(buffer) = NativeBuffer::from_fluxer_dmabuf_texture(
|
||||
fds,
|
||||
desc.plane_count as u32,
|
||||
desc.width,
|
||||
desc.height,
|
||||
desc.drm_format,
|
||||
desc.modifier,
|
||||
desc.strides,
|
||||
desc.offsets,
|
||||
uuid_hi,
|
||||
uuid_lo,
|
||||
) else {
|
||||
return Err(TextureEncodeError::SdkNativeTextureUnsupported);
|
||||
};
|
||||
publish_native_buffer(source, buffer, desc.timestamp_us);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn publish_native_buffer(source: &NativeVideoSource, buffer: NativeBuffer, timestamp_us: i64) {
|
||||
source.capture_frame(&VideoFrame {
|
||||
rotation: VideoRotation::VideoRotation0,
|
||||
timestamp_us,
|
||||
frame_metadata: None,
|
||||
buffer,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn good_desc() -> TextureFrameDesc {
|
||||
TextureFrameDesc {
|
||||
handle: 0xDEAD_BEEF,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
dxgi_format: DXGI_FORMAT_B8G8R8A8_UNORM,
|
||||
timestamp_us: 123_456,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dxgi_format_gate_accepts_8bit_rgba_bgra_only() {
|
||||
assert!(dxgi_format_supported(DXGI_FORMAT_B8G8R8A8_UNORM));
|
||||
assert!(dxgi_format_supported(DXGI_FORMAT_B8G8R8A8_UNORM_SRGB));
|
||||
assert!(dxgi_format_supported(DXGI_FORMAT_R8G8B8A8_UNORM));
|
||||
assert!(dxgi_format_supported(DXGI_FORMAT_R8G8B8A8_UNORM_SRGB));
|
||||
assert!(dxgi_format_supported(DXGI_FORMAT_NV12));
|
||||
assert!(!dxgi_format_supported(24));
|
||||
assert!(!dxgi_format_supported(10));
|
||||
assert!(!dxgi_format_supported(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drm_format_gate_accepts_obs_vkcapture_texture_formats() {
|
||||
assert!(drm_format_supported(DRM_FORMAT_ARGB8888));
|
||||
assert!(drm_format_supported(DRM_FORMAT_ABGR8888));
|
||||
assert!(drm_format_supported(DRM_FORMAT_ARGB2101010));
|
||||
assert!(drm_format_supported(DRM_FORMAT_ABGR2101010));
|
||||
assert!(drm_format_supported(DRM_FORMAT_NV12));
|
||||
assert!(
|
||||
!drm_format_supported(fourcc(*b"AB4H")),
|
||||
"16-bit float DMA-BUF import is not encodable by the native NVENC bridge yet"
|
||||
);
|
||||
assert!(!drm_format_supported(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_zero_handle_as_no_texture() {
|
||||
let mut d = good_desc();
|
||||
d.handle = 0;
|
||||
assert_eq!(
|
||||
validate_texture_desc(&d),
|
||||
Err(TextureEncodeError::NoTexture)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_odd_zero_and_oversized_dims() {
|
||||
for (w, h) in [
|
||||
(1921, 1080),
|
||||
(1920, 1081),
|
||||
(0, 1080),
|
||||
(1920, 0),
|
||||
(8194, 1080),
|
||||
(1920, 8194),
|
||||
] {
|
||||
let mut d = good_desc();
|
||||
d.width = w;
|
||||
d.height = h;
|
||||
assert_eq!(
|
||||
validate_texture_desc(&d),
|
||||
Err(TextureEncodeError::InvalidDimensions),
|
||||
"dims {w}x{h} should be rejected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_unsupported_format() {
|
||||
let mut d = good_desc();
|
||||
d.dxgi_format = 24;
|
||||
assert_eq!(
|
||||
validate_texture_desc(&d),
|
||||
Err(TextureEncodeError::UnsupportedFormat)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_accepts_a_clean_bgra_texture() {
|
||||
assert_eq!(validate_texture_desc(&good_desc()), Ok(()));
|
||||
}
|
||||
|
||||
fn good_dmabuf_desc() -> DmabufFrameDesc {
|
||||
DmabufFrameDesc {
|
||||
plane_count: 1,
|
||||
width: 1920,
|
||||
height: 1080,
|
||||
drm_format: DRM_FORMAT_ARGB8888,
|
||||
modifier: 0,
|
||||
strides: [1920 * 4, 0, 0, 0],
|
||||
offsets: [0, 0, 0, 0],
|
||||
device_uuid: [1; 16],
|
||||
timestamp_us: 123_456,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_dmabuf_accepts_supported_formats_with_optional_uuid() {
|
||||
assert_eq!(validate_dmabuf_desc(&good_dmabuf_desc()), Ok(()));
|
||||
let mut desc = good_dmabuf_desc();
|
||||
desc.drm_format = DRM_FORMAT_NV12;
|
||||
desc.strides[0] = 1920;
|
||||
assert_eq!(validate_dmabuf_desc(&desc), Ok(()));
|
||||
desc = good_dmabuf_desc();
|
||||
desc.device_uuid = [0; 16];
|
||||
assert_eq!(validate_dmabuf_desc(&desc), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_dmabuf_rejects_invalid_planes() {
|
||||
let mut desc = good_dmabuf_desc();
|
||||
desc.plane_count = 0;
|
||||
assert_eq!(
|
||||
validate_dmabuf_desc(&desc),
|
||||
Err(TextureEncodeError::InvalidPlanes)
|
||||
);
|
||||
desc = good_dmabuf_desc();
|
||||
desc.strides[0] = 0;
|
||||
assert_eq!(
|
||||
validate_dmabuf_desc(&desc),
|
||||
Err(TextureEncodeError::InvalidPlanes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dmabuf_desc_from_parts_rejects_negative_fds() {
|
||||
assert!(
|
||||
dmabuf_desc_from_parts(
|
||||
&[-1],
|
||||
1,
|
||||
1920,
|
||||
1080,
|
||||
DRM_FORMAT_ARGB8888,
|
||||
0,
|
||||
&[1920 * 4],
|
||||
&[0],
|
||||
&[1; 16],
|
||||
123.0,
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dmabuf_desc_from_parts_populates_all_plane_arrays() {
|
||||
let uuid = [7u8; 16];
|
||||
let (desc, fds) = dmabuf_desc_from_parts(
|
||||
&[10, 11, 12, 99],
|
||||
3,
|
||||
1920,
|
||||
1080,
|
||||
DRM_FORMAT_NV12,
|
||||
0xABCD,
|
||||
&[1920, 960, 960, 777],
|
||||
&[0, 2_073_600, 3_110_400, 999],
|
||||
&uuid,
|
||||
123_456.75,
|
||||
)
|
||||
.expect("valid multi-plane descriptor");
|
||||
|
||||
assert_eq!(desc.plane_count, 3);
|
||||
assert_eq!(desc.width, 1920);
|
||||
assert_eq!(desc.height, 1080);
|
||||
assert_eq!(desc.drm_format, DRM_FORMAT_NV12);
|
||||
assert_eq!(desc.modifier, 0xABCD);
|
||||
assert_eq!(desc.strides, [1920, 960, 960, 0]);
|
||||
assert_eq!(desc.offsets, [0, 2_073_600, 3_110_400, 0]);
|
||||
assert_eq!(desc.device_uuid, uuid);
|
||||
assert_eq!(desc.timestamp_us, 123_456);
|
||||
assert_eq!(fds, [10, 11, 12, -1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dmabuf_desc_from_parts_rejects_incomplete_native_inputs() {
|
||||
let uuid = [1u8; 16];
|
||||
for (fds, strides, offsets, uuid_bytes, label) in [
|
||||
(&[4][..], &[128][..], &[0][..], &uuid[..], "too few fds"),
|
||||
(
|
||||
&[4, 5][..],
|
||||
&[128][..],
|
||||
&[0, 64][..],
|
||||
&uuid[..],
|
||||
"too few strides",
|
||||
),
|
||||
(
|
||||
&[4, 5][..],
|
||||
&[128, 128][..],
|
||||
&[0][..],
|
||||
&uuid[..],
|
||||
"too few offsets",
|
||||
),
|
||||
(
|
||||
&[4, 5][..],
|
||||
&[128, 128][..],
|
||||
&[0, 64][..],
|
||||
&[1u8; 15][..],
|
||||
"bad uuid",
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
dmabuf_desc_from_parts(
|
||||
fds,
|
||||
2,
|
||||
128,
|
||||
128,
|
||||
DRM_FORMAT_ARGB8888,
|
||||
0,
|
||||
strides,
|
||||
offsets,
|
||||
uuid_bytes,
|
||||
0.0,
|
||||
)
|
||||
.is_none(),
|
||||
"{label} should be rejected"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
dmabuf_desc_from_parts(
|
||||
&[4, 5, 6, 7, 8],
|
||||
5,
|
||||
128,
|
||||
128,
|
||||
DRM_FORMAT_ARGB8888,
|
||||
0,
|
||||
&[128; 5],
|
||||
&[0; 5],
|
||||
&uuid,
|
||||
0.0,
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_precedence_handle_before_dims_before_format() {
|
||||
let d = TextureFrameDesc {
|
||||
handle: 0,
|
||||
width: 1921,
|
||||
height: 0,
|
||||
dxgi_format: 999,
|
||||
timestamp_us: 0,
|
||||
};
|
||||
assert_eq!(
|
||||
validate_texture_desc(&d),
|
||||
Err(TextureEncodeError::NoTexture)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_is_available_only_for_explicit_hardware_codecs() {
|
||||
for codec in ["h264", "H264", "h265", "hevc", "HEVC"] {
|
||||
let cap = TextureCapability::for_screen_codec(codec, true);
|
||||
assert!(cap.available, "{codec} should allow native GPU buffers");
|
||||
assert_eq!(cap.reason, TextureEncodeError::NoTexture);
|
||||
}
|
||||
for codec in ["", "vp8", "vp9", "av1", "rubbish"] {
|
||||
let cap = TextureCapability::for_screen_codec(codec, true);
|
||||
assert!(
|
||||
!cap.available,
|
||||
"{codec} should not allow native GPU buffers"
|
||||
);
|
||||
assert_eq!(cap.reason, TextureEncodeError::UnsupportedCodec);
|
||||
}
|
||||
let cap = TextureCapability::for_screen_codec("h264", false);
|
||||
assert!(!cap.available);
|
||||
assert_eq!(cap.reason, TextureEncodeError::NoHardwareEncoder);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_attempt_falls_back_when_capability_unavailable() {
|
||||
let cap = TextureCapability {
|
||||
available: false,
|
||||
reason: TextureEncodeError::SdkNativeTextureUnsupported,
|
||||
};
|
||||
assert_eq!(
|
||||
should_attempt_texture_encode(&cap, &good_desc()),
|
||||
Err(TextureEncodeError::SdkNativeTextureUnsupported)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_attempt_validates_frame_when_capability_available() {
|
||||
let open = TextureCapability {
|
||||
available: true,
|
||||
reason: TextureEncodeError::NoTexture,
|
||||
};
|
||||
assert_eq!(
|
||||
should_attempt_texture_encode_for_tests(&open, &good_desc(), true),
|
||||
Ok(())
|
||||
);
|
||||
assert_eq!(should_attempt_texture_encode(&open, &good_desc()), Ok(()));
|
||||
let mut bad = good_desc();
|
||||
bad.handle = 0;
|
||||
assert_eq!(
|
||||
should_attempt_texture_encode_for_tests(&open, &bad, true),
|
||||
Err(TextureEncodeError::NoTexture)
|
||||
);
|
||||
bad = good_desc();
|
||||
bad.dxgi_format = 24;
|
||||
assert_eq!(
|
||||
should_attempt_texture_encode_for_tests(&open, &bad, true),
|
||||
Err(TextureEncodeError::UnsupportedFormat)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_attempt_dmabuf_is_sdk_gated_after_validation_capability() {
|
||||
let open = TextureCapability {
|
||||
available: true,
|
||||
reason: TextureEncodeError::NoTexture,
|
||||
};
|
||||
assert_eq!(
|
||||
should_attempt_dmabuf_encode_for_tests(&open, &good_dmabuf_desc(), true),
|
||||
Ok(())
|
||||
);
|
||||
assert_eq!(
|
||||
should_attempt_dmabuf_encode_for_tests(&open, &good_dmabuf_desc(), false),
|
||||
Err(TextureEncodeError::SdkNativeTextureUnsupported)
|
||||
);
|
||||
let mut invalid = good_dmabuf_desc();
|
||||
invalid.plane_count = 5;
|
||||
assert_eq!(
|
||||
should_attempt_dmabuf_encode_for_tests(&open, &invalid, true),
|
||||
Err(TextureEncodeError::InvalidPlanes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_strings_are_stable() {
|
||||
assert_eq!(TextureEncodeError::NoTexture.as_str(), "noTexture");
|
||||
assert_eq!(
|
||||
TextureEncodeError::InvalidDimensions.as_str(),
|
||||
"invalidDimensions"
|
||||
);
|
||||
assert_eq!(
|
||||
TextureEncodeError::UnsupportedFormat.as_str(),
|
||||
"unsupportedFormat"
|
||||
);
|
||||
assert_eq!(TextureEncodeError::InvalidPlanes.as_str(), "invalidPlanes");
|
||||
assert_eq!(
|
||||
TextureEncodeError::UnsupportedCodec.as_str(),
|
||||
"unsupportedCodec"
|
||||
);
|
||||
assert_eq!(
|
||||
TextureEncodeError::NoHardwareEncoder.as_str(),
|
||||
"noHardwareEncoder"
|
||||
);
|
||||
assert_eq!(
|
||||
TextureEncodeError::SdkNativeTextureUnsupported.as_str(),
|
||||
"sdkNativeTextureUnsupported"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,732 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Debug)]
|
||||
pub struct I420 {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub y: Vec<u8>,
|
||||
pub u: Vec<u8>,
|
||||
pub v: Vec<u8>,
|
||||
}
|
||||
|
||||
impl I420 {
|
||||
pub fn new(width: u32, height: u32) -> Option<Self> {
|
||||
if !dims_ok(width, height) {
|
||||
return None;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
Some(Self {
|
||||
width,
|
||||
height,
|
||||
y: vec![0u8; w * h],
|
||||
u: vec![0u8; (w / 2) * (h / 2)],
|
||||
v: vec![0u8; (w / 2) * (h / 2)],
|
||||
})
|
||||
}
|
||||
|
||||
fn has_layout(&self, width: u32, height: u32) -> bool {
|
||||
if self.width != width || self.height != height {
|
||||
return false;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
self.y.len() == w * h && self.u.len() == (w / 2) * (h / 2) && self.v.len() == self.u.len()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tight_i420_byte_len(width: u32, height: u32) -> Option<usize> {
|
||||
if !dims_ok(width, height) {
|
||||
return None;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
let y_len = w.checked_mul(h)?;
|
||||
let chroma_len = (w / 2).checked_mul(h / 2)?;
|
||||
y_len.checked_add(chroma_len.checked_mul(2)?)
|
||||
}
|
||||
|
||||
pub fn copy_tight_i420_into(src: &[u8], width: u32, height: u32, dst: &mut I420) -> bool {
|
||||
if !dst.has_layout(width, height) {
|
||||
return false;
|
||||
}
|
||||
let Some(total_len) = tight_i420_byte_len(width, height) else {
|
||||
return false;
|
||||
};
|
||||
if src.len() != total_len {
|
||||
return false;
|
||||
}
|
||||
let y_len = (width as usize) * (height as usize);
|
||||
let chroma_len = y_len / 4;
|
||||
dst.y.copy_from_slice(&src[..y_len]);
|
||||
dst.u.copy_from_slice(&src[y_len..y_len + chroma_len]);
|
||||
dst.v.copy_from_slice(&src[y_len + chroma_len..]);
|
||||
true
|
||||
}
|
||||
|
||||
fn dims_ok(width: u32, height: u32) -> bool {
|
||||
width >= 2 && height >= 2 && width.is_multiple_of(2) && height.is_multiple_of(2)
|
||||
}
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub fn nv12_to_i420(
|
||||
src: &[u8],
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_uv: u32,
|
||||
) -> Option<I420> {
|
||||
let mut dst = I420::new(width, height)?;
|
||||
if !nv12_to_i420_into(src, width, height, stride_y, stride_uv, &mut dst) {
|
||||
return None;
|
||||
}
|
||||
Some(dst)
|
||||
}
|
||||
|
||||
pub fn nv12_to_i420_into(
|
||||
src: &[u8],
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_uv: u32,
|
||||
dst: &mut I420,
|
||||
) -> bool {
|
||||
if !dims_ok(width, height) {
|
||||
return false;
|
||||
}
|
||||
if !dst.has_layout(width, height) {
|
||||
return false;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
let cw = w / 2;
|
||||
let ch = h / 2;
|
||||
let sy = stride_y.max(width) as usize;
|
||||
let suv = stride_uv.max(width) as usize;
|
||||
|
||||
let Some(uv_offset) = sy.checked_mul(h) else {
|
||||
return false;
|
||||
};
|
||||
let Some(uv_len) = suv.checked_mul(ch) else {
|
||||
return false;
|
||||
};
|
||||
let Some(needed) = uv_offset.checked_add(uv_len) else {
|
||||
return false;
|
||||
};
|
||||
if src.len() < needed {
|
||||
return false;
|
||||
}
|
||||
|
||||
for row in 0..h {
|
||||
let s = row * sy;
|
||||
dst.y[row * w..row * w + w].copy_from_slice(&src[s..s + w]);
|
||||
}
|
||||
|
||||
for row in 0..ch {
|
||||
let base = uv_offset + row * suv;
|
||||
for x in 0..cw {
|
||||
dst.u[row * cw + x] = src[base + 2 * x];
|
||||
dst.v[row * cw + x] = src[base + 2 * x + 1];
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn copy_nv12_planes(
|
||||
src: &[u8],
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_uv: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_uv: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_stride_uv: u32,
|
||||
) -> bool {
|
||||
if !dims_ok(width, height) {
|
||||
return false;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
let ch = h / 2;
|
||||
let sy = stride_y.max(width) as usize;
|
||||
let suv = stride_uv.max(width) as usize;
|
||||
let dsy = dst_stride_y as usize;
|
||||
let dsuv = dst_stride_uv as usize;
|
||||
if dsy < w || dsuv < w {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(uv_offset) = sy.checked_mul(h) else {
|
||||
return false;
|
||||
};
|
||||
let Some(uv_len) = suv.checked_mul(ch) else {
|
||||
return false;
|
||||
};
|
||||
let Some(needed) = uv_offset.checked_add(uv_len) else {
|
||||
return false;
|
||||
};
|
||||
if src.len() < needed || dst_y.len() < dsy * h || dst_uv.len() < dsuv * ch {
|
||||
return false;
|
||||
}
|
||||
|
||||
for row in 0..h {
|
||||
let s = row * sy;
|
||||
let d = row * dsy;
|
||||
dst_y[d..d + w].copy_from_slice(&src[s..s + w]);
|
||||
}
|
||||
for row in 0..ch {
|
||||
let s = uv_offset + row * suv;
|
||||
let d = row * dsuv;
|
||||
dst_uv[d..d + w].copy_from_slice(&src[s..s + w]);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn yuyv_to_i420(src: &[u8], width: u32, height: u32, stride: u32) -> Option<I420> {
|
||||
let mut dst = I420::new(width, height)?;
|
||||
if !yuyv_to_i420_into(src, width, height, stride, &mut dst) {
|
||||
return None;
|
||||
}
|
||||
Some(dst)
|
||||
}
|
||||
|
||||
pub fn yuyv_to_i420_into(src: &[u8], width: u32, height: u32, stride: u32, dst: &mut I420) -> bool {
|
||||
if !dims_ok(width, height) {
|
||||
return false;
|
||||
}
|
||||
if !dst.has_layout(width, height) {
|
||||
return false;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
let cw = w / 2;
|
||||
let ch = h / 2;
|
||||
let stride = stride.max(width * 2) as usize;
|
||||
if src.len() < stride * h {
|
||||
return false;
|
||||
}
|
||||
|
||||
for row in 0..h {
|
||||
let row_base = row * stride;
|
||||
for pair in 0..cw {
|
||||
let src_offset = row_base + pair * 4;
|
||||
let dst_offset = row * w + pair * 2;
|
||||
dst.y[dst_offset] = src[src_offset];
|
||||
dst.y[dst_offset + 1] = src[src_offset + 2];
|
||||
}
|
||||
}
|
||||
for cy in 0..ch {
|
||||
for cx in 0..cw {
|
||||
let top = (cy * 2) * stride + cx * 4;
|
||||
let bottom = (cy * 2 + 1) * stride + cx * 4;
|
||||
dst.u[cy * cw + cx] =
|
||||
((u16::from(src[top + 1]) + u16::from(src[bottom + 1])) / 2) as u8;
|
||||
dst.v[cy * cw + cx] =
|
||||
((u16::from(src[top + 3]) + u16::from(src[bottom + 3])) / 2) as u8;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
fn clamp_u8(value: i32) -> u8 {
|
||||
value.clamp(0, 255) as u8
|
||||
}
|
||||
|
||||
fn rgb_to_y(r: i32, g: i32, b: i32) -> u8 {
|
||||
clamp_u8(((66 * r + 129 * g + 25 * b + 128) >> 8) + 16)
|
||||
}
|
||||
fn rgb_to_u(r: i32, g: i32, b: i32) -> u8 {
|
||||
clamp_u8(((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128)
|
||||
}
|
||||
fn rgb_to_v(r: i32, g: i32, b: i32) -> u8 {
|
||||
clamp_u8(((112 * r - 94 * g - 18 * b + 128) >> 8) + 128)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn bgra_to_i420(src: &[u8], width: u32, height: u32, stride: u32) -> Option<I420> {
|
||||
let mut dst = I420::new(width, height)?;
|
||||
if !bgra_to_i420_planes(
|
||||
src,
|
||||
width,
|
||||
height,
|
||||
stride,
|
||||
&mut dst.y,
|
||||
&mut dst.u,
|
||||
&mut dst.v,
|
||||
width,
|
||||
width / 2,
|
||||
width / 2,
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
Some(dst)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn bgra_to_i420_planes(
|
||||
src: &[u8],
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_u: &mut [u8],
|
||||
dst_v: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_stride_u: u32,
|
||||
dst_stride_v: u32,
|
||||
) -> bool {
|
||||
if !dims_ok(width, height) {
|
||||
return false;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
let cw = w / 2;
|
||||
let ch = h / 2;
|
||||
let stride = stride.max(width * 4) as usize;
|
||||
if src.len() < stride * h {
|
||||
return false;
|
||||
}
|
||||
let dsy = dst_stride_y as usize;
|
||||
let dsu = dst_stride_u as usize;
|
||||
let dsv = dst_stride_v as usize;
|
||||
if dsy < w || dsu < cw || dsv < cw {
|
||||
return false;
|
||||
}
|
||||
if dst_y.len() < dsy * h || dst_u.len() < dsu * ch || dst_v.len() < dsv * ch {
|
||||
return false;
|
||||
}
|
||||
|
||||
let px = |row: usize, col: usize| -> (i32, i32, i32) {
|
||||
let o = row * stride + col * 4;
|
||||
let b = src[o] as i32;
|
||||
let g = src[o + 1] as i32;
|
||||
let r = src[o + 2] as i32;
|
||||
(r, g, b)
|
||||
};
|
||||
|
||||
for row in 0..h {
|
||||
for col in 0..w {
|
||||
let (r, g, b) = px(row, col);
|
||||
dst_y[row * dsy + col] = rgb_to_y(r, g, b);
|
||||
}
|
||||
}
|
||||
for cy in 0..ch {
|
||||
for cx in 0..cw {
|
||||
let mut rs = 0;
|
||||
let mut gs = 0;
|
||||
let mut bs = 0;
|
||||
for dy in 0..2 {
|
||||
for dx in 0..2 {
|
||||
let (r, g, b) = px(cy * 2 + dy, cx * 2 + dx);
|
||||
rs += r;
|
||||
gs += g;
|
||||
bs += b;
|
||||
}
|
||||
}
|
||||
let (r, g, b) = (rs / 4, gs / 4, bs / 4);
|
||||
dst_u[cy * dsu + cx] = rgb_to_u(r, g, b);
|
||||
dst_v[cy * dsv + cx] = rgb_to_v(r, g, b);
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg_attr(not(feature = "camera-native"), allow(dead_code))]
|
||||
pub fn rgb_to_i420(src: &[u8], width: u32, height: u32) -> Option<I420> {
|
||||
let mut dst = I420::new(width, height)?;
|
||||
if !rgb_to_i420_into(src, width, height, &mut dst) {
|
||||
return None;
|
||||
}
|
||||
Some(dst)
|
||||
}
|
||||
|
||||
pub fn rgb_to_i420_into(src: &[u8], width: u32, height: u32, dst: &mut I420) -> bool {
|
||||
if !dims_ok(width, height) {
|
||||
return false;
|
||||
}
|
||||
if !dst.has_layout(width, height) {
|
||||
return false;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
let cw = w / 2;
|
||||
let ch = h / 2;
|
||||
let stride = w * 3;
|
||||
if src.len() < stride * h {
|
||||
return false;
|
||||
}
|
||||
|
||||
let px = |row: usize, col: usize| -> (i32, i32, i32) {
|
||||
let o = row * stride + col * 3;
|
||||
let r = src[o] as i32;
|
||||
let g = src[o + 1] as i32;
|
||||
let b = src[o + 2] as i32;
|
||||
(r, g, b)
|
||||
};
|
||||
|
||||
for row in 0..h {
|
||||
for col in 0..w {
|
||||
let (r, g, b) = px(row, col);
|
||||
dst.y[row * w + col] = rgb_to_y(r, g, b);
|
||||
}
|
||||
}
|
||||
for cy in 0..ch {
|
||||
for cx in 0..cw {
|
||||
let mut rs = 0;
|
||||
let mut gs = 0;
|
||||
let mut bs = 0;
|
||||
for dy in 0..2 {
|
||||
for dx in 0..2 {
|
||||
let (r, g, b) = px(cy * 2 + dy, cx * 2 + dx);
|
||||
rs += r;
|
||||
gs += g;
|
||||
bs += b;
|
||||
}
|
||||
}
|
||||
let (r, g, b) = (rs / 4, gs / 4, bs / 4);
|
||||
dst.u[cy * cw + cx] = rgb_to_u(r, g, b);
|
||||
dst.v[cy * cw + cx] = rgb_to_v(r, g, b);
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg_attr(not(feature = "camera-native"), allow(dead_code))]
|
||||
pub fn bgr_to_i420_into(src: &[u8], width: u32, height: u32, dst: &mut I420) -> bool {
|
||||
if !dims_ok(width, height) {
|
||||
return false;
|
||||
}
|
||||
if !dst.has_layout(width, height) {
|
||||
return false;
|
||||
}
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
let cw = w / 2;
|
||||
let ch = h / 2;
|
||||
let stride = w * 3;
|
||||
if src.len() < stride * h {
|
||||
return false;
|
||||
}
|
||||
|
||||
let px = |row: usize, col: usize| -> (i32, i32, i32) {
|
||||
let o = row * stride + col * 3;
|
||||
let b = src[o] as i32;
|
||||
let g = src[o + 1] as i32;
|
||||
let r = src[o + 2] as i32;
|
||||
(r, g, b)
|
||||
};
|
||||
|
||||
for row in 0..h {
|
||||
for col in 0..w {
|
||||
let (r, g, b) = px(row, col);
|
||||
dst.y[row * w + col] = rgb_to_y(r, g, b);
|
||||
}
|
||||
}
|
||||
for cy in 0..ch {
|
||||
for cx in 0..cw {
|
||||
let mut rs = 0;
|
||||
let mut gs = 0;
|
||||
let mut bs = 0;
|
||||
for dy in 0..2 {
|
||||
for dx in 0..2 {
|
||||
let (r, g, b) = px(cy * 2 + dy, cx * 2 + dx);
|
||||
rs += r;
|
||||
gs += g;
|
||||
bs += b;
|
||||
}
|
||||
}
|
||||
let (r, g, b) = (rs / 4, gs / 4, bs / 4);
|
||||
dst.u[cy * cw + cx] = rgb_to_u(r, g, b);
|
||||
dst.v[cy * cw + cx] = rgb_to_v(r, g, b);
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rejects_odd_or_tiny_dimensions() {
|
||||
assert!(nv12_to_i420(&[0u8; 64], 3, 2, 3, 3).is_none());
|
||||
assert!(nv12_to_i420(&[0u8; 64], 2, 1, 2, 2).is_none());
|
||||
assert!(bgra_to_i420(&[0u8; 256], 2, 3, 8).is_none());
|
||||
assert!(bgra_to_i420(&[0u8; 256], 0, 2, 0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nv12_short_buffer_is_rejected() {
|
||||
assert!(nv12_to_i420(&[0u8; 5], 2, 2, 2, 2).is_none());
|
||||
assert!(nv12_to_i420(&[0u8; 6], 2, 2, 2, 2).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nv12_packed_2x2_deinterleaves() {
|
||||
let src = [1u8, 2, 3, 4, 10, 20];
|
||||
let out = nv12_to_i420(&src, 2, 2, 2, 2).unwrap();
|
||||
assert_eq!(out.y, vec![1, 2, 3, 4]);
|
||||
assert_eq!(out.u, vec![10]);
|
||||
assert_eq!(out.v, vec![20]);
|
||||
assert_eq!((out.width / 2, out.height / 2), (1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn yuyv_2x2_deinterleaves_and_vertically_averages_chroma() {
|
||||
let src = [1u8, 10, 2, 20, 3, 30, 4, 40];
|
||||
let out = yuyv_to_i420(&src, 2, 2, 4).unwrap();
|
||||
|
||||
assert_eq!(out.y, vec![1, 2, 3, 4]);
|
||||
assert_eq!(out.u, vec![20]);
|
||||
assert_eq!(out.v, vec![30]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn yuyv_respects_row_padding() {
|
||||
let src = [1u8, 10, 2, 20, 99, 99, 3, 30, 4, 40, 88, 88];
|
||||
let out = yuyv_to_i420(&src, 2, 2, 6).unwrap();
|
||||
|
||||
assert_eq!(out.y, vec![1, 2, 3, 4]);
|
||||
assert_eq!(out.u, vec![20]);
|
||||
assert_eq!(out.v, vec![30]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nv12_4x4_deinterleaves_two_chroma_columns() {
|
||||
let mut src = Vec::new();
|
||||
src.extend(0u8..16);
|
||||
src.extend([100, 101, 102, 103, 104, 105, 106, 107]);
|
||||
let out = nv12_to_i420(&src, 4, 4, 4, 4).unwrap();
|
||||
assert_eq!(out.y, (0u8..16).collect::<Vec<_>>());
|
||||
assert_eq!(out.u, vec![100, 102, 104, 106]);
|
||||
assert_eq!(out.v, vec![101, 103, 105, 107]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nv12_respects_row_padding() {
|
||||
let src = [1u8, 2, 0xFF, 0xFF, 3, 4, 0xFF, 0xFF, 10, 20, 0xFF, 0xFF];
|
||||
let out = nv12_to_i420(&src, 2, 2, 4, 4).unwrap();
|
||||
assert_eq!(out.y, vec![1, 2, 3, 4]);
|
||||
assert_eq!(out.u, vec![10]);
|
||||
assert_eq!(out.v, vec![20]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_nv12_planes_preserves_nv12_layout() {
|
||||
let src = [1u8, 2, 3, 4, 10, 20];
|
||||
let mut y = [0u8; 4];
|
||||
let mut uv = [0u8; 2];
|
||||
assert!(copy_nv12_planes(&src, 2, 2, 2, 2, &mut y, &mut uv, 2, 2));
|
||||
assert_eq!(y, [1, 2, 3, 4]);
|
||||
assert_eq!(uv, [10, 20]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_nv12_planes_respects_destination_stride() {
|
||||
let src = [1u8, 2, 0xFF, 0xFF, 3, 4, 0xFF, 0xFF, 10, 20, 0xFF, 0xFF];
|
||||
let mut y = [0u8; 8];
|
||||
let mut uv = [0u8; 4];
|
||||
assert!(copy_nv12_planes(&src, 2, 2, 4, 4, &mut y, &mut uv, 4, 4));
|
||||
assert_eq!(y, [1, 2, 0, 0, 3, 4, 0, 0]);
|
||||
assert_eq!(uv, [10, 20, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_nv12_planes_rejects_short_buffers() {
|
||||
let src = [0u8; 6];
|
||||
let mut y = [0u8; 3];
|
||||
let mut uv = [0u8; 2];
|
||||
assert!(!copy_nv12_planes(&src, 2, 2, 2, 2, &mut y, &mut uv, 2, 2));
|
||||
let mut y = [0u8; 4];
|
||||
assert!(!copy_nv12_planes(
|
||||
&src[..5],
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
&mut y,
|
||||
&mut uv,
|
||||
2,
|
||||
2
|
||||
));
|
||||
}
|
||||
|
||||
fn solid_bgra(width: u32, height: u32, b: u8, g: u8, r: u8) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity((width * height * 4) as usize);
|
||||
for _ in 0..(width * height) {
|
||||
v.extend([b, g, r, 255]);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
fn near(a: u8, b: u8, tol: i32) -> bool {
|
||||
(a as i32 - b as i32).abs() <= tol
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bgra_black_white_grey_levels() {
|
||||
let black = bgra_to_i420(&solid_bgra(2, 2, 0, 0, 0), 2, 2, 8).unwrap();
|
||||
assert!(near(black.y[0], 16, 1), "black Y={}", black.y[0]);
|
||||
assert!(near(black.u[0], 128, 1) && near(black.v[0], 128, 1));
|
||||
|
||||
let white = bgra_to_i420(&solid_bgra(2, 2, 255, 255, 255), 2, 2, 8).unwrap();
|
||||
assert!(near(white.y[0], 235, 2), "white Y={}", white.y[0]);
|
||||
assert!(near(white.u[0], 128, 2) && near(white.v[0], 128, 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bgra_primaries_have_expected_chroma_signs() {
|
||||
let red = bgra_to_i420(&solid_bgra(2, 2, 0, 0, 255), 2, 2, 8).unwrap();
|
||||
assert!(red.v[0] > 200, "red V={}", red.v[0]);
|
||||
let blue = bgra_to_i420(&solid_bgra(2, 2, 255, 0, 0), 2, 2, 8).unwrap();
|
||||
assert!(blue.u[0] > 200, "blue U={}", blue.u[0]);
|
||||
let green = bgra_to_i420(&solid_bgra(2, 2, 0, 255, 0), 2, 2, 8).unwrap();
|
||||
assert!(
|
||||
green.u[0] < 60 && green.v[0] < 60,
|
||||
"green U={} V={}",
|
||||
green.u[0],
|
||||
green.v[0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bgra_plane_sizes() {
|
||||
let out = bgra_to_i420(&solid_bgra(8, 6, 10, 20, 30), 8, 6, 32).unwrap();
|
||||
assert_eq!(out.y.len(), 8 * 6);
|
||||
assert_eq!(out.u.len(), 4 * 3);
|
||||
assert_eq!(out.v.len(), 4 * 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bgra_golden_2x2_solid_colour_exact_bytes() {
|
||||
let src = solid_bgra(2, 2, 32, 64, 128);
|
||||
let out = bgra_to_i420(&src, 2, 2, 8).unwrap();
|
||||
assert_eq!(out.width, 2);
|
||||
assert_eq!(out.height, 2);
|
||||
assert_eq!(out.y, vec![84, 84, 84, 84]);
|
||||
assert_eq!(out.u, vec![105]);
|
||||
assert_eq!(out.v, vec![158]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bgra_golden_strided_2x2_skips_row_padding() {
|
||||
let mut src = vec![0xFFu8; 16 * 2];
|
||||
for px in 0..2 {
|
||||
let o = px * 4;
|
||||
src[o..o + 4].copy_from_slice(&[0, 0, 0, 255]);
|
||||
}
|
||||
for px in 0..2 {
|
||||
let o = 16 + px * 4;
|
||||
src[o..o + 4].copy_from_slice(&[255, 255, 255, 255]);
|
||||
}
|
||||
let out = bgra_to_i420(&src, 2, 2, 16).unwrap();
|
||||
assert!(
|
||||
near(out.y[0], 16, 1) && near(out.y[1], 16, 1),
|
||||
"row0 Y={:?}",
|
||||
&out.y[0..2]
|
||||
);
|
||||
assert!(
|
||||
near(out.y[2], 235, 2) && near(out.y[3], 235, 2),
|
||||
"row1 Y={:?}",
|
||||
&out.y[2..4]
|
||||
);
|
||||
assert!(
|
||||
near(out.u[0], 128, 2) && near(out.v[0], 128, 2),
|
||||
"U={} V={}",
|
||||
out.u[0],
|
||||
out.v[0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bgra_to_i420_planes_matches_tight_conversion_with_destination_padding() {
|
||||
let src = solid_bgra(4, 2, 32, 64, 128);
|
||||
let tight = bgra_to_i420(&src, 4, 2, 16).unwrap();
|
||||
let mut y = [0u8; 10];
|
||||
let mut u = [0u8; 4];
|
||||
let mut v = [0u8; 4];
|
||||
assert!(bgra_to_i420_planes(
|
||||
&src, 4, 2, 16, &mut y, &mut u, &mut v, 5, 2, 2
|
||||
));
|
||||
assert_eq!(&y[0..4], &tight.y[0..4]);
|
||||
assert_eq!(&y[5..9], &tight.y[4..8]);
|
||||
assert_eq!(&u[0..2], &tight.u[0..2]);
|
||||
assert_eq!(&v[0..2], &tight.v[0..2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_tight_i420_into_reuses_existing_plane_storage() {
|
||||
let mut dst = I420::new(4, 2).unwrap();
|
||||
let ptrs = (dst.y.as_ptr(), dst.u.as_ptr(), dst.v.as_ptr());
|
||||
let src: Vec<u8> = (0u8..12).collect();
|
||||
|
||||
assert_eq!(tight_i420_byte_len(4, 2), Some(12));
|
||||
assert!(copy_tight_i420_into(&src, 4, 2, &mut dst));
|
||||
|
||||
assert_eq!(dst.y.as_ptr(), ptrs.0);
|
||||
assert_eq!(dst.u.as_ptr(), ptrs.1);
|
||||
assert_eq!(dst.v.as_ptr(), ptrs.2);
|
||||
assert_eq!(dst.y, vec![0, 1, 2, 3, 4, 5, 6, 7]);
|
||||
assert_eq!(dst.u, vec![8, 9]);
|
||||
assert_eq!(dst.v, vec![10, 11]);
|
||||
}
|
||||
|
||||
fn solid_rgb(width: u32, height: u32, r: u8, g: u8, b: u8) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity((width * height * 3) as usize);
|
||||
for _ in 0..(width * height) {
|
||||
v.extend([r, g, b]);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgb_rejects_odd_dims_and_short_buffer() {
|
||||
assert!(rgb_to_i420(&[0u8; 64], 3, 2).is_none());
|
||||
assert!(rgb_to_i420(&[0u8; 64], 2, 1).is_none());
|
||||
assert!(rgb_to_i420(&[0u8; 11], 2, 2).is_none());
|
||||
assert!(rgb_to_i420(&[0u8; 12], 2, 2).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgb_golden_2x2_solid_colour_matches_bgra_path() {
|
||||
let out = rgb_to_i420(&solid_rgb(2, 2, 128, 64, 32), 2, 2).unwrap();
|
||||
assert_eq!(out.width, 2);
|
||||
assert_eq!(out.height, 2);
|
||||
assert_eq!(out.y, vec![84, 84, 84, 84]);
|
||||
assert_eq!(out.u, vec![105]);
|
||||
assert_eq!(out.v, vec![158]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgb_plane_sizes_and_levels() {
|
||||
let black = rgb_to_i420(&solid_rgb(2, 2, 0, 0, 0), 2, 2).unwrap();
|
||||
assert!(near(black.y[0], 16, 1));
|
||||
assert!(near(black.u[0], 128, 1) && near(black.v[0], 128, 1));
|
||||
let white = rgb_to_i420(&solid_rgb(4, 4, 255, 255, 255), 4, 4).unwrap();
|
||||
assert_eq!(white.y.len(), 16);
|
||||
assert_eq!(white.u.len(), 4);
|
||||
assert_eq!(white.v.len(), 4);
|
||||
assert!(near(white.y[0], 235, 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nv12_golden_4x2_packed_to_i420() {
|
||||
let mut src = Vec::new();
|
||||
src.extend(0u8..8);
|
||||
src.extend([40, 41, 42, 43]);
|
||||
let out = nv12_to_i420(&src, 4, 2, 4, 4).unwrap();
|
||||
assert_eq!(out.width, 4);
|
||||
assert_eq!(out.height, 2);
|
||||
assert_eq!(out.y, (0u8..8).collect::<Vec<_>>());
|
||||
assert_eq!(out.u, vec![40, 42]);
|
||||
assert_eq!(out.v, vec![41, 43]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,520 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
const OUTBOUND: &str = include_str!("fixtures/outbound_events.json");
|
||||
const INBOUND: &str = include_str!("fixtures/inbound_events.json");
|
||||
const VIDEO_FRAME_META: &str = include_str!("fixtures/video_frame_meta.json");
|
||||
|
||||
const TRACK_KINDS: &[&str] = &["audio", "video"];
|
||||
const TRACK_SOURCES: &[&str] = &[
|
||||
"unknown",
|
||||
"camera",
|
||||
"microphone",
|
||||
"screen_share",
|
||||
"screen_share_audio",
|
||||
];
|
||||
const QUALITIES: &[&str] = &["excellent", "good", "poor", "lost"];
|
||||
const SUBSCRIPTION_STATUSES: &[&str] = &["desired", "subscribed", "unsubscribed"];
|
||||
|
||||
fn records(json: &str) -> Vec<(String, Value)> {
|
||||
let parsed: Value = serde_json::from_str(json).expect("fixture is valid JSON");
|
||||
parsed
|
||||
.as_array()
|
||||
.expect("fixture is a JSON array")
|
||||
.iter()
|
||||
.map(|rec| {
|
||||
let ty = rec
|
||||
.get("eventType")
|
||||
.and_then(Value::as_str)
|
||||
.expect("record has an eventType string")
|
||||
.to_string();
|
||||
let payload = rec.get("payload").expect("record has a payload").clone();
|
||||
(ty, payload)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn assert_exact_keys(event_type: &str, payload: &Value, expected: &[&str]) {
|
||||
let obj = payload
|
||||
.as_object()
|
||||
.unwrap_or_else(|| panic!("{event_type}: payload is not a JSON object"));
|
||||
let mut got: Vec<&str> = obj.keys().map(String::as_str).collect();
|
||||
got.sort_unstable();
|
||||
let mut want: Vec<&str> = expected.to_vec();
|
||||
want.sort_unstable();
|
||||
assert_eq!(got, want, "{event_type}: payload keys mismatch");
|
||||
}
|
||||
|
||||
fn str_field<'a>(event_type: &str, payload: &'a Value, key: &str) -> &'a str {
|
||||
payload
|
||||
.get(key)
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_else(|| panic!("{event_type}.{key} is not a string"))
|
||||
}
|
||||
|
||||
fn bool_field(event_type: &str, payload: &Value, key: &str) -> bool {
|
||||
payload
|
||||
.get(key)
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or_else(|| panic!("{event_type}.{key} is not a boolean"))
|
||||
}
|
||||
|
||||
fn validate_string_map(event_type: &str, payload: &Value, key: &str) {
|
||||
let obj = payload
|
||||
.get(key)
|
||||
.and_then(Value::as_object)
|
||||
.unwrap_or_else(|| panic!("{event_type}.{key} is not an object"));
|
||||
for (map_key, value) in obj {
|
||||
assert!(
|
||||
!map_key.is_empty(),
|
||||
"{event_type}.{key} contains an empty key"
|
||||
);
|
||||
assert!(
|
||||
value.as_str().is_some(),
|
||||
"{event_type}.{key}.{map_key} is not a string"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_participant_snapshot(event_type: &str, payload: &Value) {
|
||||
assert_exact_keys(event_type, payload, &["sid", "identity", "name"]);
|
||||
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "identity");
|
||||
let _ = str_field(event_type, payload, "name");
|
||||
}
|
||||
|
||||
fn validate_track_payload(event_type: &str, payload: &Value, includes_subscription: bool) {
|
||||
let mut keys = vec![
|
||||
"participantSid",
|
||||
"identity",
|
||||
"participantName",
|
||||
"trackSid",
|
||||
"trackName",
|
||||
"kind",
|
||||
"source",
|
||||
"muted",
|
||||
];
|
||||
if includes_subscription {
|
||||
keys.extend(["subscribed", "subscriptionStatus"]);
|
||||
}
|
||||
assert_exact_keys(event_type, payload, &keys);
|
||||
assert!(str_field(event_type, payload, "participantSid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "identity");
|
||||
let _ = str_field(event_type, payload, "participantName");
|
||||
assert!(str_field(event_type, payload, "trackSid").starts_with("TR_"));
|
||||
let _ = str_field(event_type, payload, "trackName");
|
||||
assert!(TRACK_KINDS.contains(&str_field(event_type, payload, "kind")));
|
||||
assert!(TRACK_SOURCES.contains(&str_field(event_type, payload, "source")));
|
||||
let _ = bool_field(event_type, payload, "muted");
|
||||
if includes_subscription {
|
||||
let _ = bool_field(event_type, payload, "subscribed");
|
||||
assert!(SUBSCRIPTION_STATUSES.contains(&str_field(
|
||||
event_type,
|
||||
payload,
|
||||
"subscriptionStatus"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_record(event_type: &str, payload: &Value) {
|
||||
match event_type {
|
||||
"connected" | "Reconnecting" | "Reconnected" => assert_exact_keys(event_type, payload, &[]),
|
||||
"connectionState" => {
|
||||
assert_exact_keys(event_type, payload, &["state"]);
|
||||
assert!(!str_field(event_type, payload, "state").is_empty());
|
||||
}
|
||||
"disconnected" => {
|
||||
assert_exact_keys(event_type, payload, &["reason"]);
|
||||
assert!(!str_field(event_type, payload, "reason").is_empty());
|
||||
}
|
||||
"participantJoined" => {
|
||||
assert_exact_keys(event_type, payload, &["sid", "identity", "name"]);
|
||||
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "identity");
|
||||
let _ = str_field(event_type, payload, "name");
|
||||
}
|
||||
"participantLeft" => {
|
||||
assert_exact_keys(event_type, payload, &["sid", "identity", "name"]);
|
||||
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "name");
|
||||
}
|
||||
"trackPublished" | "trackUnpublished" | "trackSubscribed" | "trackUnsubscribed" => {
|
||||
validate_track_payload(event_type, payload, true);
|
||||
}
|
||||
"trackMuted" | "trackUnmuted" => {
|
||||
validate_track_payload(event_type, payload, false);
|
||||
assert_eq!(
|
||||
bool_field(event_type, payload, "muted"),
|
||||
event_type == "trackMuted"
|
||||
);
|
||||
}
|
||||
"trackSubscriptionFailed" => {
|
||||
let has_publication = payload.get("kind").is_some();
|
||||
if has_publication {
|
||||
assert_exact_keys(
|
||||
event_type,
|
||||
payload,
|
||||
&[
|
||||
"participantSid",
|
||||
"identity",
|
||||
"participantName",
|
||||
"trackSid",
|
||||
"trackName",
|
||||
"kind",
|
||||
"source",
|
||||
"muted",
|
||||
"subscribed",
|
||||
"subscriptionStatus",
|
||||
"error",
|
||||
],
|
||||
);
|
||||
} else {
|
||||
assert_exact_keys(
|
||||
event_type,
|
||||
payload,
|
||||
&[
|
||||
"participantSid",
|
||||
"identity",
|
||||
"participantName",
|
||||
"trackSid",
|
||||
"error",
|
||||
],
|
||||
);
|
||||
}
|
||||
assert!(str_field(event_type, payload, "participantSid").starts_with("PA_"));
|
||||
assert!(str_field(event_type, payload, "trackSid").starts_with("TR_"));
|
||||
if has_publication {
|
||||
assert!(TRACK_KINDS.contains(&str_field(event_type, payload, "kind")));
|
||||
assert!(TRACK_SOURCES.contains(&str_field(event_type, payload, "source")));
|
||||
assert!(SUBSCRIPTION_STATUSES.contains(&str_field(
|
||||
event_type,
|
||||
payload,
|
||||
"subscriptionStatus"
|
||||
)));
|
||||
let _ = bool_field(event_type, payload, "muted");
|
||||
let _ = bool_field(event_type, payload, "subscribed");
|
||||
}
|
||||
assert!(!str_field(event_type, payload, "error").is_empty());
|
||||
}
|
||||
"localTrackPublished" => {
|
||||
validate_track_payload(event_type, payload, false);
|
||||
}
|
||||
"localTrackUnpublished" => {
|
||||
validate_track_payload(event_type, payload, false);
|
||||
}
|
||||
"localTrackRepublished" => {
|
||||
let keys = [
|
||||
"participantSid",
|
||||
"identity",
|
||||
"participantName",
|
||||
"previousTrackSid",
|
||||
"trackSid",
|
||||
"trackName",
|
||||
"kind",
|
||||
"source",
|
||||
"muted",
|
||||
];
|
||||
assert_exact_keys(event_type, payload, &keys);
|
||||
assert!(str_field(event_type, payload, "participantSid").starts_with("PA_"));
|
||||
assert!(str_field(event_type, payload, "previousTrackSid").starts_with("TR_"));
|
||||
assert!(str_field(event_type, payload, "trackSid").starts_with("TR_"));
|
||||
assert!(TRACK_KINDS.contains(&str_field(event_type, payload, "kind")));
|
||||
assert!(TRACK_SOURCES.contains(&str_field(event_type, payload, "source")));
|
||||
let _ = bool_field(event_type, payload, "muted");
|
||||
}
|
||||
"activeSpeakers" => {
|
||||
assert_exact_keys(event_type, payload, &["sids", "participants"]);
|
||||
let sids = payload
|
||||
.get("sids")
|
||||
.and_then(Value::as_array)
|
||||
.expect("activeSpeakers.sids is an array");
|
||||
for sid in sids {
|
||||
let sid = sid.as_str().expect("activeSpeakers.sids entry is a string");
|
||||
assert!(
|
||||
sid.starts_with("PA_"),
|
||||
"activeSpeakers sid {sid} lacks PA_ prefix"
|
||||
);
|
||||
}
|
||||
let participants = payload
|
||||
.get("participants")
|
||||
.and_then(Value::as_array)
|
||||
.expect("activeSpeakers.participants is an array");
|
||||
for participant in participants {
|
||||
validate_participant_snapshot("activeSpeakers.participants[]", participant);
|
||||
}
|
||||
}
|
||||
"connectionQuality" => {
|
||||
assert_exact_keys(event_type, payload, &["sid", "identity", "name", "quality"]);
|
||||
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "identity");
|
||||
let _ = str_field(event_type, payload, "name");
|
||||
assert!(QUALITIES.contains(&str_field(event_type, payload, "quality")));
|
||||
}
|
||||
"e2eeState" => {
|
||||
assert_exact_keys(event_type, payload, &["sid", "identity", "name", "state"]);
|
||||
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "identity");
|
||||
let _ = str_field(event_type, payload, "name");
|
||||
assert!(!str_field(event_type, payload, "state").is_empty());
|
||||
}
|
||||
"stats" => {
|
||||
assert_exact_keys(event_type, payload, &["rttMs", "outbound", "inbound"]);
|
||||
let rtt = payload.get("rttMs").expect("stats.rttMs exists");
|
||||
assert!(
|
||||
rtt.is_null() || rtt.as_f64().is_some(),
|
||||
"stats.rttMs is null or number"
|
||||
);
|
||||
let outbound = payload
|
||||
.get("outbound")
|
||||
.and_then(Value::as_array)
|
||||
.expect("stats.outbound is an array");
|
||||
for entry in outbound {
|
||||
let obj = entry
|
||||
.as_object()
|
||||
.expect("stats.outbound[] is a JSON object");
|
||||
assert!(obj.contains_key("trackSid"));
|
||||
assert!(TRACK_SOURCES.contains(&str_field("stats.outbound[]", entry, "source")));
|
||||
assert!(TRACK_KINDS.contains(&str_field("stats.outbound[]", entry, "kind")));
|
||||
assert!(entry.get("bitrateKbps").and_then(Value::as_f64).is_some());
|
||||
assert!(entry.get("packetsLost").and_then(Value::as_u64).is_some());
|
||||
}
|
||||
let inbound = payload
|
||||
.get("inbound")
|
||||
.and_then(Value::as_array)
|
||||
.expect("stats.inbound is an array");
|
||||
for entry in inbound {
|
||||
let obj = entry.as_object().expect("stats.inbound[] is a JSON object");
|
||||
assert!(obj.contains_key("participantSid"));
|
||||
assert!(obj.contains_key("trackSid"));
|
||||
assert!(TRACK_KINDS.contains(&str_field("stats.inbound[]", entry, "kind")));
|
||||
assert!(entry.get("bitrateKbps").and_then(Value::as_f64).is_some());
|
||||
assert!(entry.get("packetsLost").and_then(Value::as_u64).is_some());
|
||||
}
|
||||
}
|
||||
"audioPlaybackUnavailable" => {
|
||||
assert_exact_keys(event_type, payload, &["message"]);
|
||||
assert!(!str_field(event_type, payload, "message").is_empty());
|
||||
}
|
||||
"participantNameChanged" => {
|
||||
assert_exact_keys(event_type, payload, &["sid", "identity", "oldName", "name"]);
|
||||
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "identity");
|
||||
let _ = str_field(event_type, payload, "oldName");
|
||||
let _ = str_field(event_type, payload, "name");
|
||||
}
|
||||
"participantMetadataChanged" => {
|
||||
assert_exact_keys(
|
||||
event_type,
|
||||
payload,
|
||||
&[
|
||||
"sid",
|
||||
"identity",
|
||||
"name",
|
||||
"oldMetadata",
|
||||
"metadata",
|
||||
"attributes",
|
||||
],
|
||||
);
|
||||
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "identity");
|
||||
let _ = str_field(event_type, payload, "name");
|
||||
let _ = str_field(event_type, payload, "oldMetadata");
|
||||
let _ = str_field(event_type, payload, "metadata");
|
||||
validate_string_map(event_type, payload, "attributes");
|
||||
}
|
||||
"participantAttributesChanged" => {
|
||||
assert_exact_keys(
|
||||
event_type,
|
||||
payload,
|
||||
&["sid", "identity", "name", "attributes", "changedAttributes"],
|
||||
);
|
||||
assert!(str_field(event_type, payload, "sid").starts_with("PA_"));
|
||||
let _ = str_field(event_type, payload, "identity");
|
||||
let _ = str_field(event_type, payload, "name");
|
||||
validate_string_map(event_type, payload, "attributes");
|
||||
validate_string_map(event_type, payload, "changedAttributes");
|
||||
}
|
||||
other => panic!("unknown eventType in fixture: {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_fixtures_match_contract() {
|
||||
let recs = records(OUTBOUND);
|
||||
assert!(!recs.is_empty(), "outbound fixture is non-empty");
|
||||
for (ty, payload) in &recs {
|
||||
validate_record(ty, payload);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inbound_fixtures_match_contract() {
|
||||
let recs = records(INBOUND);
|
||||
assert!(!recs.is_empty(), "inbound fixture is non-empty");
|
||||
for (ty, payload) in &recs {
|
||||
validate_record(ty, payload);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_covers_the_single_identity_publish_path() {
|
||||
let types: Vec<String> = records(OUTBOUND).into_iter().map(|(ty, _)| ty).collect();
|
||||
for expected in [
|
||||
"connected",
|
||||
"connectionState",
|
||||
"Reconnecting",
|
||||
"Reconnected",
|
||||
"localTrackPublished",
|
||||
"localTrackUnpublished",
|
||||
"localTrackRepublished",
|
||||
"e2eeState",
|
||||
"activeSpeakers",
|
||||
"stats",
|
||||
"audioPlaybackUnavailable",
|
||||
] {
|
||||
assert!(
|
||||
types.iter().any(|t| t == expected),
|
||||
"outbound missing {expected}"
|
||||
);
|
||||
}
|
||||
let published: Vec<(String, String)> = records(OUTBOUND)
|
||||
.into_iter()
|
||||
.filter(|(ty, _)| ty == "localTrackPublished")
|
||||
.map(|(_, p)| {
|
||||
(
|
||||
p.get("kind").and_then(Value::as_str).unwrap().to_string(),
|
||||
p.get("source").and_then(Value::as_str).unwrap().to_string(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert!(published.contains(&("video".into(), "screen_share".into())));
|
||||
assert!(published.contains(&("audio".into(), "microphone".into())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inbound_covers_subscribe_and_lifecycle() {
|
||||
let types: Vec<String> = records(INBOUND).into_iter().map(|(ty, _)| ty).collect();
|
||||
for expected in [
|
||||
"participantJoined",
|
||||
"participantNameChanged",
|
||||
"participantMetadataChanged",
|
||||
"participantAttributesChanged",
|
||||
"trackPublished",
|
||||
"trackSubscribed",
|
||||
"trackMuted",
|
||||
"trackUnmuted",
|
||||
"trackSubscriptionFailed",
|
||||
"trackUnsubscribed",
|
||||
"trackUnpublished",
|
||||
"e2eeState",
|
||||
"activeSpeakers",
|
||||
"connectionQuality",
|
||||
"participantLeft",
|
||||
] {
|
||||
assert!(
|
||||
types.iter().any(|t| t == expected),
|
||||
"inbound missing {expected}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_frame_meta_matches_contract() {
|
||||
let parsed: Value = serde_json::from_str(VIDEO_FRAME_META).expect("fixture is valid JSON");
|
||||
let recs = parsed.as_array().expect("video_frame_meta is a JSON array");
|
||||
assert!(!recs.is_empty(), "video_frame_meta fixture is non-empty");
|
||||
for meta in recs {
|
||||
let obj = meta.as_object().expect("meta is a JSON object");
|
||||
let mut got: Vec<&str> = obj.keys().map(String::as_str).collect();
|
||||
got.sort_unstable();
|
||||
assert_eq!(
|
||||
got,
|
||||
vec![
|
||||
"bridgeVersion",
|
||||
"height",
|
||||
"participantSid",
|
||||
"source",
|
||||
"timestampUs",
|
||||
"trackName",
|
||||
"trackSid",
|
||||
"width"
|
||||
],
|
||||
"video_frame_meta key set mismatch"
|
||||
);
|
||||
let bridge_version = obj
|
||||
.get("bridgeVersion")
|
||||
.and_then(Value::as_u64)
|
||||
.expect("bridgeVersion is an unsigned integer");
|
||||
assert!(
|
||||
bridge_version >= 1,
|
||||
"bridgeVersion {bridge_version} must be positive"
|
||||
);
|
||||
let participant_sid = obj
|
||||
.get("participantSid")
|
||||
.and_then(Value::as_str)
|
||||
.expect("participantSid is a string");
|
||||
let track_sid = obj
|
||||
.get("trackSid")
|
||||
.and_then(Value::as_str)
|
||||
.expect("trackSid is a string");
|
||||
let track_name = obj
|
||||
.get("trackName")
|
||||
.and_then(Value::as_str)
|
||||
.expect("trackName is a string");
|
||||
let source = obj
|
||||
.get("source")
|
||||
.and_then(Value::as_str)
|
||||
.expect("source is a string");
|
||||
assert!(
|
||||
participant_sid.starts_with("PA_"),
|
||||
"participantSid {participant_sid} lacks PA_"
|
||||
);
|
||||
assert!(
|
||||
track_sid.starts_with("TR_"),
|
||||
"trackSid {track_sid} lacks TR_"
|
||||
);
|
||||
assert!(!track_name.is_empty(), "trackName is not empty");
|
||||
assert!(
|
||||
TRACK_SOURCES.contains(&source),
|
||||
"source {source} is a known LiveKit source"
|
||||
);
|
||||
let width = obj
|
||||
.get("width")
|
||||
.and_then(Value::as_u64)
|
||||
.expect("width is an integer");
|
||||
let height = obj
|
||||
.get("height")
|
||||
.and_then(Value::as_u64)
|
||||
.expect("height is an integer");
|
||||
assert!(
|
||||
width >= 2 && width % 2 == 0,
|
||||
"width {width} must be even and >= 2"
|
||||
);
|
||||
assert!(
|
||||
height >= 2 && height % 2 == 0,
|
||||
"height {height} must be even and >= 2"
|
||||
);
|
||||
assert!(
|
||||
obj.get("timestampUs").and_then(Value::as_i64).is_some(),
|
||||
"timestampUs is an i64-range integer"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
recs.iter()
|
||||
.filter_map(|m| m.get("timestampUs").and_then(Value::as_i64))
|
||||
.any(|ts| ts > u32::MAX as i64),
|
||||
"expect a >2^32 timestampUs in the fixture to lock the i64 contract"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn e2ee_state_is_ok_on_the_wire() {
|
||||
for json in [OUTBOUND, INBOUND] {
|
||||
for (ty, payload) in records(json) {
|
||||
if ty == "e2eeState" {
|
||||
assert_eq!(payload.get("state").and_then(Value::as_str), Some("ok"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
[
|
||||
{
|
||||
"eventType": "participantJoined",
|
||||
"payload": {
|
||||
"sid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"name": "Test User B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "participantNameChanged",
|
||||
"payload": {
|
||||
"sid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"oldName": "Test User",
|
||||
"name": "Test User B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "participantMetadataChanged",
|
||||
"payload": {
|
||||
"sid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"name": "Test User B",
|
||||
"oldMetadata": "",
|
||||
"metadata": "{\"status\":\"presenting\"}",
|
||||
"attributes": {
|
||||
"role": "speaker"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "participantAttributesChanged",
|
||||
"payload": {
|
||||
"sid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"name": "Test User B",
|
||||
"attributes": {
|
||||
"role": "speaker",
|
||||
"hand": "raised"
|
||||
},
|
||||
"changedAttributes": {
|
||||
"hand": "raised"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackPublished",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteVideo01",
|
||||
"trackName": "remote-screen",
|
||||
"kind": "video",
|
||||
"source": "screen_share",
|
||||
"muted": false,
|
||||
"subscribed": false,
|
||||
"subscriptionStatus": "desired"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackSubscribed",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteVideo01",
|
||||
"trackName": "remote-screen",
|
||||
"kind": "video",
|
||||
"source": "screen_share",
|
||||
"muted": false,
|
||||
"subscribed": true,
|
||||
"subscriptionStatus": "subscribed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackPublished",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteAudio01",
|
||||
"trackName": "remote-microphone",
|
||||
"kind": "audio",
|
||||
"source": "microphone",
|
||||
"muted": false,
|
||||
"subscribed": false,
|
||||
"subscriptionStatus": "desired"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackSubscribed",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteAudio01",
|
||||
"trackName": "remote-microphone",
|
||||
"kind": "audio",
|
||||
"source": "microphone",
|
||||
"muted": false,
|
||||
"subscribed": true,
|
||||
"subscriptionStatus": "subscribed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackMuted",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteAudio01",
|
||||
"trackName": "remote-microphone",
|
||||
"kind": "audio",
|
||||
"source": "microphone",
|
||||
"muted": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackUnmuted",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteAudio01",
|
||||
"trackName": "remote-microphone",
|
||||
"kind": "audio",
|
||||
"source": "microphone",
|
||||
"muted": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackSubscriptionFailed",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteCamera01",
|
||||
"trackName": "remote-camera",
|
||||
"kind": "video",
|
||||
"source": "camera",
|
||||
"muted": false,
|
||||
"subscribed": false,
|
||||
"subscriptionStatus": "desired",
|
||||
"error": "could not find published track with sid: \"TR_remoteCamera01\""
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackSubscriptionFailed",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_missingPublication01",
|
||||
"error": "could not find published track with sid: \"TR_missingPublication01\""
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "e2eeState",
|
||||
"payload": {
|
||||
"sid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"name": "Test User B",
|
||||
"state": "ok"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "activeSpeakers",
|
||||
"payload": {
|
||||
"sids": ["PA_remoteB12345"],
|
||||
"participants": [
|
||||
{
|
||||
"sid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"name": "Test User B"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "connectionQuality",
|
||||
"payload": {
|
||||
"sid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"name": "Test User B",
|
||||
"quality": "excellent"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackUnsubscribed",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteVideo01",
|
||||
"trackName": "remote-screen",
|
||||
"kind": "video",
|
||||
"source": "screen_share",
|
||||
"muted": false,
|
||||
"subscribed": false,
|
||||
"subscriptionStatus": "unsubscribed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "trackUnpublished",
|
||||
"payload": {
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"participantName": "Test User B",
|
||||
"trackSid": "TR_remoteVideo01",
|
||||
"trackName": "remote-screen",
|
||||
"kind": "video",
|
||||
"source": "screen_share",
|
||||
"muted": false,
|
||||
"subscribed": false,
|
||||
"subscriptionStatus": "unsubscribed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "participantLeft",
|
||||
"payload": {
|
||||
"sid": "PA_remoteB12345",
|
||||
"identity": "1428486264293441600",
|
||||
"name": "Test User B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "disconnected",
|
||||
"payload": {
|
||||
"reason": "clientinitiated"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,134 @@
|
||||
[
|
||||
{
|
||||
"eventType": "connected",
|
||||
"payload": {}
|
||||
},
|
||||
{
|
||||
"eventType": "connectionState",
|
||||
"payload": {
|
||||
"state": "connected"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "Reconnecting",
|
||||
"payload": {}
|
||||
},
|
||||
{
|
||||
"eventType": "Reconnected",
|
||||
"payload": {}
|
||||
},
|
||||
{
|
||||
"eventType": "localTrackPublished",
|
||||
"payload": {
|
||||
"participantSid": "PA_77jesWWtSMge",
|
||||
"identity": "1428486264293441599",
|
||||
"participantName": "Test User A",
|
||||
"trackSid": "TR_VShJ9y2A49ai6i",
|
||||
"trackName": "local-screen",
|
||||
"kind": "video",
|
||||
"source": "screen_share",
|
||||
"muted": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "localTrackPublished",
|
||||
"payload": {
|
||||
"participantSid": "PA_77jesWWtSMge",
|
||||
"identity": "1428486264293441599",
|
||||
"participantName": "Test User A",
|
||||
"trackSid": "TR_AMannbg6PLtcVz",
|
||||
"trackName": "local-microphone",
|
||||
"kind": "audio",
|
||||
"source": "microphone",
|
||||
"muted": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "localTrackUnpublished",
|
||||
"payload": {
|
||||
"participantSid": "PA_77jesWWtSMge",
|
||||
"identity": "1428486264293441599",
|
||||
"participantName": "Test User A",
|
||||
"trackSid": "TR_AMannbg6PLtcVz",
|
||||
"trackName": "local-microphone",
|
||||
"kind": "audio",
|
||||
"source": "microphone",
|
||||
"muted": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "localTrackRepublished",
|
||||
"payload": {
|
||||
"participantSid": "PA_77jesWWtSMge",
|
||||
"identity": "1428486264293441599",
|
||||
"participantName": "Test User A",
|
||||
"previousTrackSid": "TR_VShJ9y2A49ai6i",
|
||||
"trackSid": "TR_VSsTf93n5GqJrK",
|
||||
"trackName": "local-screen",
|
||||
"kind": "video",
|
||||
"source": "screen_share",
|
||||
"muted": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "e2eeState",
|
||||
"payload": {
|
||||
"sid": "PA_77jesWWtSMge",
|
||||
"identity": "1428486264293441599",
|
||||
"name": "Test User A",
|
||||
"state": "ok"
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "activeSpeakers",
|
||||
"payload": {
|
||||
"sids": ["PA_77jesWWtSMge"],
|
||||
"participants": [
|
||||
{
|
||||
"sid": "PA_77jesWWtSMge",
|
||||
"identity": "1428486264293441599",
|
||||
"name": "Test User A"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "stats",
|
||||
"payload": {
|
||||
"rttMs": 24.5,
|
||||
"outbound": [
|
||||
{
|
||||
"trackSid": "TR_VShJ9y2A49ai6i",
|
||||
"source": "screen_share",
|
||||
"kind": "video",
|
||||
"bitrateKbps": 2500,
|
||||
"packetsLost": 0,
|
||||
"fps": 60
|
||||
},
|
||||
{
|
||||
"trackSid": "TR_AMannbg6PLtcVz",
|
||||
"source": "microphone",
|
||||
"kind": "audio",
|
||||
"bitrateKbps": 48,
|
||||
"packetsLost": 1
|
||||
}
|
||||
],
|
||||
"inbound": [
|
||||
{
|
||||
"participantSid": "PA_Subscriber",
|
||||
"trackSid": "TR_RemoteVideo",
|
||||
"kind": "video",
|
||||
"bitrateKbps": 1200,
|
||||
"packetsLost": 0,
|
||||
"jitterMs": 3.5
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"eventType": "audioPlaybackUnavailable",
|
||||
"payload": {
|
||||
"message": "platform audio unavailable: test fixture"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,32 @@
|
||||
[
|
||||
{
|
||||
"bridgeVersion": 14,
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"trackSid": "TR_remoteVideo01",
|
||||
"trackName": "screen",
|
||||
"source": "screen_share",
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"timestampUs": 123456789
|
||||
},
|
||||
{
|
||||
"bridgeVersion": 14,
|
||||
"participantSid": "PA_77jesWWtSMge",
|
||||
"trackSid": "TR_VShJ9y2A49ai6i",
|
||||
"trackName": "camera",
|
||||
"source": "camera",
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"timestampUs": 0
|
||||
},
|
||||
{
|
||||
"bridgeVersion": 14,
|
||||
"participantSid": "PA_remoteB12345",
|
||||
"trackSid": "TR_remoteVideo02",
|
||||
"trackName": "screen",
|
||||
"source": "screen_share",
|
||||
"width": 640,
|
||||
"height": 360,
|
||||
"timestampUs": 4294967296
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
{"v":1}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"git": {
|
||||
"sha1": "ba0c31290ad6f7836ef96efd62167c3501ed07fa"
|
||||
},
|
||||
"path_in_vcs": "libwebrtc"
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.3.26](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.25...rust-sdks/libwebrtc@0.3.26) - 2026-02-16
|
||||
|
||||
### Other
|
||||
|
||||
- add is_screencast to VideoSource ([#896](https://github.com/livekit/rust-sdks/pull/896))
|
||||
|
||||
## [0.3.25](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.24...rust-sdks/libwebrtc@0.3.25) - 2026-02-09
|
||||
|
||||
### Fixed
|
||||
|
||||
- fix the 440->441 samples issue and pass a noop callback for release ([#848](https://github.com/livekit/rust-sdks/pull/848))
|
||||
|
||||
### Other
|
||||
|
||||
- Use workspace dependencies & settings ([#856](https://github.com/livekit/rust-sdks/pull/856))
|
||||
- allow apm >=10ms frames ([#843](https://github.com/livekit/rust-sdks/pull/843))
|
||||
|
||||
## [0.3.24](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.23...rust-sdks/libwebrtc@0.3.24) - 2026-01-15
|
||||
|
||||
### Other
|
||||
|
||||
- updated the following local packages: webrtc-sys
|
||||
|
||||
## [0.3.23](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.22...rust-sdks/libwebrtc@0.3.23) - 2025-12-19
|
||||
|
||||
### Fixed
|
||||
|
||||
- Exclude the desktop-capturer module link for mobile. ([#817](https://github.com/livekit/rust-sdks/pull/817))
|
||||
|
||||
## [0.3.22](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.21...rust-sdks/libwebrtc@0.3.22) - 2025-12-17
|
||||
|
||||
### Other
|
||||
|
||||
- Expose WebRTC's audio_mixer ([#806](https://github.com/livekit/rust-sdks/pull/806))
|
||||
|
||||
## [0.3.21](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.20...rust-sdks/libwebrtc@0.3.21) - 2025-12-04
|
||||
|
||||
### Other
|
||||
|
||||
- move starting/stopping GLib event loop into libwebrtc crate ([#798](https://github.com/livekit/rust-sdks/pull/798))
|
||||
- Expose desktop capturer ([#725](https://github.com/livekit/rust-sdks/pull/725))
|
||||
|
||||
## [0.3.20](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.19...rust-sdks/libwebrtc@0.3.20) - 2025-11-20
|
||||
|
||||
### Other
|
||||
|
||||
- Fix the fast path in capture_frame function, without buffering ([#778](https://github.com/livekit/rust-sdks/pull/778))
|
||||
|
||||
## [0.3.19](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.18...rust-sdks/libwebrtc@0.3.19) - 2025-10-27
|
||||
|
||||
### Other
|
||||
|
||||
- updated the following local packages: webrtc-sys
|
||||
|
||||
## [0.3.18](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.17...rust-sdks/libwebrtc@0.3.18) - 2025-10-22
|
||||
|
||||
### Other
|
||||
|
||||
- License check ([#746](https://github.com/livekit/rust-sdks/pull/746))
|
||||
|
||||
## [0.3.17](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.16...rust-sdks/libwebrtc@0.3.17) - 2025-10-13
|
||||
|
||||
### Added
|
||||
|
||||
- *(e2ee)* add data channel encryption ([#708](https://github.com/livekit/rust-sdks/pull/708))
|
||||
|
||||
### Other
|
||||
|
||||
- Enable buffer scaling ([#473](https://github.com/livekit/rust-sdks/pull/473))
|
||||
|
||||
## [0.3.16](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.15...rust-sdks/libwebrtc@0.3.16) - 2025-10-03
|
||||
|
||||
### Other
|
||||
|
||||
- updated the following local packages: webrtc-sys
|
||||
|
||||
## [0.3.15](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.14...rust-sdks/libwebrtc@0.3.15) - 2025-09-29
|
||||
|
||||
### Fixed
|
||||
|
||||
- fix Builds/E2E Tests CI. ([#715](https://github.com/livekit/rust-sdks/pull/715))
|
||||
|
||||
## [0.3.14](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.13...rust-sdks/libwebrtc@0.3.14) - 2025-09-09
|
||||
|
||||
### Other
|
||||
|
||||
- updated the following local packages: webrtc-sys
|
||||
|
||||
## [0.3.13](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.12...rust-sdks/libwebrtc@0.3.13) - 2025-09-03
|
||||
|
||||
### Other
|
||||
|
||||
- updated the following local packages: webrtc-sys
|
||||
# Changelog
|
||||
|
||||
## [0.3.12](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.11...rust-sdks/libwebrtc@0.3.12) - 2025-06-17
|
||||
|
||||
### Other
|
||||
|
||||
- updated the following local packages: livekit-protocol, webrtc-sys
|
||||
|
||||
## [0.3.11](https://github.com/livekit/rust-sdks/compare/rust-sdks/libwebrtc@0.3.10...rust-sdks/libwebrtc@0.3.11) - 2025-06-11
|
||||
|
||||
### Fixed
|
||||
|
||||
- fix uint32 overflow ([#615](https://github.com/livekit/rust-sdks/pull/615))
|
||||
|
||||
### Other
|
||||
|
||||
- remove ([#633](https://github.com/livekit/rust-sdks/pull/633))
|
||||
- expose apm stream_delay ([#616](https://github.com/livekit/rust-sdks/pull/616))
|
||||
- Add i420_to_nv12 ([#605](https://github.com/livekit/rust-sdks/pull/605))
|
||||
- ffi-v0.13.0 ([#590](https://github.com/livekit/rust-sdks/pull/590))
|
||||
- add AudioProcessingModule ([#580](https://github.com/livekit/rust-sdks/pull/580))
|
||||
|
||||
## [0.3.10] - 2025-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix build issue
|
||||
|
||||
## [0.3.9] - 2025-01-17
|
||||
|
||||
### Added
|
||||
|
||||
- Expose DataChannel.bufferedAmount property
|
||||
|
||||
## [0.3.8] - 2024-12-14
|
||||
|
||||
### Added
|
||||
|
||||
- bump libwebrtc to m125
|
||||
## 0.3.35 (2026-05-29)
|
||||
|
||||
### Fixes
|
||||
|
||||
- Add native video pipeline timing instrumentation for local video measurements, exposing local publish and subscribe timing through async streams and subscriber overlay GPU upload and receive-to-GPU latency metrics through explicit timing observers.
|
||||
|
||||
## 0.3.34 (2026-05-21)
|
||||
|
||||
### Fixes
|
||||
|
||||
#### feat: add Android application context initialization for PlatformAudio support.
|
||||
|
||||
Android requires `ContextUtils.initialize(applicationContext)` before WebRTC audio components can be created. This change:
|
||||
|
||||
- Adds `livekit_ffi_initialize_android_context()` C FFI function for Unity and other FFI consumers
|
||||
- Uses `CreateAndroidAudioDeviceModule()` instead of generic `CreateAudioDeviceModule()` on Android
|
||||
- Handles empty device GUIDs on Android (falls back to index 0)
|
||||
- Documents Android-specific limitations: single default device, no app-level device selection
|
||||
|
||||
Platform notes:
|
||||
- Android device enumeration returns only one "default" device with empty name/GUID
|
||||
- Audio routing (speaker/earpiece/Bluetooth) is controlled by Android's AudioManager, not WebRTC
|
||||
|
||||
## 0.3.33 (2026-05-14)
|
||||
|
||||
### Fixes
|
||||
|
||||
- feat: add scalability mode for AV1/VP9. - #1076 (@cloudwebrtc)
|
||||
- Add `LIVEKIT_PREFERRED_HW_ENCODER` to prefer `nvenc` or `vaapi` hardware video encoding when both are available.
|
||||
- Relocate unrelated types out of `livekit-protocol`
|
||||
|
||||
#### Get WebRTC ADM into Rust - #1037 (@xianshijing-lk)
|
||||
|
||||
This PR introduces platform audio device management via WebRTC's Audio Device Module (ADM).
|
||||
|
||||
#### Features
|
||||
- **ADM Proxy**: New `AdmProxy` class that switches between Dummy ADM (synthetic mode) and Platform ADM (real audio I/O)
|
||||
- **PlatformAudio API**: High-level Rust API for microphone capture and speaker playout with AEC/AGC/NS
|
||||
- **Device enumeration**: List and select recording/playout devices by index or GUID
|
||||
- **Mode switching**: Seamlessly switch between synthetic mode (FFI callbacks) and platform mode (native speakers) while audio is active
|
||||
- **FFI platform audio support**: Expose platform audio device enumeration and selection through `livekit-ffi`
|
||||
- **Audio processing**: Configure echo cancellation, noise suppression, and auto gain control with platform-specific defaults (hardware on iOS, software elsewhere)
|
||||
|
||||
#### Audio Modes
|
||||
| Mode | Recording | Playout | Use Case |
|
||||
|------|-----------|---------|----------|
|
||||
| Synthetic | NativeAudioSource | Dummy ADM + FFI | Unity audio, agents |
|
||||
| Platform | Platform ADM mic | Platform ADM speakers | VoIP with AEC |
|
||||
|
||||
#### API
|
||||
```rust
|
||||
// Create PlatformAudio for microphone/speaker access
|
||||
let audio = PlatformAudio::new()?;
|
||||
|
||||
// Enumerate and select devices
|
||||
for i in 0..audio.recording_devices() as u16 {
|
||||
println!("Mic {}: {}", i, audio.recording_device_name(i));
|
||||
}
|
||||
audio.set_recording_device(0)?;
|
||||
|
||||
// Create audio track for publishing
|
||||
let track = LocalAudioTrack::create_audio_track("mic", audio.rtc_source());
|
||||
```
|
||||
|
||||
## 0.3.32 (2026-05-11)
|
||||
|
||||
### Fixes
|
||||
|
||||
- Upgrade protocol to v1.45.8
|
||||
|
||||
## 0.3.31 (2026-05-10)
|
||||
|
||||
### Fixes
|
||||
|
||||
- Fix missing `libwebrtc.jar` for Android builds, harden build scripts
|
||||
- fix race in download_webrtc to reduce flaky build - #1047 (@hechen-eng)
|
||||
- Improve WebRTC build scripts and add external_audio_source patch - #1053 (@xianshijing-lk)
|
||||
|
||||
## 0.3.30 (2026-04-23)
|
||||
|
||||
### Features
|
||||
|
||||
#### Add support for frame level packet trailer
|
||||
|
||||
##890 by @chenosaurus
|
||||
|
||||
- Add support to attach/parse frame level timestamps & frame ID to VideoTracks as a custom payload trailer.
|
||||
- Breaking change in VideoFrame API, must include `frame_metadata` or use VideoFrame::new().
|
||||
|
||||
## 0.3.29 (2026-04-02)
|
||||
|
||||
### Features
|
||||
|
||||
#### chore: upgrade libwebrtc to m144.
|
||||
|
||||
##965 by @cloudwebrtc
|
||||
|
||||
### Fixes
|
||||
|
||||
#### use the bounded buffer for video stream
|
||||
|
||||
##956 by @xianshijing-lk
|
||||
|
||||
Before this PR, it uses an unbounded buffer for video stream, that will cause multiple problems:
|
||||
1, video will be lagged behind if rendering is slow or just wake up from background
|
||||
2, it will be out of sync with audio
|
||||
|
||||
This PRs provides options to set a bounded buffer for video stream, and use 1 buffer as the default option.
|
||||
|
||||
## 0.3.28 (2026-03-31)
|
||||
|
||||
### Fixes
|
||||
|
||||
- Upgrade to thiserror 2
|
||||
|
||||
#### fix: fix unavailable sem symbol for Linux aarch64.
|
||||
|
||||
##975 by @cloudwebrtc
|
||||
|
||||
## 0.3.27 (2026-03-22)
|
||||
|
||||
### Features
|
||||
|
||||
#### E2EE: allow setting key_ring_size and key_derivation_algorithm, update webrtc to m144
|
||||
|
||||
##921 by @onestacked
|
||||
|
||||
This PR uses [this webrtc-sdk PR](https://github.com/webrtc-sdk/webrtc/pull/224) to configure the KDF.
|
||||
|
||||
I've tested this with https://codeberg.org/esoteric_programmer/matrix-jukebox and it is compatible with Element Call.
|
||||
|
||||
Fixed: https://github.com/livekit/rust-sdks/issues/796
|
||||
|
||||
### Fixes
|
||||
|
||||
- Fix H.264 codec matching
|
||||
|
||||
#### add bounded buffer to audio_stream, and use 10 frames as the default
|
||||
|
||||
##945 by @xianshijing-lk
|
||||
|
||||
#### fix clang build issue from zed patches (#949)
|
||||
|
||||
##950 by @cloudwebrtc
|
||||
|
||||
* webrtc-sys: Use clang instead of gcc
|
||||
|
||||
* Debug CI output for aarch64-linux
|
||||
|
||||
* ci: Install lld for aarch64-linux FFI builders
|
||||
|
||||
* webrtc-sys: Disable CREL
|
||||
+2439
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,110 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2021"
|
||||
name = "libwebrtc"
|
||||
version = "0.3.35"
|
||||
build = false
|
||||
autolib = false
|
||||
autobins = false
|
||||
autoexamples = false
|
||||
autotests = false
|
||||
autobenches = false
|
||||
description = "Livekit safe bindings to libwebrtc"
|
||||
homepage = "https://livekit.io"
|
||||
readme = false
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/livekit/rust-sdks"
|
||||
|
||||
[features]
|
||||
default = ["glib-main-loop"]
|
||||
glib-main-loop = ["dep:glib"]
|
||||
|
||||
[lib]
|
||||
name = "libwebrtc"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies.log]
|
||||
version = "0.4"
|
||||
|
||||
[dependencies.serde]
|
||||
version = "1"
|
||||
features = ["derive"]
|
||||
|
||||
[dependencies.serde_json]
|
||||
version = "1.0"
|
||||
|
||||
[dependencies.thiserror]
|
||||
version = "2"
|
||||
|
||||
[dev-dependencies.env_logger]
|
||||
version = "0.11"
|
||||
|
||||
[target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies.glib]
|
||||
version = "0.21.3"
|
||||
optional = true
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.cxx]
|
||||
version = "1.0"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.lazy_static]
|
||||
version = "1.4"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.livekit-runtime]
|
||||
version = "0.4.0"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.parking_lot]
|
||||
version = "0.12"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.rtrb]
|
||||
version = "0.3.3"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.tokio]
|
||||
version = "1"
|
||||
features = [
|
||||
"sync",
|
||||
"macros",
|
||||
]
|
||||
default-features = false
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies.webrtc-sys]
|
||||
version = "0.3.33"
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies.js-sys]
|
||||
version = "0.3"
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies.wasm-bindgen]
|
||||
version = "0.2"
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies.wasm-bindgen-futures]
|
||||
version = "0.4"
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
|
||||
version = "0.3"
|
||||
features = [
|
||||
"MessageEvent",
|
||||
"RtcPeerConnection",
|
||||
"RtcSignalingState",
|
||||
"RtcSdpType",
|
||||
"RtcSessionDescriptionInit",
|
||||
"RtcPeerConnectionIceEvent",
|
||||
"RtcIceCandidate",
|
||||
"RtcDataChannel",
|
||||
"RtcDataChannelEvent",
|
||||
"RtcDataChannelState",
|
||||
"EventTarget",
|
||||
"WebGlRenderingContext",
|
||||
"WebGlTexture",
|
||||
]
|
||||
|
||||
[target.'cfg(target_os = "android")'.dependencies.jni]
|
||||
version = "0.21"
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
[package]
|
||||
name = "libwebrtc"
|
||||
version = "0.3.35"
|
||||
edition.workspace = true
|
||||
homepage = "https://livekit.io"
|
||||
license.workspace = true
|
||||
description = "Livekit safe bindings to libwebrtc"
|
||||
repository.workspace = true
|
||||
|
||||
[features]
|
||||
default = [ "glib-main-loop" ]
|
||||
# On Wayland, libwebrtc uses GDBus to communicate with the XDG Desktop Portal.
|
||||
# GDBus requires a GLib event loop to be running. If you already have a GLib
|
||||
# event loop running in your application, for example if you are using the
|
||||
# GTK or GStreamer Rust bindings, disable this feature.
|
||||
glib-main-loop = [ "dep:glib" ]
|
||||
|
||||
[dependencies]
|
||||
log = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies]
|
||||
glib = { version = "0.21.3", optional = true }
|
||||
|
||||
[target.'cfg(target_os = "android")'.dependencies]
|
||||
jni = "0.21"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
webrtc-sys = { workspace = true }
|
||||
livekit-runtime = { workspace = true }
|
||||
lazy_static = { workspace = true }
|
||||
parking_lot = { workspace = true }
|
||||
tokio = { workspace = true, default-features = false, features = ["sync", "macros"] }
|
||||
cxx = "1.0"
|
||||
rtrb = "0.3.3"
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies]
|
||||
wasm-bindgen = "0.2"
|
||||
js-sys = "0.3"
|
||||
wasm-bindgen-futures = "0.4"
|
||||
web-sys = { version = "0.3", features = [
|
||||
"MessageEvent",
|
||||
"RtcPeerConnection",
|
||||
"RtcSignalingState",
|
||||
"RtcSdpType",
|
||||
"RtcSessionDescriptionInit",
|
||||
"RtcPeerConnectionIceEvent",
|
||||
"RtcIceCandidate",
|
||||
"RtcDataChannel",
|
||||
"RtcDataChannelEvent",
|
||||
"RtcDataChannelState",
|
||||
"EventTarget",
|
||||
"WebGlRenderingContext",
|
||||
"WebGlTexture",
|
||||
] }
|
||||
|
||||
[dev-dependencies]
|
||||
env_logger = { workspace = true }
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AudioFrame<'a> {
|
||||
pub data: Cow<'a, [i16]>,
|
||||
pub sample_rate: u32,
|
||||
pub num_channels: u32,
|
||||
pub samples_per_channel: u32,
|
||||
}
|
||||
|
||||
impl AudioFrame<'_> {
|
||||
// Owned
|
||||
pub fn new(sample_rate: u32, num_channels: u32, samples_per_channel: u32) -> Self {
|
||||
Self {
|
||||
data: vec![0; (num_channels * samples_per_channel) as usize].into(),
|
||||
sample_rate,
|
||||
num_channels,
|
||||
samples_per_channel,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::imp::audio_source as imp_as;
|
||||
|
||||
/// Default sample rate used by WebRTC audio pipelines (48kHz).
|
||||
pub const DEFAULT_SAMPLE_RATE: u32 = 48000;
|
||||
|
||||
/// Default number of audio channels (mono).
|
||||
pub const DEFAULT_NUM_CHANNELS: u32 = 1;
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub struct AudioSourceOptions {
|
||||
pub echo_cancellation: bool,
|
||||
pub noise_suppression: bool,
|
||||
pub auto_gain_control: bool,
|
||||
}
|
||||
|
||||
/// Audio source type for creating audio tracks.
|
||||
///
|
||||
/// Choose the appropriate source based on your use case:
|
||||
///
|
||||
/// | Use Case | Source | Description |
|
||||
/// |----------|--------|-------------|
|
||||
/// | Manual audio (TTS, files) | `RtcAudioSource::Native(source)` | Push frames manually |
|
||||
/// | Microphone capture | `RtcAudioSource::Device` | Automatic via Platform ADM |
|
||||
/// | Both (mic + screen) | Use both types | Multiple tracks supported |
|
||||
///
|
||||
/// # Combining Sources
|
||||
///
|
||||
/// You can have multiple audio tracks with different source types:
|
||||
/// - Track A: `RtcAudioSource::Device` for microphone (via `PlatformAudio`)
|
||||
/// - Track B: `RtcAudioSource::Native` for screen capture or TTS
|
||||
#[non_exhaustive]
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum RtcAudioSource {
|
||||
/// Native audio source for manual audio frame capture.
|
||||
///
|
||||
/// Use this with Synthetic ADM mode (the default). You push audio frames
|
||||
/// manually via `NativeAudioSource::capture_frame()`.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// use livekit::webrtc::audio_source::native::NativeAudioSource;
|
||||
/// use livekit::webrtc::audio_source::{AudioSourceOptions, RtcAudioSource};
|
||||
///
|
||||
/// let source = NativeAudioSource::new(
|
||||
/// AudioSourceOptions::default(),
|
||||
/// 48000, 2, 100,
|
||||
/// );
|
||||
/// source.capture_frame(&frame).await?;
|
||||
///
|
||||
/// let track = LocalAudioTrack::create_audio_track(
|
||||
/// "audio",
|
||||
/// RtcAudioSource::Native(source),
|
||||
/// );
|
||||
/// ```
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
Native(native::NativeAudioSource),
|
||||
|
||||
/// Device audio source - uses Platform ADM for automatic microphone capture.
|
||||
///
|
||||
/// WebRTC automatically captures audio from the selected recording device
|
||||
/// (microphone). You do NOT push frames manually.
|
||||
///
|
||||
/// # Usage
|
||||
///
|
||||
/// Use `PlatformAudio` from the `livekit` crate, which manages the Platform ADM
|
||||
/// lifecycle and provides `RtcAudioSource::Device` via `rtc_source()`:
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// use livekit::prelude::*;
|
||||
///
|
||||
/// // Create PlatformAudio (enables Platform ADM)
|
||||
/// let audio = PlatformAudio::new()?;
|
||||
///
|
||||
/// // Optionally select a specific device
|
||||
/// if let Some(device) = audio.recording_devices().next() {
|
||||
/// audio.set_recording_device(&device.id)?;
|
||||
/// }
|
||||
///
|
||||
/// // Create track using the device source
|
||||
/// let track = LocalAudioTrack::create_audio_track("mic", audio.rtc_source());
|
||||
/// ```
|
||||
///
|
||||
/// # Combining with NativeAudioSource
|
||||
///
|
||||
/// You CAN use `NativeAudioSource` alongside Platform ADM to have multiple
|
||||
/// audio tracks with different sources (e.g., microphone + screen capture).
|
||||
///
|
||||
/// # Platform Support
|
||||
///
|
||||
/// - **iOS**: CoreAudio with VPIO (Voice Processing IO)
|
||||
/// - **macOS**: CoreAudio
|
||||
/// - **Windows**: WASAPI
|
||||
/// - **Linux**: PulseAudio / ALSA
|
||||
/// - **Android**: AAudio / OpenSL ES
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
Device,
|
||||
}
|
||||
|
||||
impl RtcAudioSource {
|
||||
/// Set audio processing options.
|
||||
/// Note: For `Device` source, options are controlled by the Platform ADM.
|
||||
pub fn set_audio_options(&self, options: AudioSourceOptions) {
|
||||
match self {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
RtcAudioSource::Native(source) => source.set_audio_options(options),
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
RtcAudioSource::Device => {
|
||||
// Device source options are managed by the Platform ADM
|
||||
// This is a no-op
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get audio processing options.
|
||||
/// Note: For `Device` source, returns default options (actual options are managed by ADM).
|
||||
pub fn audio_options(&self) -> AudioSourceOptions {
|
||||
match self {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
RtcAudioSource::Native(source) => source.audio_options(),
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
RtcAudioSource::Device => AudioSourceOptions::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the sample rate.
|
||||
/// Note: For `Device` source, returns [`DEFAULT_SAMPLE_RATE`] (48kHz).
|
||||
pub fn sample_rate(&self) -> u32 {
|
||||
match self {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
RtcAudioSource::Native(source) => source.sample_rate(),
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
RtcAudioSource::Device => DEFAULT_SAMPLE_RATE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the number of channels.
|
||||
/// Note: For `Device` source, returns [`DEFAULT_NUM_CHANNELS`] (mono).
|
||||
pub fn num_channels(&self) -> u32 {
|
||||
match self {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
RtcAudioSource::Native(source) => source.num_channels(),
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
RtcAudioSource::Device => DEFAULT_NUM_CHANNELS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod native {
|
||||
use std::fmt::{Debug, Formatter};
|
||||
|
||||
use super::*;
|
||||
use crate::{audio_frame::AudioFrame, RtcError};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NativeAudioSource {
|
||||
pub(crate) handle: imp_as::NativeAudioSource,
|
||||
}
|
||||
|
||||
impl Debug for NativeAudioSource {
|
||||
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
|
||||
f.debug_struct("NativeAudioSource").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeAudioSource {
|
||||
pub fn new(
|
||||
options: AudioSourceOptions,
|
||||
sample_rate: u32,
|
||||
num_channels: u32,
|
||||
queue_size_ms: u32,
|
||||
) -> NativeAudioSource {
|
||||
Self {
|
||||
handle: imp_as::NativeAudioSource::new(
|
||||
options,
|
||||
sample_rate,
|
||||
num_channels,
|
||||
queue_size_ms,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear_buffer(&self) {
|
||||
self.handle.clear_buffer()
|
||||
}
|
||||
|
||||
pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> {
|
||||
self.handle.capture_frame(frame).await
|
||||
}
|
||||
|
||||
pub fn set_audio_options(&self, options: AudioSourceOptions) {
|
||||
self.handle.set_audio_options(options)
|
||||
}
|
||||
|
||||
pub fn audio_options(&self) -> AudioSourceOptions {
|
||||
self.handle.audio_options()
|
||||
}
|
||||
|
||||
pub fn sample_rate(&self) -> u32 {
|
||||
self.handle.sample_rate()
|
||||
}
|
||||
|
||||
pub fn num_channels(&self) -> u32 {
|
||||
self.handle.num_channels()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::imp::audio_stream as stream_imp;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod native {
|
||||
use std::{
|
||||
fmt::{Debug, Formatter},
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
use livekit_runtime::Stream;
|
||||
|
||||
use super::stream_imp;
|
||||
use crate::{audio_frame::AudioFrame, audio_track::RtcAudioTrack};
|
||||
|
||||
const DEFAULT_QUEUE_SIZE_FRAMES: usize = 10;
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct NativeAudioStreamOptions {
|
||||
/// Maximum number of queued WebRTC sink frames after the audio callback.
|
||||
///
|
||||
/// Each queued frame corresponds to roughly 10 ms of decoded PCM audio
|
||||
/// on the WebRTC sink path.
|
||||
///
|
||||
/// `None` uses the default bounded queue size of 10 frames. `Some(0)`
|
||||
/// opts into unbounded buffering. Positive values bound the queue, and
|
||||
/// the stream drops the oldest queued frames on overflow so latency
|
||||
/// stays bounded.
|
||||
///
|
||||
/// If your application consumes both audio and video, keep the queue
|
||||
/// sizing strategy coordinated across both streams. Using a much larger
|
||||
/// queue, or unbounded buffering, for only one of them can increase
|
||||
/// end-to-end latency for that stream and cause audio/video drift.
|
||||
pub queue_size_frames: Option<usize>,
|
||||
}
|
||||
|
||||
pub struct NativeAudioStream {
|
||||
pub(crate) handle: stream_imp::NativeAudioStream,
|
||||
}
|
||||
|
||||
impl Debug for NativeAudioStream {
|
||||
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
|
||||
f.debug_struct("NativeAudioStream").field("track", &self.track()).finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeAudioStream {
|
||||
pub fn new(audio_track: RtcAudioTrack, sample_rate: i32, num_channels: i32) -> Self {
|
||||
Self {
|
||||
handle: stream_imp::NativeAudioStream::new(
|
||||
audio_track,
|
||||
sample_rate,
|
||||
num_channels,
|
||||
Some(DEFAULT_QUEUE_SIZE_FRAMES),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_options(
|
||||
audio_track: RtcAudioTrack,
|
||||
sample_rate: i32,
|
||||
num_channels: i32,
|
||||
options: NativeAudioStreamOptions,
|
||||
) -> Self {
|
||||
Self {
|
||||
handle: stream_imp::NativeAudioStream::new(
|
||||
audio_track,
|
||||
sample_rate,
|
||||
num_channels,
|
||||
normalize_queue_size_frames(options.queue_size_frames),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn track(&self) -> RtcAudioTrack {
|
||||
self.handle.track()
|
||||
}
|
||||
|
||||
pub fn close(&mut self) {
|
||||
self.handle.close()
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for NativeAudioStream {
|
||||
type Item = AudioFrame<'static>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
Pin::new(&mut self.get_mut().handle).poll_next(cx)
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_queue_size_frames(queue_size_frames: Option<usize>) -> Option<usize> {
|
||||
match queue_size_frames {
|
||||
None => Some(DEFAULT_QUEUE_SIZE_FRAMES),
|
||||
Some(0) => None,
|
||||
Some(value) => Some(value),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{
|
||||
imp::audio_track as imp_at,
|
||||
media_stream_track::{media_stream_track, RtcTrackState},
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtcAudioTrack {
|
||||
pub(crate) handle: imp_at::RtcAudioTrack,
|
||||
}
|
||||
|
||||
impl RtcAudioTrack {
|
||||
media_stream_track!();
|
||||
}
|
||||
|
||||
impl Debug for RtcAudioTrack {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RtcAudioTrack")
|
||||
.field("id", &self.id())
|
||||
.field("enabled", &self.enabled())
|
||||
.field("state", &self.state())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{fmt::Debug, str::Utf8Error};
|
||||
|
||||
use serde::Deserialize;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::{imp::data_channel as dc_imp, rtp_parameters::Priority};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DataChannelInit {
|
||||
pub ordered: bool,
|
||||
pub max_retransmit_time: Option<i32>,
|
||||
pub max_retransmits: Option<i32>,
|
||||
pub protocol: String,
|
||||
pub negotiated: bool,
|
||||
pub id: i32,
|
||||
pub priority: Option<Priority>,
|
||||
}
|
||||
|
||||
impl Default for DataChannelInit {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
ordered: true,
|
||||
max_retransmit_time: None,
|
||||
max_retransmits: None,
|
||||
protocol: String::new(),
|
||||
negotiated: false,
|
||||
id: -1,
|
||||
priority: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum DataChannelError {
|
||||
#[error("failed to send data, dc not open? send buffer is full ?")]
|
||||
Send,
|
||||
#[error("only utf8 strings can be sent")]
|
||||
Utf8(#[from] Utf8Error),
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DataChannelState {
|
||||
Connecting,
|
||||
Open,
|
||||
Closing,
|
||||
Closed,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DataBuffer<'a> {
|
||||
pub data: &'a [u8],
|
||||
pub binary: bool,
|
||||
}
|
||||
|
||||
pub type OnStateChange = Box<dyn FnMut(DataChannelState) + Send + Sync>;
|
||||
pub type OnMessage = Box<dyn FnMut(DataBuffer) + Send + Sync>;
|
||||
pub type OnBufferedAmountChange = Box<dyn FnMut(u64) + Send + Sync>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct DataChannel {
|
||||
pub(crate) handle: dc_imp::DataChannel,
|
||||
}
|
||||
|
||||
impl DataChannel {
|
||||
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
|
||||
self.handle.send(data, binary)
|
||||
}
|
||||
|
||||
pub fn id(&self) -> i32 {
|
||||
self.handle.id()
|
||||
}
|
||||
|
||||
pub fn label(&self) -> String {
|
||||
self.handle.label()
|
||||
}
|
||||
|
||||
pub fn state(&self) -> DataChannelState {
|
||||
self.handle.state()
|
||||
}
|
||||
|
||||
pub fn close(&self) {
|
||||
self.handle.close()
|
||||
}
|
||||
|
||||
pub fn buffered_amount(&self) -> u64 {
|
||||
self.handle.buffered_amount()
|
||||
}
|
||||
|
||||
pub fn on_state_change(&self, callback: Option<OnStateChange>) {
|
||||
self.handle.on_state_change(callback)
|
||||
}
|
||||
|
||||
pub fn on_message(&self, callback: Option<OnMessage>) {
|
||||
self.handle.on_message(callback)
|
||||
}
|
||||
|
||||
pub fn on_buffered_amount_change(&self, callback: Option<OnBufferedAmountChange>) {
|
||||
self.handle.on_buffered_amount_change(callback)
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for DataChannel {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("DataChannel")
|
||||
.field("id", &self.id())
|
||||
.field("label", &self.label())
|
||||
.field("state", &self.state())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::imp::desktop_capturer as imp_dc;
|
||||
|
||||
/// Configuration options for creating a desktop capturer.
|
||||
///
|
||||
/// It contains a subset of libwebrtc's DesktopCaptureOptions.
|
||||
///
|
||||
/// By default, it captures the entire screen and does not include the cursor.
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use libwebrtc::desktop_capturer::{DesktopCapturerOptions, DesktopCaptureSourceType};
|
||||
///
|
||||
/// let mut options = DesktopCapturerOptions::new(DesktopCaptureSourceType::Screen);
|
||||
/// options.set_include_cursor(true);
|
||||
/// ```
|
||||
pub struct DesktopCapturerOptions {
|
||||
sys_handle: imp_dc::DesktopCapturerOptions,
|
||||
}
|
||||
|
||||
/// Specifies the type of source that a desktop capturer should capture.
|
||||
#[derive(Debug, Copy, Clone, PartialEq)]
|
||||
pub enum DesktopCaptureSourceType {
|
||||
Screen,
|
||||
Window,
|
||||
#[cfg(any(target_os = "macos", target_os = "linux"))]
|
||||
Generic,
|
||||
}
|
||||
|
||||
impl DesktopCapturerOptions {
|
||||
/// Creates a new `DesktopCapturerOptions` with default values.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `source_type` - The type of source to capture (screen or window).
|
||||
///
|
||||
/// # Defaults
|
||||
///
|
||||
/// - Cursor is not included in captured frames (use [`set_include_cursor`](Self::set_include_cursor) to change)
|
||||
/// - On macOS, the ScreenCaptureKit system picker is enabled (use [`set_sck_system_picker`](Self::set_sck_system_picker) to change)
|
||||
pub fn new(source_type: DesktopCaptureSourceType) -> Self {
|
||||
let source_type = match source_type {
|
||||
DesktopCaptureSourceType::Screen => imp_dc::SourceType::Screen,
|
||||
DesktopCaptureSourceType::Window => imp_dc::SourceType::Window,
|
||||
#[cfg(any(target_os = "macos", target_os = "linux"))]
|
||||
DesktopCaptureSourceType::Generic => imp_dc::SourceType::Generic,
|
||||
};
|
||||
Self { sys_handle: imp_dc::DesktopCapturerOptions::new(source_type) }
|
||||
}
|
||||
|
||||
/// Sets whether to include the cursor in captured frames.
|
||||
pub fn set_include_cursor(&mut self, include: bool) {
|
||||
self.sys_handle = self.sys_handle.with_cursor(include);
|
||||
}
|
||||
|
||||
/// Sets whether to allow the ScreenCaptureKit system picker on macOS.
|
||||
///
|
||||
/// This is enabled by default.
|
||||
///
|
||||
/// When disabled, for capturing displays the client should get the source id
|
||||
/// via a different way as [`DesktopCapturer::get_source_list`] returns an empty vector.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn set_sck_system_picker(&mut self, allow_sck_system_picker: bool) {
|
||||
self.sys_handle = self.sys_handle.with_sck_system_picker(allow_sck_system_picker);
|
||||
}
|
||||
}
|
||||
|
||||
/// A desktop capturer for capturing screens or windows.
|
||||
pub struct DesktopCapturer {
|
||||
handle: imp_dc::DesktopCapturer,
|
||||
}
|
||||
|
||||
impl DesktopCapturer {
|
||||
/// Creates a new `DesktopCapturer` with the specified callback and options.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `options` - Configuration options for the capturer
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Returns `Some(DesktopCapturer)` if the capturer was created successfully,
|
||||
/// or `None` if creation failed (e.g., due to platform limitations or permissions).
|
||||
pub fn new(options: DesktopCapturerOptions) -> Option<Self> {
|
||||
let desktop_capturer = imp_dc::DesktopCapturer::new(options.sys_handle);
|
||||
if desktop_capturer.is_none() {
|
||||
return None;
|
||||
}
|
||||
Some(Self { handle: desktop_capturer.unwrap() })
|
||||
}
|
||||
|
||||
/// Starts capturing from the specified source.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `source` - The capture source to use. It should be None when the capturer
|
||||
/// is configured to use the system picker (on platforms that support it).
|
||||
/// * `callback` - A function that will be called for each captured frame. The callback
|
||||
/// receives a [`CaptureResult`] indicating success or error, and a [`DesktopFrame`]
|
||||
/// containing the captured image data.
|
||||
///
|
||||
/// # Note
|
||||
///
|
||||
/// After calling this method, you must call [`capture_frame`](Self::capture_frame)
|
||||
/// to actually capture frames. This method only initializes the capture session.
|
||||
pub fn start_capture<T>(&mut self, source: Option<CaptureSource>, mut callback: T)
|
||||
where
|
||||
T: FnMut(Result<DesktopFrame, CaptureError>) + Send + 'static,
|
||||
{
|
||||
if let Some(source) = source {
|
||||
self.handle.select_source(source.sys_handle.id());
|
||||
}
|
||||
let inner_callback = move |result: Result<imp_dc::DesktopFrame, imp_dc::CaptureError>| {
|
||||
callback(capture_result_from_sys(result));
|
||||
};
|
||||
self.handle.start(inner_callback);
|
||||
}
|
||||
|
||||
/// Captures a single frame.
|
||||
///
|
||||
/// You must call [`start_capture`](Self::start_capture) before calling this method.
|
||||
pub fn capture_frame(&mut self) {
|
||||
self.handle.capture_frame();
|
||||
}
|
||||
|
||||
/// Retrieves a list of available capture sources.
|
||||
///
|
||||
/// Returns a list of screens or windows that can be captured, depending
|
||||
/// on whether the capturer was configured for window or screen capture.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A vector of [`CaptureSource`] objects representing available capture sources.
|
||||
pub fn get_source_list(&self) -> Vec<CaptureSource> {
|
||||
let source_list = self.handle.get_source_list();
|
||||
source_list.into_iter().map(|source| CaptureSource { sys_handle: source }).collect()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DesktopFrame {
|
||||
sys_handle: imp_dc::DesktopFrame,
|
||||
}
|
||||
|
||||
impl DesktopFrame {
|
||||
fn new(sys_handle: imp_dc::DesktopFrame) -> Self {
|
||||
Self { sys_handle }
|
||||
}
|
||||
|
||||
pub fn width(&self) -> i32 {
|
||||
self.sys_handle.width() as i32
|
||||
}
|
||||
|
||||
pub fn height(&self) -> i32 {
|
||||
self.sys_handle.height() as i32
|
||||
}
|
||||
|
||||
pub fn stride(&self) -> u32 {
|
||||
self.sys_handle.stride() as u32
|
||||
}
|
||||
|
||||
pub fn left(&self) -> i32 {
|
||||
self.sys_handle.left()
|
||||
}
|
||||
|
||||
pub fn top(&self) -> i32 {
|
||||
self.sys_handle.top()
|
||||
}
|
||||
|
||||
pub fn data(&self) -> &[u8] {
|
||||
self.sys_handle.data()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CaptureSource {
|
||||
sys_handle: imp_dc::CaptureSource,
|
||||
}
|
||||
|
||||
impl CaptureSource {
|
||||
pub fn id(&self) -> u64 {
|
||||
self.sys_handle.id()
|
||||
}
|
||||
pub fn title(&self) -> String {
|
||||
self.sys_handle.title()
|
||||
}
|
||||
pub fn display_id(&self) -> i64 {
|
||||
self.sys_handle.display_id()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for CaptureSource {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("CaptureSource")
|
||||
.field("id", &self.id())
|
||||
.field("title", &self.title())
|
||||
.field("display_id", &self.display_id())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub enum CaptureError {
|
||||
Temporary,
|
||||
Permanent,
|
||||
}
|
||||
|
||||
fn capture_result_from_sys(
|
||||
result: Result<imp_dc::DesktopFrame, imp_dc::CaptureError>,
|
||||
) -> Result<DesktopFrame, CaptureError> {
|
||||
match result {
|
||||
Ok(frame) => Ok(DesktopFrame::new(frame)),
|
||||
Err(error) => Err(match error {
|
||||
imp_dc::CaptureError::Temporary => CaptureError::Temporary,
|
||||
imp_dc::CaptureError::Permanent => CaptureError::Permanent,
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// TODO(theomonnom): Async methods
|
||||
#[macro_export]
|
||||
macro_rules! enum_dispatch {
|
||||
// This arm is used to avoid nested loops with the arguments
|
||||
// The arguments are transformed to $combined_args tt
|
||||
(@match [$($variant:ident),+]: $fnc:ident, $self:ident, $combined_args:tt) => {
|
||||
match $self {
|
||||
$(
|
||||
Self::$variant(inner) => inner.$fnc$combined_args,
|
||||
)+
|
||||
}
|
||||
};
|
||||
|
||||
// Create the function and extract self fron the $args tt (little hack)
|
||||
(@fnc [$($variant:ident),+]: $vis:vis fn $fnc:ident($self:ident: $sty:ty $(, $arg:ident: $t:ty)*) -> $ret:ty) => {
|
||||
#[inline]
|
||||
$vis fn $fnc($self: $sty, $($arg: $t),*) -> $ret {
|
||||
$crate::enum_dispatch!(@match [$($variant),+]: $fnc, $self, ($($arg,)*))
|
||||
}
|
||||
};
|
||||
|
||||
($variants:tt; $($vis:vis fn $fnc:ident$args:tt -> $ret:ty;)+) => {
|
||||
$(
|
||||
$crate::enum_dispatch!(@fnc $variants: $vis fn $fnc$args -> $ret);
|
||||
)+
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{imp::ice_candidate as imp_ic, session_description::SdpParseError};
|
||||
|
||||
pub struct IceCandidate {
|
||||
pub(crate) handle: imp_ic::IceCandidate,
|
||||
}
|
||||
|
||||
impl IceCandidate {
|
||||
pub fn parse(
|
||||
sdp_mid: &str,
|
||||
sdp_mline_index: i32,
|
||||
sdp: &str,
|
||||
) -> Result<IceCandidate, SdpParseError> {
|
||||
imp_ic::IceCandidate::parse(sdp_mid, sdp_mline_index, sdp)
|
||||
}
|
||||
|
||||
pub fn sdp_mid(&self) -> String {
|
||||
self.handle.sdp_mid()
|
||||
}
|
||||
|
||||
pub fn sdp_mline_index(&self) -> i32 {
|
||||
self.handle.sdp_mline_index()
|
||||
}
|
||||
|
||||
pub fn candidate(&self) -> String {
|
||||
self.handle.candidate()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToString for IceCandidate {
|
||||
fn to_string(&self) -> String {
|
||||
self.handle.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for IceCandidate {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("IceCandidate").field("candidate", &self.to_string()).finish()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
#[cfg_attr(target_arch = "wasm32", path = "web/mod.rs")]
|
||||
#[cfg_attr(not(target_arch = "wasm32"), path = "native/mod.rs")]
|
||||
mod imp;
|
||||
|
||||
mod enum_dispatch;
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum MediaType {
|
||||
Audio,
|
||||
Video,
|
||||
Data,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum RtcErrorType {
|
||||
Internal,
|
||||
InvalidSdp,
|
||||
InvalidState,
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
#[error("an RtcError occurred: {error_type:?} - {message}")]
|
||||
pub struct RtcError {
|
||||
pub error_type: RtcErrorType,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
pub mod audio_frame;
|
||||
pub mod audio_source;
|
||||
pub mod audio_stream;
|
||||
pub mod audio_track;
|
||||
pub mod data_channel;
|
||||
#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
|
||||
pub mod desktop_capturer;
|
||||
pub mod ice_candidate;
|
||||
pub mod media_stream;
|
||||
pub mod media_stream_track;
|
||||
pub mod peer_connection;
|
||||
pub mod peer_connection_factory;
|
||||
pub mod prelude;
|
||||
pub mod recorded_audio;
|
||||
pub mod rtp_parameters;
|
||||
pub mod rtp_receiver;
|
||||
pub mod rtp_sender;
|
||||
pub mod rtp_transceiver;
|
||||
pub mod session_description;
|
||||
pub mod stats;
|
||||
pub mod video_frame;
|
||||
pub mod video_source;
|
||||
pub mod video_stream;
|
||||
pub mod video_track;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod native {
|
||||
pub use webrtc_sys::webrtc::ffi::create_random_uuid;
|
||||
|
||||
pub use crate::imp::{
|
||||
apm, audio_mixer, audio_resampler, frame_cryptor, packet_trailer, yuv_helper,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
pub mod android {
|
||||
pub use crate::imp::android::*;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{audio_track::RtcAudioTrack, imp::media_stream as imp_ms, video_track::RtcVideoTrack};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct MediaStream {
|
||||
pub(crate) handle: imp_ms::MediaStream,
|
||||
}
|
||||
|
||||
impl MediaStream {
|
||||
pub fn id(&self) -> String {
|
||||
self.handle.id()
|
||||
}
|
||||
|
||||
pub fn audio_tracks(&self) -> Vec<RtcAudioTrack> {
|
||||
self.handle.audio_tracks()
|
||||
}
|
||||
|
||||
pub fn video_tracks(&self) -> Vec<RtcVideoTrack> {
|
||||
self.handle.video_tracks()
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for MediaStream {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("MediaStream")
|
||||
.field("id", &self.id())
|
||||
.field("audio_tracks", &self.audio_tracks())
|
||||
.field("video_tracks", &self.video_tracks())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{audio_track::RtcAudioTrack, enum_dispatch, video_track::RtcVideoTrack};
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum RtcTrackState {
|
||||
Live,
|
||||
Ended,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum MediaStreamTrack {
|
||||
Video(RtcVideoTrack),
|
||||
Audio(RtcAudioTrack),
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
impl MediaStreamTrack {
|
||||
enum_dispatch!(
|
||||
[Video, Audio];
|
||||
pub(crate) fn sys_handle(self: &Self) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack>;
|
||||
);
|
||||
}
|
||||
|
||||
impl MediaStreamTrack {
|
||||
enum_dispatch!(
|
||||
[Video, Audio];
|
||||
pub fn id(self: &Self) -> String;
|
||||
pub fn enabled(self: &Self) -> bool;
|
||||
pub fn set_enabled(self: &Self, enabled: bool) -> bool;
|
||||
pub fn state(self: &Self) -> RtcTrackState;
|
||||
);
|
||||
}
|
||||
|
||||
macro_rules! media_stream_track {
|
||||
() => {
|
||||
pub fn id(&self) -> String {
|
||||
self.handle.id()
|
||||
}
|
||||
|
||||
pub fn enabled(&self) -> bool {
|
||||
self.handle.enabled()
|
||||
}
|
||||
|
||||
pub fn set_enabled(&self, enabled: bool) -> bool {
|
||||
self.handle.set_enabled(enabled)
|
||||
}
|
||||
|
||||
pub fn state(&self) -> RtcTrackState {
|
||||
self.handle.state().into()
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub(crate) fn sys_handle(
|
||||
&self,
|
||||
) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack> {
|
||||
self.handle.sys_handle()
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) use media_stream_track;
|
||||
|
||||
impl From<RtcAudioTrack> for MediaStreamTrack {
|
||||
fn from(track: RtcAudioTrack) -> Self {
|
||||
Self::Audio(track)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtcVideoTrack> for MediaStreamTrack {
|
||||
fn from(track: RtcVideoTrack) -> Self {
|
||||
Self::Video(track)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use jni::objects::JObject;
|
||||
use webrtc_sys::android::ffi as sys_android;
|
||||
|
||||
/// Initialize Android WebRTC with the JVM.
|
||||
///
|
||||
/// This is automatically called by [`initialize_android_context`], so you only
|
||||
/// need to call this directly if you don't have access to an Android Context
|
||||
/// (e.g., in `JNI_OnLoad`).
|
||||
///
|
||||
/// This function is idempotent - safe to call multiple times.
|
||||
pub fn initialize_android(vm: &jni::JavaVM) {
|
||||
unsafe {
|
||||
sys_android::init_android(vm.get_java_vm_pointer() as *mut _);
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize Android WebRTC with the application context.
|
||||
///
|
||||
/// This is the main initialization function for Android. It performs both:
|
||||
/// 1. JVM initialization (same as [`initialize_android`])
|
||||
/// 2. Context initialization (required for PlatformAudio)
|
||||
///
|
||||
/// This function is idempotent - safe to call multiple times.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `vm` - The JavaVM instance
|
||||
/// * `context` - The Android application context
|
||||
///
|
||||
/// # Returns
|
||||
/// `true` if context initialization succeeded, `false` otherwise.
|
||||
/// Note: JVM initialization always happens regardless of return value.
|
||||
///
|
||||
/// # Example
|
||||
/// ```ignore
|
||||
/// use jni::JavaVM;
|
||||
/// use jni::objects::JObject;
|
||||
/// use livekit::webrtc::android::initialize_android_context;
|
||||
///
|
||||
/// fn init(vm: JavaVM, context: JObject) {
|
||||
/// // Just one call needed - handles both JVM and context init
|
||||
/// initialize_android_context(&vm, &context);
|
||||
/// }
|
||||
/// ```
|
||||
pub fn initialize_android_context(vm: &jni::JavaVM, context: &JObject) -> bool {
|
||||
unsafe {
|
||||
sys_android::init_android_context(
|
||||
vm.get_java_vm_pointer() as *mut _,
|
||||
context.as_raw() as usize,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::UniquePtr;
|
||||
use webrtc_sys::apm::ffi as sys_apm;
|
||||
|
||||
use crate::{RtcError, RtcErrorType};
|
||||
|
||||
pub struct AudioProcessingModule {
|
||||
sys_handle: UniquePtr<sys_apm::AudioProcessingModule>,
|
||||
}
|
||||
|
||||
impl AudioProcessingModule {
|
||||
pub fn new(
|
||||
echo_canceller_enabled: bool,
|
||||
gain_controller_enabled: bool,
|
||||
high_pass_filter_enabled: bool,
|
||||
noise_suppression_enabled: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
sys_handle: sys_apm::create_apm(
|
||||
echo_canceller_enabled,
|
||||
gain_controller_enabled,
|
||||
high_pass_filter_enabled,
|
||||
noise_suppression_enabled,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn process_stream(
|
||||
&mut self,
|
||||
data: &mut [i16],
|
||||
sample_rate: i32,
|
||||
num_channels: i32,
|
||||
) -> Result<(), RtcError> {
|
||||
let samples_per_10ms = (sample_rate as usize / 100) * num_channels as usize;
|
||||
assert!(
|
||||
data.len() % samples_per_10ms == 0 && data.len() >= samples_per_10ms,
|
||||
"slice must have a multiple of 10ms worth of samples"
|
||||
);
|
||||
|
||||
for chunk in data.chunks_mut(samples_per_10ms) {
|
||||
if unsafe {
|
||||
self.sys_handle.pin_mut().process_stream(
|
||||
chunk.as_mut_ptr(),
|
||||
chunk.len(),
|
||||
chunk.as_mut_ptr(),
|
||||
chunk.len(),
|
||||
sample_rate,
|
||||
num_channels,
|
||||
)
|
||||
} != 0
|
||||
{
|
||||
return Err(RtcError {
|
||||
error_type: RtcErrorType::Internal,
|
||||
message: "Failed to process stream".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn process_reverse_stream(
|
||||
&mut self,
|
||||
data: &mut [i16],
|
||||
sample_rate: i32,
|
||||
num_channels: i32,
|
||||
) -> Result<(), RtcError> {
|
||||
let samples_per_10ms = (sample_rate as usize / 100) * num_channels as usize;
|
||||
assert!(
|
||||
data.len() % samples_per_10ms == 0 && data.len() >= samples_per_10ms,
|
||||
"slice must have a multiple of 10ms worth of samples"
|
||||
);
|
||||
|
||||
for chunk in data.chunks_mut(samples_per_10ms) {
|
||||
if unsafe {
|
||||
self.sys_handle.pin_mut().process_reverse_stream(
|
||||
chunk.as_mut_ptr(),
|
||||
chunk.len(),
|
||||
chunk.as_mut_ptr(),
|
||||
chunk.len(),
|
||||
sample_rate,
|
||||
num_channels,
|
||||
)
|
||||
} != 0
|
||||
{
|
||||
return Err(RtcError {
|
||||
error_type: RtcErrorType::Internal,
|
||||
message: "Failed to process reverse stream".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_stream_delay_ms(&mut self, delay_ms: i32) -> Result<(), RtcError> {
|
||||
if self.sys_handle.pin_mut().set_stream_delay_ms(delay_ms) == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(RtcError {
|
||||
error_type: RtcErrorType::Internal,
|
||||
message: "Failed to set stream delay".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::audio_frame::AudioFrame;
|
||||
use cxx::UniquePtr;
|
||||
use std::sync::Arc;
|
||||
use webrtc_sys::audio_mixer as sys;
|
||||
use webrtc_sys::audio_mixer::ffi;
|
||||
|
||||
pub struct AudioMixer {
|
||||
sys_handle: UniquePtr<ffi::AudioMixer>,
|
||||
}
|
||||
|
||||
pub use ffi::AudioFrameInfo;
|
||||
|
||||
pub trait AudioMixerSource {
|
||||
fn ssrc(&self) -> i32;
|
||||
fn preferred_sample_rate(&self) -> u32;
|
||||
fn get_audio_frame_with_info(&self, target_sample_rate: u32) -> Option<AudioFrame<'_>>;
|
||||
}
|
||||
|
||||
struct AudioMixerSourceImpl<T> {
|
||||
inner: T,
|
||||
}
|
||||
impl<T: AudioMixerSource> sys::AudioMixerSource for AudioMixerSourceImpl<T> {
|
||||
fn ssrc(&self) -> i32 {
|
||||
self.inner.ssrc()
|
||||
}
|
||||
|
||||
fn preferred_sample_rate(&self) -> i32 {
|
||||
self.inner.preferred_sample_rate() as i32
|
||||
}
|
||||
|
||||
fn get_audio_frame_with_info(
|
||||
&self,
|
||||
target_sample_rate: i32,
|
||||
native_frame: sys::NativeAudioFrame,
|
||||
) -> AudioFrameInfo {
|
||||
if let Some(frame) = self.inner.get_audio_frame_with_info(target_sample_rate as u32) {
|
||||
let samples_count = (frame.sample_rate as usize / 100) as usize;
|
||||
assert_eq!(
|
||||
frame.sample_rate, target_sample_rate as u32,
|
||||
"sample rate must match target_sample_rate"
|
||||
);
|
||||
assert_eq!(
|
||||
frame.samples_per_channel as usize, samples_count,
|
||||
"frame must contain 10ms of samples"
|
||||
);
|
||||
assert_eq!(
|
||||
frame.data.len(),
|
||||
samples_count * frame.num_channels as usize,
|
||||
"slice must contain 10ms of samples"
|
||||
);
|
||||
|
||||
unsafe {
|
||||
native_frame.update_frame(
|
||||
0,
|
||||
frame.data.as_ptr(),
|
||||
frame.samples_per_channel as usize,
|
||||
frame.sample_rate as i32,
|
||||
frame.num_channels as usize,
|
||||
);
|
||||
}
|
||||
return ffi::AudioFrameInfo::Normal;
|
||||
} else {
|
||||
return ffi::AudioFrameInfo::Muted;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AudioMixer {
|
||||
pub fn new() -> Self {
|
||||
let sys_handle = ffi::create_audio_mixer();
|
||||
Self { sys_handle }
|
||||
}
|
||||
|
||||
pub fn add_source(&mut self, source: impl AudioMixerSource + 'static) {
|
||||
let source_impl = AudioMixerSourceImpl { inner: source };
|
||||
let wrapper = Box::new(sys::AudioMixerSourceWrapper::new(Arc::new(source_impl)));
|
||||
unsafe {
|
||||
self.sys_handle.pin_mut().add_source(wrapper);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove_source(&mut self, ssrc: i32) {
|
||||
unsafe {
|
||||
self.sys_handle.pin_mut().remove_source(ssrc);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mix(&mut self, num_channels: usize) -> &[i16] {
|
||||
unsafe {
|
||||
let len = self.sys_handle.pin_mut().mix(num_channels);
|
||||
std::slice::from_raw_parts(self.sys_handle.data(), len)
|
||||
}
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::UniquePtr;
|
||||
use webrtc_sys::audio_resampler as sys_ar;
|
||||
|
||||
pub struct AudioResampler {
|
||||
sys_handle: UniquePtr<sys_ar::ffi::AudioResampler>,
|
||||
}
|
||||
|
||||
impl Default for AudioResampler {
|
||||
fn default() -> Self {
|
||||
Self { sys_handle: sys_ar::ffi::create_audio_resampler() }
|
||||
}
|
||||
}
|
||||
|
||||
impl AudioResampler {
|
||||
pub fn remix_and_resample<'a>(
|
||||
&'a mut self,
|
||||
src: &[i16],
|
||||
samples_per_channel: u32,
|
||||
num_channels: u32,
|
||||
sample_rate: u32,
|
||||
dst_num_channels: u32,
|
||||
dst_sample_rate: u32,
|
||||
) -> &'a [i16] {
|
||||
assert!(src.len() >= (samples_per_channel * num_channels) as usize, "src buffer too small");
|
||||
|
||||
unsafe {
|
||||
let len = self.sys_handle.pin_mut().remix_and_resample(
|
||||
src.as_ptr(),
|
||||
samples_per_channel as usize,
|
||||
num_channels as usize,
|
||||
sample_rate as i32,
|
||||
dst_num_channels as usize,
|
||||
dst_sample_rate as i32,
|
||||
);
|
||||
|
||||
std::slice::from_raw_parts(self.sys_handle.data(), len / 2)
|
||||
}
|
||||
}
|
||||
}
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use tokio::sync::oneshot;
|
||||
use webrtc_sys::audio_track as sys_at;
|
||||
|
||||
use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions, RtcError, RtcErrorType};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NativeAudioSource {
|
||||
sys_handle: SharedPtr<sys_at::ffi::AudioTrackSource>,
|
||||
sample_rate: u32,
|
||||
num_channels: u32,
|
||||
queue_size_samples: u32,
|
||||
}
|
||||
|
||||
impl NativeAudioSource {
|
||||
/// Creates a new [`NativeAudioSource`].
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `options` – Configuration options for the source (e.g. echo cancellation, noise suppression).
|
||||
/// * `sample_rate` – Sampling rate in Hz (for example, `48000`).
|
||||
/// * `num_channels` – Number of audio channels (`1` for mono, `2` for stereo, etc.).
|
||||
/// * `queue_size_ms` – Size of the internal buffering queue, in milliseconds.
|
||||
///
|
||||
/// # Behavior
|
||||
/// - If `queue_size_ms` is **zero**, buffering is **disabled** and audio frames are
|
||||
/// delivered directly to webrtc sinks. In this mode, the caller **must provide 10 ms frames**
|
||||
/// (i.e., `sample_rate / 100` samples per channel) when calling [`capture_frame`].
|
||||
/// - If `queue_size_ms` is **non-zero**, buffering is enabled. The value must be a
|
||||
/// **multiple of 10**, representing the total buffering duration in milliseconds.
|
||||
/// Frames will be queued and flushed to sinks asynchronously once the buffer
|
||||
/// reaches the configured threshold.
|
||||
///
|
||||
/// # Panics
|
||||
/// assert if `queue_size_ms` is not a multiple of 10.
|
||||
pub fn new(
|
||||
options: AudioSourceOptions,
|
||||
sample_rate: u32,
|
||||
num_channels: u32,
|
||||
queue_size_ms: u32,
|
||||
) -> NativeAudioSource {
|
||||
assert!(queue_size_ms % 10 == 0, "queue_size_ms must be a multiple of 10");
|
||||
|
||||
let sys_handle = sys_at::ffi::new_audio_track_source(
|
||||
options.into(),
|
||||
sample_rate.try_into().unwrap(),
|
||||
num_channels.try_into().unwrap(),
|
||||
queue_size_ms.try_into().unwrap(),
|
||||
);
|
||||
|
||||
let queue_size_samples = (queue_size_ms * sample_rate * num_channels) / 1000;
|
||||
Self { sys_handle, sample_rate, num_channels, queue_size_samples }
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> SharedPtr<sys_at::ffi::AudioTrackSource> {
|
||||
self.sys_handle.clone()
|
||||
}
|
||||
|
||||
pub fn set_audio_options(&self, options: AudioSourceOptions) {
|
||||
self.sys_handle.set_audio_options(&sys_at::ffi::AudioSourceOptions::from(options))
|
||||
}
|
||||
|
||||
pub fn audio_options(&self) -> AudioSourceOptions {
|
||||
self.sys_handle.audio_options().into()
|
||||
}
|
||||
|
||||
pub fn sample_rate(&self) -> u32 {
|
||||
self.sample_rate
|
||||
}
|
||||
|
||||
pub fn num_channels(&self) -> u32 {
|
||||
self.num_channels
|
||||
}
|
||||
|
||||
pub fn clear_buffer(&self) {
|
||||
self.sys_handle.clear_buffer();
|
||||
}
|
||||
|
||||
pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> {
|
||||
if self.sample_rate != frame.sample_rate || self.num_channels != frame.num_channels {
|
||||
return Err(RtcError {
|
||||
error_type: RtcErrorType::InvalidState,
|
||||
message: "sample_rate and num_channels don't match".to_owned(),
|
||||
});
|
||||
}
|
||||
|
||||
// Fast path: no buffering
|
||||
if self.queue_size_samples == 0 {
|
||||
// frame size must be 10ms for fast path
|
||||
let expected_frames_per_ch = (self.sample_rate / 100) as usize;
|
||||
if frame.data.len() % (self.num_channels as usize) != 0 {
|
||||
return Err(RtcError {
|
||||
error_type: RtcErrorType::InvalidState,
|
||||
message: "frame.data length not divisible by channel count".to_owned(),
|
||||
});
|
||||
}
|
||||
let nb_frames = frame.data.len() / (self.num_channels as usize);
|
||||
if nb_frames != expected_frames_per_ch {
|
||||
return Err(RtcError {
|
||||
error_type: RtcErrorType::InvalidState,
|
||||
message: format!(
|
||||
"direct capture requires 10ms frames: got {} frames, expected {}",
|
||||
nb_frames, expected_frames_per_ch
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
// Define a no-op callback for fast path (queue_size_ms=0)
|
||||
// This is safer than passing null, which can cause UB in release mode optimizations
|
||||
extern "C" fn noop_complete_callback(_ctx: *const sys_at::SourceContext) {
|
||||
// No-op: fast path completes synchronously, no callback needed
|
||||
}
|
||||
|
||||
unsafe {
|
||||
let data: &[i16] = frame.data.as_ref();
|
||||
// Use a valid no-op callback instead of null for safety
|
||||
// In release mode, transmuting null pointers can cause UB
|
||||
let noop_callback = sys_at::CompleteCallback(noop_complete_callback);
|
||||
let ok = self.sys_handle.capture_frame(
|
||||
data,
|
||||
self.sample_rate,
|
||||
self.num_channels,
|
||||
nb_frames,
|
||||
std::ptr::null(), // Context is still null - callback won't use it
|
||||
noop_callback,
|
||||
);
|
||||
if !ok {
|
||||
return Err(RtcError {
|
||||
error_type: RtcErrorType::InvalidState,
|
||||
message: "failed to capture frame without buffering".to_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Buffered path.
|
||||
extern "C" fn lk_audio_source_complete(userdata: *const sys_at::SourceContext) {
|
||||
let tx = unsafe { Box::from_raw(userdata as *mut oneshot::Sender<()>) };
|
||||
let _ = tx.send(());
|
||||
}
|
||||
|
||||
// iterate over chunks of self._queue_size_samples
|
||||
for chunk in frame.data.chunks(self.queue_size_samples as usize) {
|
||||
let nb_frames = chunk.len() / self.num_channels as usize;
|
||||
let (tx, rx) = oneshot::channel::<()>();
|
||||
let ctx = Box::new(tx);
|
||||
let ctx_ptr = Box::into_raw(ctx) as *const sys_at::SourceContext;
|
||||
|
||||
unsafe {
|
||||
// In the fast path, C++ never store / invoke on_complete / ctx.
|
||||
if !self.sys_handle.capture_frame(
|
||||
chunk,
|
||||
self.sample_rate,
|
||||
self.num_channels,
|
||||
nb_frames,
|
||||
ctx_ptr,
|
||||
sys_at::CompleteCallback(lk_audio_source_complete),
|
||||
) {
|
||||
return Err(RtcError {
|
||||
error_type: RtcErrorType::InvalidState,
|
||||
message: "failed to capture frame".to_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let _ = rx.await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_at::ffi::AudioSourceOptions> for AudioSourceOptions {
|
||||
fn from(options: sys_at::ffi::AudioSourceOptions) -> Self {
|
||||
Self {
|
||||
echo_cancellation: options.echo_cancellation,
|
||||
noise_suppression: options.noise_suppression,
|
||||
auto_gain_control: options.auto_gain_control,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AudioSourceOptions> for sys_at::ffi::AudioSourceOptions {
|
||||
fn from(options: AudioSourceOptions) -> Self {
|
||||
Self {
|
||||
echo_cancellation: options.echo_cancellation,
|
||||
noise_suppression: options.noise_suppression,
|
||||
auto_gain_control: options.auto_gain_control,
|
||||
}
|
||||
}
|
||||
}
|
||||
+314
@@ -0,0 +1,314 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
task::{Context, Poll, Waker},
|
||||
};
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use livekit_runtime::Stream;
|
||||
use parking_lot::Mutex;
|
||||
use rtrb::{Consumer, Producer, PushError, RingBuffer};
|
||||
use webrtc_sys::audio_track as sys_at;
|
||||
|
||||
use crate::{audio_frame::AudioFrame, audio_track::RtcAudioTrack};
|
||||
|
||||
pub struct NativeAudioStream {
|
||||
native_sink: SharedPtr<sys_at::ffi::NativeAudioSink>,
|
||||
audio_track: RtcAudioTrack,
|
||||
frame_queue: Arc<AudioFrameQueue>,
|
||||
}
|
||||
|
||||
impl NativeAudioStream {
|
||||
pub fn new(
|
||||
audio_track: RtcAudioTrack,
|
||||
sample_rate: i32,
|
||||
num_channels: i32,
|
||||
queue_size_frames: Option<usize>,
|
||||
) -> Self {
|
||||
let frame_queue = Arc::new(AudioFrameQueue::new(queue_size_frames));
|
||||
let observer = Arc::new(AudioTrackObserver { frame_queue: frame_queue.clone() });
|
||||
let native_sink = sys_at::ffi::new_native_audio_sink(
|
||||
Box::new(sys_at::AudioSinkWrapper::new(observer.clone())),
|
||||
sample_rate,
|
||||
num_channels,
|
||||
);
|
||||
|
||||
let audio = unsafe { sys_at::ffi::media_to_audio(audio_track.sys_handle()) };
|
||||
audio.add_sink(&native_sink);
|
||||
|
||||
Self { native_sink, audio_track, frame_queue }
|
||||
}
|
||||
|
||||
pub fn track(&self) -> RtcAudioTrack {
|
||||
self.audio_track.clone()
|
||||
}
|
||||
|
||||
pub fn close(&mut self) {
|
||||
let audio = unsafe { sys_at::ffi::media_to_audio(self.audio_track.sys_handle()) };
|
||||
audio.remove_sink(&self.native_sink);
|
||||
|
||||
self.frame_queue.close();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for NativeAudioStream {
|
||||
fn drop(&mut self) {
|
||||
self.close();
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for NativeAudioStream {
|
||||
type Item = AudioFrame<'static>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
self.frame_queue.poll_recv(cx)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AudioTrackObserver {
|
||||
frame_queue: Arc<AudioFrameQueue>,
|
||||
}
|
||||
|
||||
impl sys_at::AudioSink for AudioTrackObserver {
|
||||
fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) {
|
||||
self.frame_queue.push(AudioFrame {
|
||||
data: data.to_owned().into(),
|
||||
sample_rate: sample_rate as u32,
|
||||
num_channels: nb_channels as u32,
|
||||
samples_per_channel: nb_frames as u32,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
struct AudioFrameQueue {
|
||||
kind: AudioFrameQueueKind,
|
||||
closed: AtomicBool,
|
||||
dropped_frames: AtomicU64,
|
||||
waker: Mutex<Option<Waker>>,
|
||||
}
|
||||
|
||||
enum AudioFrameQueueKind {
|
||||
Bounded(BoundedAudioFrameQueue),
|
||||
Unbounded(UnboundedAudioFrameQueue),
|
||||
}
|
||||
|
||||
struct BoundedAudioFrameQueue {
|
||||
producer: Mutex<Producer<AudioFrame<'static>>>,
|
||||
consumer: Mutex<Consumer<AudioFrame<'static>>>,
|
||||
}
|
||||
|
||||
struct UnboundedAudioFrameQueue {
|
||||
frames: Mutex<VecDeque<AudioFrame<'static>>>,
|
||||
}
|
||||
|
||||
impl AudioFrameQueue {
|
||||
fn new(capacity: Option<usize>) -> Self {
|
||||
let kind = match capacity.filter(|capacity| *capacity > 0) {
|
||||
Some(capacity) => {
|
||||
let (producer, consumer) = RingBuffer::new(capacity);
|
||||
AudioFrameQueueKind::Bounded(BoundedAudioFrameQueue {
|
||||
producer: Mutex::new(producer),
|
||||
consumer: Mutex::new(consumer),
|
||||
})
|
||||
}
|
||||
None => AudioFrameQueueKind::Unbounded(UnboundedAudioFrameQueue {
|
||||
frames: Mutex::new(VecDeque::new()),
|
||||
}),
|
||||
};
|
||||
|
||||
Self {
|
||||
kind,
|
||||
closed: AtomicBool::new(false),
|
||||
dropped_frames: AtomicU64::new(0),
|
||||
waker: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn push(&self, frame: AudioFrame<'static>) {
|
||||
if self.closed.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
|
||||
match &self.kind {
|
||||
AudioFrameQueueKind::Bounded(queue) => self.push_bounded(queue, frame),
|
||||
AudioFrameQueueKind::Unbounded(queue) => {
|
||||
queue.frames.lock().push_back(frame);
|
||||
}
|
||||
}
|
||||
|
||||
self.wake_receiver();
|
||||
}
|
||||
|
||||
fn push_bounded(&self, queue: &BoundedAudioFrameQueue, mut frame: AudioFrame<'static>) {
|
||||
loop {
|
||||
let push_result = queue.producer.lock().push(frame);
|
||||
match push_result {
|
||||
Ok(()) => return,
|
||||
Err(PushError::Full(returned_frame)) => {
|
||||
frame = returned_frame;
|
||||
|
||||
let dropped = queue.consumer.lock().pop().is_ok();
|
||||
|
||||
if dropped {
|
||||
self.record_drop();
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&self) {
|
||||
self.closed.store(true, Ordering::Release);
|
||||
self.wake_receiver();
|
||||
|
||||
match &self.kind {
|
||||
AudioFrameQueueKind::Bounded(queue) => {
|
||||
let mut consumer = queue.consumer.lock();
|
||||
while consumer.pop().is_ok() {}
|
||||
}
|
||||
AudioFrameQueueKind::Unbounded(queue) => {
|
||||
queue.frames.lock().clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Option<AudioFrame<'static>>> {
|
||||
if let Some(frame) = self.try_pop() {
|
||||
return Poll::Ready(Some(frame));
|
||||
}
|
||||
|
||||
if self.closed.load(Ordering::Acquire) {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
|
||||
*self.waker.lock() = Some(cx.waker().clone());
|
||||
|
||||
if let Some(frame) = self.try_pop() {
|
||||
self.waker.lock().take();
|
||||
Poll::Ready(Some(frame))
|
||||
} else if self.closed.load(Ordering::Acquire) {
|
||||
Poll::Ready(None)
|
||||
} else {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
fn try_pop(&self) -> Option<AudioFrame<'static>> {
|
||||
match &self.kind {
|
||||
AudioFrameQueueKind::Bounded(queue) => queue.consumer.lock().pop().ok(),
|
||||
AudioFrameQueueKind::Unbounded(queue) => queue.frames.lock().pop_front(),
|
||||
}
|
||||
}
|
||||
|
||||
fn wake_receiver(&self) {
|
||||
let waker = self.waker.lock().take();
|
||||
if let Some(waker) = waker {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
|
||||
fn record_drop(&self) {
|
||||
let dropped_frames = self.dropped_frames.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if dropped_frames == 1 || dropped_frames % 100 == 0 {
|
||||
log::warn!(
|
||||
"native audio stream queue overflow; dropped {} queued frames",
|
||||
dropped_frames
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use super::AudioFrameQueue;
|
||||
use crate::audio_frame::AudioFrame;
|
||||
|
||||
fn test_frame(marker: i16) -> AudioFrame<'static> {
|
||||
AudioFrame {
|
||||
data: vec![marker].into(),
|
||||
sample_rate: 48_000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn pop_marker(queue: &AudioFrameQueue) -> Option<i16> {
|
||||
queue.try_pop().map(|frame| frame.data[0])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_queue_preserves_fifo_order_under_capacity() {
|
||||
let queue = AudioFrameQueue::new(Some(3));
|
||||
|
||||
queue.push(test_frame(1));
|
||||
queue.push(test_frame(2));
|
||||
queue.push(test_frame(3));
|
||||
|
||||
assert_eq!(pop_marker(&queue), Some(1));
|
||||
assert_eq!(pop_marker(&queue), Some(2));
|
||||
assert_eq!(pop_marker(&queue), Some(3));
|
||||
assert_eq!(pop_marker(&queue), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_queue_drops_oldest_when_full() {
|
||||
let queue = AudioFrameQueue::new(Some(2));
|
||||
|
||||
queue.push(test_frame(1));
|
||||
queue.push(test_frame(2));
|
||||
queue.push(test_frame(3));
|
||||
|
||||
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(pop_marker(&queue), Some(2));
|
||||
assert_eq!(pop_marker(&queue), Some(3));
|
||||
assert_eq!(pop_marker(&queue), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unbounded_queue_retains_all_frames() {
|
||||
let queue = AudioFrameQueue::new(None);
|
||||
|
||||
for marker in 1..=4 {
|
||||
queue.push(test_frame(marker));
|
||||
}
|
||||
|
||||
for marker in 1..=4 {
|
||||
assert_eq!(pop_marker(&queue), Some(marker));
|
||||
}
|
||||
assert_eq!(pop_marker(&queue), None);
|
||||
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn close_clears_buffer_and_rejects_future_pushes() {
|
||||
let queue = AudioFrameQueue::new(Some(2));
|
||||
|
||||
queue.push(test_frame(1));
|
||||
queue.close();
|
||||
queue.push(test_frame(2));
|
||||
|
||||
assert_eq!(pop_marker(&queue), None);
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use sys_at::ffi::audio_to_media;
|
||||
use webrtc_sys::audio_track as sys_at;
|
||||
|
||||
use super::media_stream_track::impl_media_stream_track;
|
||||
use crate::media_stream_track::RtcTrackState;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtcAudioTrack {
|
||||
pub(crate) sys_handle: SharedPtr<sys_at::ffi::AudioTrack>,
|
||||
}
|
||||
|
||||
impl RtcAudioTrack {
|
||||
impl_media_stream_track!(audio_to_media);
|
||||
|
||||
pub fn sys_handle(&self) -> SharedPtr<sys_at::ffi::MediaStreamTrack> {
|
||||
audio_to_media(self.sys_handle.clone())
|
||||
}
|
||||
}
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{str, sync::Arc};
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use parking_lot::Mutex;
|
||||
use webrtc_sys::data_channel as sys_dc;
|
||||
|
||||
use crate::data_channel::{
|
||||
DataBuffer, DataChannelError, DataChannelInit, DataChannelState, OnBufferedAmountChange,
|
||||
OnMessage, OnStateChange,
|
||||
};
|
||||
|
||||
impl From<sys_dc::ffi::DataState> for DataChannelState {
|
||||
fn from(value: sys_dc::ffi::DataState) -> Self {
|
||||
match value {
|
||||
sys_dc::ffi::DataState::Connecting => Self::Connecting,
|
||||
sys_dc::ffi::DataState::Open => Self::Open,
|
||||
sys_dc::ffi::DataState::Closing => Self::Closing,
|
||||
sys_dc::ffi::DataState::Closed => Self::Closed,
|
||||
_ => panic!("unknown data channel state"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<DataChannelInit> for sys_dc::ffi::DataChannelInit {
|
||||
fn from(value: DataChannelInit) -> Self {
|
||||
Self {
|
||||
ordered: value.ordered,
|
||||
has_max_retransmit_time: value.max_retransmit_time.is_some(),
|
||||
max_retransmit_time: value.max_retransmit_time.unwrap_or_default(),
|
||||
has_max_retransmits: value.max_retransmits.is_some(),
|
||||
max_retransmits: value.max_retransmits.unwrap_or_default(),
|
||||
protocol: value.protocol,
|
||||
id: value.id,
|
||||
has_priority: false,
|
||||
priority: sys_dc::ffi::Priority::Medium,
|
||||
negotiated: value.negotiated,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct DataChannel {
|
||||
observer: Arc<DataChannelObserver>,
|
||||
pub(crate) sys_handle: SharedPtr<sys_dc::ffi::DataChannel>,
|
||||
}
|
||||
|
||||
impl DataChannel {
|
||||
pub fn configure(sys_handle: SharedPtr<sys_dc::ffi::DataChannel>) -> Self {
|
||||
let observer = Arc::new(DataChannelObserver::default());
|
||||
let dc = Self { sys_handle: sys_handle.clone(), observer: observer.clone() };
|
||||
|
||||
dc.sys_handle
|
||||
.register_observer(Box::new(sys_dc::DataChannelObserverWrapper::new(observer)));
|
||||
dc
|
||||
}
|
||||
|
||||
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
|
||||
if !binary {
|
||||
str::from_utf8(data)?;
|
||||
}
|
||||
|
||||
let buffer = sys_dc::ffi::DataBuffer { ptr: data.as_ptr(), len: data.len(), binary };
|
||||
|
||||
self.sys_handle.send(&buffer).then_some(()).ok_or(DataChannelError::Send)
|
||||
}
|
||||
|
||||
pub fn id(&self) -> i32 {
|
||||
self.sys_handle.id()
|
||||
}
|
||||
|
||||
pub fn label(&self) -> String {
|
||||
self.sys_handle.label()
|
||||
}
|
||||
|
||||
pub fn state(&self) -> DataChannelState {
|
||||
self.sys_handle.state().into()
|
||||
}
|
||||
|
||||
pub fn close(&self) {
|
||||
self.sys_handle.close();
|
||||
}
|
||||
|
||||
pub fn buffered_amount(&self) -> u64 {
|
||||
self.sys_handle.buffered_amount()
|
||||
}
|
||||
|
||||
pub fn on_state_change(&self, handler: Option<OnStateChange>) {
|
||||
*self.observer.state_change_handler.lock() = handler;
|
||||
}
|
||||
|
||||
pub fn on_message(&self, handler: Option<OnMessage>) {
|
||||
*self.observer.message_handler.lock() = handler;
|
||||
}
|
||||
|
||||
pub fn on_buffered_amount_change(&self, handler: Option<OnBufferedAmountChange>) {
|
||||
*self.observer.buffered_amount_change_handler.lock() = handler;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DataChannelObserver {
|
||||
state_change_handler: Mutex<Option<OnStateChange>>,
|
||||
message_handler: Mutex<Option<OnMessage>>,
|
||||
buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChange>>,
|
||||
}
|
||||
|
||||
impl sys_dc::DataChannelObserver for DataChannelObserver {
|
||||
fn on_state_change(&self, state: sys_dc::ffi::DataState) {
|
||||
let mut handler = self.state_change_handler.lock();
|
||||
if let Some(f) = handler.as_mut() {
|
||||
f(state.into());
|
||||
}
|
||||
}
|
||||
|
||||
fn on_message(&self, data: &[u8], binary: bool) {
|
||||
let mut handler = self.message_handler.lock();
|
||||
if let Some(f) = handler.as_mut() {
|
||||
f(DataBuffer { data, binary });
|
||||
}
|
||||
}
|
||||
|
||||
fn on_buffered_amount_change(&self, sent_data_size: u64) {
|
||||
let mut handler = self.buffered_amount_change_handler.lock();
|
||||
if let Some(f) = handler.as_mut() {
|
||||
f(sent_data_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
+241
@@ -0,0 +1,241 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::UniquePtr;
|
||||
use webrtc_sys::desktop_capturer::{self as sys_dc, ffi::new_desktop_capturer};
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq)]
|
||||
pub(crate) enum SourceType {
|
||||
Screen,
|
||||
Window,
|
||||
Generic,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub(crate) struct DesktopCapturerOptions {
|
||||
source_type: SourceType,
|
||||
include_cursor: bool,
|
||||
#[cfg(target_os = "macos")]
|
||||
allow_sck_system_picker: bool,
|
||||
}
|
||||
|
||||
impl Default for DesktopCapturerOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
source_type: SourceType::Screen,
|
||||
include_cursor: false,
|
||||
#[cfg(target_os = "macos")]
|
||||
allow_sck_system_picker: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DesktopCapturerOptions {
|
||||
pub(crate) fn new(source_type: SourceType) -> Self {
|
||||
Self { source_type, ..Default::default() }
|
||||
}
|
||||
|
||||
pub(crate) fn with_cursor(mut self, include: bool) -> Self {
|
||||
self.include_cursor = include;
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) fn with_sck_system_picker(mut self, allow_sck_system_picker: bool) -> Self {
|
||||
self.allow_sck_system_picker = allow_sck_system_picker;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn to_sys_handle(&self) -> sys_dc::ffi::DesktopCapturerOptions {
|
||||
let source_type = match self.source_type {
|
||||
SourceType::Screen => sys_dc::ffi::SourceType::Screen,
|
||||
SourceType::Window => sys_dc::ffi::SourceType::Window,
|
||||
SourceType::Generic => sys_dc::ffi::SourceType::Generic,
|
||||
};
|
||||
let mut sys_handle = sys_dc::ffi::DesktopCapturerOptions {
|
||||
source_type,
|
||||
include_cursor: self.include_cursor,
|
||||
allow_sck_system_picker: false,
|
||||
};
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
sys_handle.allow_sck_system_picker = self.allow_sck_system_picker;
|
||||
}
|
||||
sys_handle
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DesktopCapturer {
|
||||
sys_handle: UniquePtr<sys_dc::ffi::DesktopCapturer>,
|
||||
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
|
||||
glib_loop: Option<glib::MainLoop>,
|
||||
}
|
||||
|
||||
impl DesktopCapturer {
|
||||
pub(crate) fn new(options: DesktopCapturerOptions) -> Option<Self> {
|
||||
let sys_handle = new_desktop_capturer(options.to_sys_handle());
|
||||
if sys_handle.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(Self {
|
||||
sys_handle,
|
||||
#[cfg(all(
|
||||
any(target_os = "linux", target_os = "freebsd"),
|
||||
feature = "glib-main-loop"
|
||||
))]
|
||||
glib_loop: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn capture_frame(&self) {
|
||||
self.sys_handle.capture_frame();
|
||||
}
|
||||
|
||||
pub(crate) fn start<T>(&mut self, callback: T)
|
||||
where
|
||||
T: FnMut(Result<DesktopFrame, CaptureError>) + Send + 'static,
|
||||
{
|
||||
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
|
||||
if std::env::var("WAYLAND_DISPLAY").is_ok() {
|
||||
let main_loop = glib::MainLoop::new(None, false);
|
||||
self.glib_loop = Some(main_loop.clone());
|
||||
let _handle = std::thread::spawn(move || {
|
||||
main_loop.run();
|
||||
});
|
||||
}
|
||||
let pin_handle = self.sys_handle.pin_mut();
|
||||
let callback = DesktopCallback::new(callback);
|
||||
let callback_wrapper = sys_dc::DesktopCapturerCallbackWrapper::new(Box::new(callback));
|
||||
pin_handle.start(Box::new(callback_wrapper));
|
||||
}
|
||||
|
||||
pub(crate) fn select_source(&self, id: u64) -> bool {
|
||||
self.sys_handle.select_source(id)
|
||||
}
|
||||
|
||||
pub(crate) fn get_source_list(&self) -> Vec<CaptureSource> {
|
||||
let mut sources = Vec::new();
|
||||
let source_list = self.sys_handle.get_source_list();
|
||||
for source in source_list.iter() {
|
||||
sources.push(CaptureSource { sys_handle: source.clone() });
|
||||
}
|
||||
sources
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
|
||||
impl Drop for DesktopCapturer {
|
||||
fn drop(&mut self) {
|
||||
if let Some(glib_loop) = &self.glib_loop {
|
||||
glib_loop.quit();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DesktopFrame {
|
||||
sys_handle: UniquePtr<sys_dc::ffi::DesktopFrame>,
|
||||
}
|
||||
|
||||
impl DesktopFrame {
|
||||
fn new(sys_handle: UniquePtr<sys_dc::ffi::DesktopFrame>) -> Self {
|
||||
Self { sys_handle }
|
||||
}
|
||||
|
||||
pub(crate) fn width(&self) -> i32 {
|
||||
self.sys_handle.width()
|
||||
}
|
||||
|
||||
pub(crate) fn height(&self) -> i32 {
|
||||
self.sys_handle.height()
|
||||
}
|
||||
|
||||
pub(crate) fn stride(&self) -> u32 {
|
||||
self.sys_handle.stride() as u32
|
||||
}
|
||||
|
||||
pub(crate) fn left(&self) -> i32 {
|
||||
self.sys_handle.left()
|
||||
}
|
||||
|
||||
pub(crate) fn top(&self) -> i32 {
|
||||
self.sys_handle.top()
|
||||
}
|
||||
|
||||
pub(crate) fn data(&self) -> &[u8] {
|
||||
let data = self.sys_handle.data();
|
||||
unsafe { std::slice::from_raw_parts(data, self.stride() as usize * self.height() as usize) }
|
||||
}
|
||||
}
|
||||
|
||||
struct DesktopCallback<T: FnMut(Result<DesktopFrame, CaptureError>) + Send> {
|
||||
callback: T,
|
||||
}
|
||||
|
||||
impl<T> DesktopCallback<T>
|
||||
where
|
||||
T: FnMut(Result<DesktopFrame, CaptureError>) + Send,
|
||||
{
|
||||
fn new(callback: T) -> Self {
|
||||
Self { callback }
|
||||
}
|
||||
|
||||
fn capture_result_from_sys(
|
||||
result: Result<UniquePtr<sys_dc::ffi::DesktopFrame>, sys_dc::CaptureError>,
|
||||
) -> Result<DesktopFrame, CaptureError> {
|
||||
match result {
|
||||
Ok(frame) => Ok(DesktopFrame::new(frame)),
|
||||
Err(error) => Err(match error {
|
||||
sys_dc::CaptureError::Temporary => CaptureError::Temporary,
|
||||
sys_dc::CaptureError::Permanent => CaptureError::Permanent,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> sys_dc::DesktopCapturerCallback for DesktopCallback<T>
|
||||
where
|
||||
T: FnMut(Result<DesktopFrame, CaptureError>) + Send,
|
||||
{
|
||||
fn on_capture_result(
|
||||
&mut self,
|
||||
result: Result<UniquePtr<sys_dc::ffi::DesktopFrame>, sys_dc::CaptureError>,
|
||||
) {
|
||||
(self.callback)(DesktopCallback::<T>::capture_result_from_sys(result));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CaptureSource {
|
||||
sys_handle: sys_dc::ffi::Source,
|
||||
}
|
||||
|
||||
impl CaptureSource {
|
||||
pub(crate) fn id(&self) -> u64 {
|
||||
self.sys_handle.id
|
||||
}
|
||||
|
||||
pub(crate) fn title(&self) -> String {
|
||||
self.sys_handle.title.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn display_id(&self) -> i64 {
|
||||
self.sys_handle.display_id
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) enum CaptureError {
|
||||
Temporary,
|
||||
Permanent,
|
||||
}
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use parking_lot::Mutex;
|
||||
use webrtc_sys::frame_cryptor::{self as sys_fc};
|
||||
|
||||
use crate::{
|
||||
native::packet_trailer::PacketTrailerHandler, peer_connection_factory::PeerConnectionFactory,
|
||||
rtp_receiver::RtpReceiver, rtp_sender::RtpSender,
|
||||
};
|
||||
|
||||
pub type OnStateChange = Box<dyn FnMut(String, EncryptionState) + Send + Sync>;
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
#[non_exhaustive]
|
||||
pub enum KeyDerivationAlgorithm {
|
||||
PBKDF2,
|
||||
HKDF,
|
||||
}
|
||||
impl Into<sys_fc::ffi::KeyDerivationAlgorithm> for KeyDerivationAlgorithm {
|
||||
fn into(self) -> sys_fc::ffi::KeyDerivationAlgorithm {
|
||||
match self {
|
||||
KeyDerivationAlgorithm::PBKDF2 => sys_fc::ffi::KeyDerivationAlgorithm::PBKDF2,
|
||||
KeyDerivationAlgorithm::HKDF => sys_fc::ffi::KeyDerivationAlgorithm::HKDF,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct KeyProviderOptions {
|
||||
pub shared_key: bool,
|
||||
pub ratchet_window_size: i32,
|
||||
pub ratchet_salt: Vec<u8>,
|
||||
pub failure_tolerance: i32,
|
||||
pub key_ring_size: i32,
|
||||
pub key_derivation_algorithm: KeyDerivationAlgorithm,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EncryptionAlgorithm {
|
||||
AesGcm,
|
||||
AesCbc,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EncryptionState {
|
||||
New,
|
||||
Ok,
|
||||
EncryptionFailed,
|
||||
DecryptionFailed,
|
||||
MissingKey,
|
||||
KeyRatcheted,
|
||||
InternalError,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EncryptedPacket {
|
||||
pub data: Vec<u8>,
|
||||
pub iv: Vec<u8>,
|
||||
pub key_index: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct KeyProvider {
|
||||
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::KeyProvider>,
|
||||
}
|
||||
|
||||
impl KeyProvider {
|
||||
pub fn new(options: KeyProviderOptions) -> Self {
|
||||
Self { sys_handle: sys_fc::ffi::new_key_provider(options.into()) }
|
||||
}
|
||||
|
||||
pub fn set_shared_key(&self, key_index: i32, key: Vec<u8>) -> bool {
|
||||
self.sys_handle.set_shared_key(key_index, key)
|
||||
}
|
||||
|
||||
pub fn ratchet_shared_key(&self, key_index: i32) -> Option<Vec<u8>> {
|
||||
self.sys_handle.ratchet_shared_key(key_index).ok()
|
||||
}
|
||||
|
||||
pub fn get_shared_key(&self, key_index: i32) -> Option<Vec<u8>> {
|
||||
self.sys_handle.get_shared_key(key_index).ok()
|
||||
}
|
||||
|
||||
pub fn set_key(&self, participant_id: String, key_index: i32, key: Vec<u8>) -> bool {
|
||||
self.sys_handle.set_key(participant_id, key_index, key)
|
||||
}
|
||||
|
||||
pub fn ratchet_key(&self, participant_id: String, key_index: i32) -> Option<Vec<u8>> {
|
||||
self.sys_handle.ratchet_key(participant_id, key_index).ok()
|
||||
}
|
||||
|
||||
pub fn get_key(&self, participant_id: String, key_index: i32) -> Option<Vec<u8>> {
|
||||
self.sys_handle.get_key(participant_id, key_index).ok()
|
||||
}
|
||||
|
||||
pub fn set_sif_trailer(&self, trailer: Vec<u8>) {
|
||||
self.sys_handle.set_sif_trailer(trailer);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct FrameCryptor {
|
||||
observer: Arc<RtcFrameCryptorObserver>,
|
||||
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::FrameCryptor>,
|
||||
}
|
||||
|
||||
impl FrameCryptor {
|
||||
pub fn new_for_rtp_sender(
|
||||
peer_factory: &PeerConnectionFactory,
|
||||
participant_id: String,
|
||||
algorithm: EncryptionAlgorithm,
|
||||
key_provider: KeyProvider,
|
||||
sender: RtpSender,
|
||||
) -> Self {
|
||||
let observer = Arc::new(RtcFrameCryptorObserver::default());
|
||||
let sys_handle = sys_fc::ffi::new_frame_cryptor_for_rtp_sender(
|
||||
peer_factory.handle.sys_handle.clone(),
|
||||
participant_id,
|
||||
algorithm.into(),
|
||||
key_provider.sys_handle,
|
||||
sender.handle.sys_handle,
|
||||
);
|
||||
let fc = Self { observer: observer.clone(), sys_handle: sys_handle.clone() };
|
||||
fc.sys_handle
|
||||
.register_observer(Box::new(sys_fc::RtcFrameCryptorObserverWrapper::new(observer)));
|
||||
fc
|
||||
}
|
||||
|
||||
pub fn new_for_rtp_receiver(
|
||||
peer_factory: &PeerConnectionFactory,
|
||||
participant_id: String,
|
||||
algorithm: EncryptionAlgorithm,
|
||||
key_provider: KeyProvider,
|
||||
receiver: RtpReceiver,
|
||||
) -> Self {
|
||||
let observer = Arc::new(RtcFrameCryptorObserver::default());
|
||||
let sys_handle = sys_fc::ffi::new_frame_cryptor_for_rtp_receiver(
|
||||
peer_factory.handle.sys_handle.clone(),
|
||||
participant_id,
|
||||
algorithm.into(),
|
||||
key_provider.sys_handle,
|
||||
receiver.handle.sys_handle,
|
||||
);
|
||||
let fc = Self { observer: observer.clone(), sys_handle: sys_handle.clone() };
|
||||
fc.sys_handle
|
||||
.register_observer(Box::new(sys_fc::RtcFrameCryptorObserverWrapper::new(observer)));
|
||||
fc
|
||||
}
|
||||
|
||||
pub fn set_enabled(self: &FrameCryptor, enabled: bool) {
|
||||
self.sys_handle.set_enabled(enabled);
|
||||
}
|
||||
|
||||
pub fn enabled(self: &FrameCryptor) -> bool {
|
||||
self.sys_handle.enabled()
|
||||
}
|
||||
|
||||
pub fn set_key_index(self: &FrameCryptor, index: i32) {
|
||||
self.sys_handle.set_key_index(index);
|
||||
}
|
||||
|
||||
pub fn key_index(self: &FrameCryptor) -> i32 {
|
||||
self.sys_handle.key_index()
|
||||
}
|
||||
|
||||
pub fn participant_id(self: &FrameCryptor) -> String {
|
||||
self.sys_handle.participant_id()
|
||||
}
|
||||
|
||||
pub fn on_state_change(&self, handler: Option<OnStateChange>) {
|
||||
*self.observer.state_change_handler.lock() = handler;
|
||||
}
|
||||
|
||||
pub fn set_packet_trailer_handler(&self, handler: &PacketTrailerHandler) {
|
||||
self.sys_handle.set_packet_trailer_handler(handler.sys_handle());
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct DataPacketCryptor {
|
||||
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::DataPacketCryptor>,
|
||||
}
|
||||
|
||||
impl DataPacketCryptor {
|
||||
pub fn new(algorithm: EncryptionAlgorithm, key_provider: KeyProvider) -> Self {
|
||||
Self {
|
||||
sys_handle: sys_fc::ffi::new_data_packet_cryptor(
|
||||
algorithm.into(),
|
||||
key_provider.sys_handle,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encrypt(
|
||||
&self,
|
||||
participant_id: &str,
|
||||
key_index: u32,
|
||||
data: &[u8],
|
||||
) -> Result<EncryptedPacket, Box<dyn std::error::Error>> {
|
||||
let data_vec: Vec<u8> = data.to_vec();
|
||||
match self.sys_handle.encrypt_data_packet(participant_id.to_string(), key_index, data_vec) {
|
||||
Ok(packet) => Ok(packet.into()),
|
||||
Err(e) => Err(format!("Encryption failed: {}", e).into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decrypt(
|
||||
&self,
|
||||
participant_id: &str,
|
||||
encrypted_packet: &EncryptedPacket,
|
||||
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
|
||||
match self
|
||||
.sys_handle
|
||||
.decrypt_data_packet(participant_id.to_string(), &encrypted_packet.clone().into())
|
||||
{
|
||||
Ok(data) => Ok(data.into_iter().collect()),
|
||||
Err(e) => Err(format!("Decryption failed: {}", e).into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RtcFrameCryptorObserver {
|
||||
state_change_handler: Mutex<Option<OnStateChange>>,
|
||||
}
|
||||
|
||||
impl sys_fc::RtcFrameCryptorObserver for RtcFrameCryptorObserver {
|
||||
fn on_frame_cryption_state_change(
|
||||
&self,
|
||||
participant_id: String,
|
||||
state: sys_fc::ffi::FrameCryptionState,
|
||||
) {
|
||||
let mut handler = self.state_change_handler.lock();
|
||||
if let Some(f) = handler.as_mut() {
|
||||
f(participant_id, state.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_fc::ffi::Algorithm> for EncryptionAlgorithm {
|
||||
fn from(value: sys_fc::ffi::Algorithm) -> Self {
|
||||
match value {
|
||||
sys_fc::ffi::Algorithm::AesGcm => Self::AesGcm,
|
||||
sys_fc::ffi::Algorithm::AesCbc => Self::AesCbc,
|
||||
_ => panic!("unknown frame cyrptor Algorithm"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<EncryptionAlgorithm> for sys_fc::ffi::Algorithm {
|
||||
fn from(value: EncryptionAlgorithm) -> Self {
|
||||
match value {
|
||||
EncryptionAlgorithm::AesGcm => Self::AesGcm,
|
||||
EncryptionAlgorithm::AesCbc => Self::AesCbc,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_fc::ffi::FrameCryptionState> for EncryptionState {
|
||||
fn from(value: sys_fc::ffi::FrameCryptionState) -> Self {
|
||||
match value {
|
||||
sys_fc::ffi::FrameCryptionState::New => Self::New,
|
||||
sys_fc::ffi::FrameCryptionState::Ok => Self::Ok,
|
||||
sys_fc::ffi::FrameCryptionState::EncryptionFailed => Self::EncryptionFailed,
|
||||
sys_fc::ffi::FrameCryptionState::DecryptionFailed => Self::DecryptionFailed,
|
||||
sys_fc::ffi::FrameCryptionState::MissingKey => Self::MissingKey,
|
||||
sys_fc::ffi::FrameCryptionState::KeyRatcheted => Self::KeyRatcheted,
|
||||
sys_fc::ffi::FrameCryptionState::InternalError => Self::InternalError,
|
||||
_ => panic!("unknown frame cyrptor FrameCryptionState"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<KeyProviderOptions> for sys_fc::ffi::KeyProviderOptions {
|
||||
fn from(value: KeyProviderOptions) -> Self {
|
||||
Self {
|
||||
shared_key: value.shared_key,
|
||||
ratchet_window_size: value.ratchet_window_size,
|
||||
ratchet_salt: value.ratchet_salt,
|
||||
failure_tolerance: value.failure_tolerance,
|
||||
key_ring_size: value.key_ring_size,
|
||||
key_derivation_algorithm: value.key_derivation_algorithm.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_fc::ffi::EncryptedPacket> for EncryptedPacket {
|
||||
fn from(value: sys_fc::ffi::EncryptedPacket) -> Self {
|
||||
Self {
|
||||
data: value.data.into_iter().collect(),
|
||||
iv: value.iv.into_iter().collect(),
|
||||
key_index: value.key_index,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<EncryptedPacket> for sys_fc::ffi::EncryptedPacket {
|
||||
fn from(value: EncryptedPacket) -> Self {
|
||||
Self { data: value.data, iv: value.iv, key_index: value.key_index }
|
||||
}
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use webrtc_sys::jsep as sys_jsep;
|
||||
|
||||
use crate::{ice_candidate as ic, session_description::SdpParseError};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct IceCandidate {
|
||||
pub(crate) sys_handle: SharedPtr<sys_jsep::ffi::IceCandidate>,
|
||||
}
|
||||
|
||||
impl IceCandidate {
|
||||
pub fn parse(
|
||||
sdp_mid: &str,
|
||||
sdp_mline_index: i32,
|
||||
sdp: &str,
|
||||
) -> Result<ic::IceCandidate, SdpParseError> {
|
||||
let res = sys_jsep::ffi::create_ice_candidate(
|
||||
sdp_mid.to_string(),
|
||||
sdp_mline_index,
|
||||
sdp.to_string(),
|
||||
);
|
||||
|
||||
match res {
|
||||
Ok(sys_handle) => Ok(ic::IceCandidate { handle: IceCandidate { sys_handle } }),
|
||||
Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sdp_mid(&self) -> String {
|
||||
self.sys_handle.sdp_mid()
|
||||
}
|
||||
|
||||
pub fn sdp_mline_index(&self) -> i32 {
|
||||
self.sys_handle.sdp_mline_index()
|
||||
}
|
||||
|
||||
pub fn candidate(&self) -> String {
|
||||
self.sys_handle.candidate()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToString for IceCandidate {
|
||||
fn to_string(&self) -> String {
|
||||
self.sys_handle.stringify()
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use webrtc_sys::media_stream as sys_ms;
|
||||
|
||||
use crate::{
|
||||
audio_track,
|
||||
imp::{audio_track::RtcAudioTrack, video_track::RtcVideoTrack},
|
||||
video_track,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct MediaStream {
|
||||
pub(crate) sys_handle: SharedPtr<sys_ms::ffi::MediaStream>,
|
||||
}
|
||||
|
||||
impl MediaStream {
|
||||
pub fn id(&self) -> String {
|
||||
self.sys_handle.id()
|
||||
}
|
||||
|
||||
pub fn audio_tracks(&self) -> Vec<audio_track::RtcAudioTrack> {
|
||||
self.sys_handle
|
||||
.get_audio_tracks()
|
||||
.into_iter()
|
||||
.map(|t| audio_track::RtcAudioTrack { handle: RtcAudioTrack { sys_handle: t.ptr } })
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn video_tracks(&self) -> Vec<video_track::RtcVideoTrack> {
|
||||
self.sys_handle
|
||||
.get_video_tracks()
|
||||
.into_iter()
|
||||
.map(|t| video_track::RtcVideoTrack { handle: RtcVideoTrack::new(t.ptr) })
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use webrtc_sys::{
|
||||
audio_track::ffi::media_to_audio, media_stream_track as sys_mst,
|
||||
video_track::ffi::media_to_video, MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
audio_track,
|
||||
imp::{audio_track::RtcAudioTrack, video_track::RtcVideoTrack},
|
||||
media_stream_track::{MediaStreamTrack, RtcTrackState},
|
||||
video_track,
|
||||
};
|
||||
|
||||
impl From<sys_mst::ffi::TrackState> for RtcTrackState {
|
||||
fn from(state: sys_mst::ffi::TrackState) -> Self {
|
||||
match state {
|
||||
sys_mst::ffi::TrackState::Live => RtcTrackState::Live,
|
||||
sys_mst::ffi::TrackState::Ended => RtcTrackState::Ended,
|
||||
_ => panic!("unknown TrackState"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_media_stream_track(
|
||||
sys_handle: SharedPtr<sys_mst::ffi::MediaStreamTrack>,
|
||||
) -> MediaStreamTrack {
|
||||
if sys_handle.kind() == MEDIA_TYPE_AUDIO {
|
||||
MediaStreamTrack::Audio(audio_track::RtcAudioTrack {
|
||||
handle: RtcAudioTrack { sys_handle: unsafe { media_to_audio(sys_handle) } },
|
||||
})
|
||||
} else if sys_handle.kind() == MEDIA_TYPE_VIDEO {
|
||||
MediaStreamTrack::Video(video_track::RtcVideoTrack {
|
||||
handle: RtcVideoTrack::new(unsafe { media_to_video(sys_handle) }),
|
||||
})
|
||||
} else {
|
||||
panic!("unknown track kind")
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! impl_media_stream_track {
|
||||
($cast:expr) => {
|
||||
pub fn id(&self) -> String {
|
||||
let ptr = $cast(self.sys_handle.clone());
|
||||
ptr.id()
|
||||
}
|
||||
|
||||
pub fn enabled(&self) -> bool {
|
||||
let ptr = $cast(self.sys_handle.clone());
|
||||
ptr.enabled()
|
||||
}
|
||||
|
||||
pub fn set_enabled(&self, enabled: bool) -> bool {
|
||||
let ptr = $cast(self.sys_handle.clone());
|
||||
ptr.set_enabled(enabled)
|
||||
}
|
||||
|
||||
pub fn state(&self) -> RtcTrackState {
|
||||
let ptr = $cast(self.sys_handle.clone());
|
||||
ptr.state().into()
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub(super) use impl_media_stream_track;
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
pub mod android;
|
||||
pub mod apm;
|
||||
pub mod audio_mixer;
|
||||
pub mod audio_resampler;
|
||||
pub mod audio_source;
|
||||
pub mod audio_stream;
|
||||
pub mod audio_track;
|
||||
pub mod data_channel;
|
||||
#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
|
||||
pub mod desktop_capturer;
|
||||
pub mod frame_cryptor;
|
||||
pub mod ice_candidate;
|
||||
pub mod media_stream;
|
||||
pub mod media_stream_track;
|
||||
pub mod packet_trailer;
|
||||
pub mod peer_connection;
|
||||
pub mod peer_connection_factory;
|
||||
pub mod rtp_parameters;
|
||||
pub mod rtp_receiver;
|
||||
pub mod rtp_sender;
|
||||
pub mod rtp_transceiver;
|
||||
pub mod session_description;
|
||||
pub mod video_frame;
|
||||
pub mod video_source;
|
||||
pub mod video_stream;
|
||||
pub mod video_track;
|
||||
pub mod yuv_helper;
|
||||
|
||||
use webrtc_sys::{rtc_error as sys_err, webrtc as sys_rtc};
|
||||
|
||||
use crate::{MediaType, RtcError, RtcErrorType};
|
||||
|
||||
impl From<sys_err::ffi::RtcErrorType> for RtcErrorType {
|
||||
fn from(value: sys_err::ffi::RtcErrorType) -> Self {
|
||||
match value {
|
||||
sys_err::ffi::RtcErrorType::InvalidState => Self::InvalidState,
|
||||
_ => Self::Internal,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_err::ffi::RtcError> for RtcError {
|
||||
fn from(value: sys_err::ffi::RtcError) -> Self {
|
||||
Self { error_type: value.error_type.into(), message: value.message }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<MediaType> for sys_rtc::ffi::MediaType {
|
||||
fn from(value: MediaType) -> Self {
|
||||
match value {
|
||||
MediaType::Audio => Self::Audio,
|
||||
MediaType::Video => Self::Video,
|
||||
MediaType::Data => Self::Data,
|
||||
MediaType::Unsupported => Self::Unsupported,
|
||||
}
|
||||
}
|
||||
}
|
||||
+273
@@ -0,0 +1,273 @@
|
||||
// Copyright 2026 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Packet trailer support for end-to-end frame metadata propagation.
|
||||
//!
|
||||
//! This module provides functionality to embed user-supplied metadata
|
||||
//! in encoded video frames as trailers. The timestamps/frameIDs are preserved
|
||||
//! through the WebRTC pipeline and can be extracted on the receiver side.
|
||||
//!
|
||||
//! On the send side, user timestamps/frameIDs are stored in the handler's internal
|
||||
//! map keyed by RTP timestamp. When the encoder produces a frame,
|
||||
//! the transformer looks up the metadata via the frame's CaptureTime().
|
||||
//!
|
||||
//! On the receive side, extracted frame metadata is stored in an
|
||||
//! internal map keyed by RTP timestamp. Decoded frames look up their
|
||||
//! metadata via lookup_frame_metadata(rtp_timestamp).
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use webrtc_sys::packet_trailer::ffi as sys_pt;
|
||||
|
||||
use crate::{
|
||||
peer_connection_factory::PeerConnectionFactory, rtp_receiver::RtpReceiver,
|
||||
rtp_sender::RtpSender,
|
||||
};
|
||||
|
||||
/// Stage reached by a native local video frame in the publish pipeline.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PublishTimingStage {
|
||||
/// The adapted raw frame was handed to WebRTC's encoder path.
|
||||
EncoderUpload,
|
||||
/// WebRTC produced an encoded frame for packetization.
|
||||
EncoderOutput,
|
||||
/// The encoded frame was handed back to WebRTC's packetizer.
|
||||
WebrtcPacketize,
|
||||
}
|
||||
|
||||
/// Stage reached by a native remote video frame in the subscribe pipeline.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SubscribeTimingStage {
|
||||
/// WebRTC produced an encoded frame after RTP depacketization.
|
||||
WebrtcReceive,
|
||||
/// The encoded frame was handed to WebRTC's decoder.
|
||||
DecoderUpload,
|
||||
/// WebRTC produced a decoded frame for the native video sink.
|
||||
DecoderOutput,
|
||||
}
|
||||
|
||||
/// Timestamped native local video publish pipeline event.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct PublishTimingEvent {
|
||||
/// Publish pipeline stage reached by the frame.
|
||||
pub stage: PublishTimingStage,
|
||||
/// Wall-clock time when this stage was observed, in microseconds since the Unix epoch.
|
||||
pub timestamp_us: u64,
|
||||
/// User capture timestamp associated with this frame, in microseconds since the Unix epoch.
|
||||
pub capture_timestamp_us: u64,
|
||||
/// Optional application frame ID associated with this frame.
|
||||
pub frame_id: Option<u32>,
|
||||
}
|
||||
|
||||
/// Timestamped native remote video subscribe pipeline event.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SubscribeTimingEvent {
|
||||
/// Subscribe pipeline stage reached by the frame.
|
||||
pub stage: SubscribeTimingStage,
|
||||
/// Wall-clock time when this stage was observed, in microseconds since the Unix epoch.
|
||||
pub timestamp_us: u64,
|
||||
/// User capture timestamp associated with this frame, in microseconds since the Unix epoch.
|
||||
pub capture_timestamp_us: u64,
|
||||
/// Optional application frame ID associated with this frame.
|
||||
pub frame_id: Option<u32>,
|
||||
}
|
||||
|
||||
/// Callback invoked for native local video publish timing events.
|
||||
pub type PublishTimingObserver = Arc<dyn Fn(PublishTimingEvent) + Send + Sync + 'static>;
|
||||
/// Callback invoked for native remote video subscribe timing events.
|
||||
pub type SubscribeTimingObserver = Arc<dyn Fn(SubscribeTimingEvent) + Send + Sync + 'static>;
|
||||
|
||||
impl From<sys_pt::VideoPublishTimingStage> for PublishTimingStage {
|
||||
fn from(stage: sys_pt::VideoPublishTimingStage) -> Self {
|
||||
match stage {
|
||||
sys_pt::VideoPublishTimingStage::EncoderUpload => Self::EncoderUpload,
|
||||
sys_pt::VideoPublishTimingStage::EncoderOutput => Self::EncoderOutput,
|
||||
sys_pt::VideoPublishTimingStage::WebrtcPacketize => Self::WebrtcPacketize,
|
||||
_ => Self::WebrtcPacketize,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_pt::VideoPublishTimingEvent> for PublishTimingEvent {
|
||||
fn from(event: sys_pt::VideoPublishTimingEvent) -> Self {
|
||||
Self {
|
||||
stage: event.stage.into(),
|
||||
timestamp_us: event.timestamp_us,
|
||||
capture_timestamp_us: event.capture_timestamp_us,
|
||||
frame_id: (event.frame_id != 0).then_some(event.frame_id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_pt::VideoSubscribeTimingStage> for SubscribeTimingStage {
|
||||
fn from(stage: sys_pt::VideoSubscribeTimingStage) -> Self {
|
||||
match stage {
|
||||
sys_pt::VideoSubscribeTimingStage::WebrtcReceive => Self::WebrtcReceive,
|
||||
sys_pt::VideoSubscribeTimingStage::DecoderUpload => Self::DecoderUpload,
|
||||
sys_pt::VideoSubscribeTimingStage::DecoderOutput => Self::DecoderOutput,
|
||||
_ => Self::DecoderOutput,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_pt::VideoSubscribeTimingEvent> for SubscribeTimingEvent {
|
||||
fn from(event: sys_pt::VideoSubscribeTimingEvent) -> Self {
|
||||
Self {
|
||||
stage: event.stage.into(),
|
||||
timestamp_us: event.timestamp_us,
|
||||
capture_timestamp_us: event.capture_timestamp_us,
|
||||
frame_id: (event.frame_id != 0).then_some(event.frame_id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handler for packet trailer embedding/extraction on RTP streams.
|
||||
///
|
||||
/// For sender side: Stores frame metadata keyed by capture timestamp
|
||||
/// and embeds them as binary payload trailers on encoded frames before they
|
||||
/// are sent. Use `store_frame_metadata()` to associate metadata with
|
||||
/// a captured frame.
|
||||
///
|
||||
/// For receiver side: Extracts frame metadata from received frames
|
||||
/// and makes them available for retrieval via `lookup_frame_metadata()`.
|
||||
#[derive(Clone)]
|
||||
pub struct PacketTrailerHandler {
|
||||
sys_handle: SharedPtr<sys_pt::PacketTrailerHandler>,
|
||||
}
|
||||
|
||||
impl PacketTrailerHandler {
|
||||
/// Enable or disable timestamp embedding/extraction.
|
||||
pub fn set_enabled(&self, enabled: bool) {
|
||||
self.sys_handle.set_enabled(enabled);
|
||||
}
|
||||
|
||||
/// Check if timestamp embedding/extraction is enabled.
|
||||
pub fn enabled(&self) -> bool {
|
||||
self.sys_handle.enabled()
|
||||
}
|
||||
|
||||
/// Lookup the frame metadata for a given RTP timestamp (receiver side).
|
||||
/// Returns `Some((user_timestamp, frame_id))` if found, `None` otherwise.
|
||||
/// The entry is removed from the map after a successful lookup.
|
||||
pub fn lookup_frame_metadata(&self, rtp_timestamp: u32) -> Option<(u64, u32)> {
|
||||
let ts = self.sys_handle.lookup_timestamp(rtp_timestamp);
|
||||
if ts != u64::MAX {
|
||||
let frame_id = self.sys_handle.last_lookup_frame_id();
|
||||
Some((ts, frame_id))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Store frame metadata for a given capture timestamp (sender side).
|
||||
///
|
||||
/// The `capture_timestamp_us` must be the TimestampAligner-adjusted
|
||||
/// timestamp (as produced by `VideoTrackSource::on_captured_frame`),
|
||||
/// NOT the original `timestamp_us` from the VideoFrame. The transformer
|
||||
/// looks up the metadata by the frame's `CaptureTime()` which is
|
||||
/// derived from the aligned value.
|
||||
///
|
||||
/// In normal usage this is called automatically by the C++ layer --
|
||||
/// callers should set `user_timestamp` and `frame_id` on the
|
||||
/// `VideoFrame` and let `capture_frame` / `on_captured_frame` handle
|
||||
/// the rest.
|
||||
pub fn store_frame_metadata(
|
||||
&self,
|
||||
capture_timestamp_us: i64,
|
||||
user_timestamp: u64,
|
||||
frame_id: u32,
|
||||
) {
|
||||
self.sys_handle.store_frame_metadata(capture_timestamp_us, user_timestamp, frame_id);
|
||||
}
|
||||
|
||||
pub(crate) fn sys_handle(&self) -> SharedPtr<sys_pt::PacketTrailerHandler> {
|
||||
self.sys_handle.clone()
|
||||
}
|
||||
|
||||
/// Set the callback receiving sender-side publish timing events.
|
||||
pub fn set_publish_timing_observer(&self, observer: Option<PublishTimingObserver>) {
|
||||
if let Some(observer) = observer {
|
||||
self.sys_handle.set_publish_timing_observer(Box::new(
|
||||
webrtc_sys::packet_trailer::VideoPublishTimingObserverWrapper::new(Box::new(
|
||||
move |event| observer(event.into()),
|
||||
)),
|
||||
));
|
||||
} else {
|
||||
self.sys_handle.clear_publish_timing_observer();
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the callback receiving receiver-side subscribe timing events.
|
||||
pub fn set_subscribe_timing_observer(&self, observer: Option<SubscribeTimingObserver>) {
|
||||
if let Some(observer) = observer {
|
||||
self.sys_handle.set_subscribe_timing_observer(Box::new(
|
||||
webrtc_sys::packet_trailer::VideoSubscribeTimingObserverWrapper::new(Box::new(
|
||||
move |event| observer(event.into()),
|
||||
)),
|
||||
));
|
||||
} else {
|
||||
self.sys_handle.clear_subscribe_timing_observer();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn emit_subscribe_timing(
|
||||
&self,
|
||||
stage: SubscribeTimingStage,
|
||||
capture_timestamp_us: u64,
|
||||
frame_id: u32,
|
||||
) {
|
||||
let stage = match stage {
|
||||
SubscribeTimingStage::WebrtcReceive => sys_pt::VideoSubscribeTimingStage::WebrtcReceive,
|
||||
SubscribeTimingStage::DecoderUpload => sys_pt::VideoSubscribeTimingStage::DecoderUpload,
|
||||
SubscribeTimingStage::DecoderOutput => sys_pt::VideoSubscribeTimingStage::DecoderOutput,
|
||||
};
|
||||
self.sys_handle.emit_subscribe_timing(stage, capture_timestamp_us, frame_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a sender-side packet trailer handler.
|
||||
///
|
||||
/// This handler will embed frame metadata into encoded frames before
|
||||
/// they are packetized and sent. Use `store_frame_metadata()` to
|
||||
/// associate metadata with a captured frame's capture timestamp.
|
||||
pub fn create_sender_handler(
|
||||
peer_factory: &PeerConnectionFactory,
|
||||
sender: &RtpSender,
|
||||
) -> PacketTrailerHandler {
|
||||
PacketTrailerHandler {
|
||||
sys_handle: sys_pt::new_packet_trailer_sender(
|
||||
peer_factory.handle.sys_handle.clone(),
|
||||
sender.handle.sys_handle.clone(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a receiver-side packet trailer handler.
|
||||
///
|
||||
/// This handler will extract frame metadata from received frames
|
||||
/// and store them in a map keyed by RTP timestamp. Use
|
||||
/// `lookup_frame_metadata(rtp_timestamp)` to retrieve the metadata
|
||||
/// for a specific decoded frame.
|
||||
pub fn create_receiver_handler(
|
||||
peer_factory: &PeerConnectionFactory,
|
||||
receiver: &RtpReceiver,
|
||||
) -> PacketTrailerHandler {
|
||||
PacketTrailerHandler {
|
||||
sys_handle: sys_pt::new_packet_trailer_receiver(
|
||||
peer_factory.handle.sys_handle.clone(),
|
||||
receiver.handle.sys_handle.clone(),
|
||||
),
|
||||
}
|
||||
}
|
||||
+619
@@ -0,0 +1,619 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use parking_lot::Mutex;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use webrtc_sys::{
|
||||
data_channel as sys_dc, jsep as sys_jsep, peer_connection as sys_pc,
|
||||
peer_connection_factory as sys_pcf, rtc_error as sys_err,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
data_channel::{DataChannel, DataChannelInit},
|
||||
ice_candidate::IceCandidate,
|
||||
imp::{
|
||||
data_channel as imp_dc, ice_candidate as imp_ic, media_stream as imp_ms,
|
||||
media_stream_track as imp_mst, rtp_receiver as imp_rr, rtp_sender as imp_rs,
|
||||
rtp_transceiver as imp_rt, session_description as imp_sdp,
|
||||
},
|
||||
media_stream::MediaStream,
|
||||
media_stream_track::MediaStreamTrack,
|
||||
peer_connection::{
|
||||
AnswerOptions, IceCandidateError, IceConnectionState, IceGatheringState, OfferOptions,
|
||||
OnConnectionChange, OnDataChannel, OnIceCandidate, OnIceCandidateError,
|
||||
OnIceConnectionChange, OnIceGatheringChange, OnNegotiationNeeded, OnSignalingChange,
|
||||
OnTrack, PeerConnectionState, SignalingState, TrackEvent,
|
||||
},
|
||||
peer_connection_factory::{
|
||||
ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration,
|
||||
},
|
||||
rtp_receiver::RtpReceiver,
|
||||
rtp_sender::RtpSender,
|
||||
rtp_transceiver::{RtpTransceiver, RtpTransceiverInit},
|
||||
session_description::SessionDescription,
|
||||
stats::RtcStats,
|
||||
MediaType, RtcError, RtcErrorType,
|
||||
};
|
||||
|
||||
impl From<OfferOptions> for sys_pc::ffi::RtcOfferAnswerOptions {
|
||||
fn from(options: OfferOptions) -> Self {
|
||||
Self {
|
||||
ice_restart: options.ice_restart,
|
||||
offer_to_receive_audio: options.offer_to_receive_audio as i32,
|
||||
offer_to_receive_video: options.offer_to_receive_video as i32,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AnswerOptions> for sys_pc::ffi::RtcOfferAnswerOptions {
|
||||
fn from(_options: AnswerOptions) -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_pc::ffi::PeerConnectionState> for PeerConnectionState {
|
||||
fn from(state: sys_pc::ffi::PeerConnectionState) -> Self {
|
||||
match state {
|
||||
sys_pc::ffi::PeerConnectionState::New => PeerConnectionState::New,
|
||||
sys_pc::ffi::PeerConnectionState::Connecting => PeerConnectionState::Connecting,
|
||||
sys_pc::ffi::PeerConnectionState::Connected => PeerConnectionState::Connected,
|
||||
sys_pc::ffi::PeerConnectionState::Disconnected => PeerConnectionState::Disconnected,
|
||||
sys_pc::ffi::PeerConnectionState::Failed => PeerConnectionState::Failed,
|
||||
sys_pc::ffi::PeerConnectionState::Closed => PeerConnectionState::Closed,
|
||||
_ => panic!("unknown PeerConnectionState"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_pc::ffi::IceConnectionState> for IceConnectionState {
|
||||
fn from(state: sys_pc::ffi::IceConnectionState) -> Self {
|
||||
match state {
|
||||
sys_pc::ffi::IceConnectionState::IceConnectionNew => IceConnectionState::New,
|
||||
sys_pc::ffi::IceConnectionState::IceConnectionChecking => IceConnectionState::Checking,
|
||||
sys_pc::ffi::IceConnectionState::IceConnectionConnected => {
|
||||
IceConnectionState::Connected
|
||||
}
|
||||
sys_pc::ffi::IceConnectionState::IceConnectionCompleted => {
|
||||
IceConnectionState::Completed
|
||||
}
|
||||
sys_pc::ffi::IceConnectionState::IceConnectionFailed => IceConnectionState::Failed,
|
||||
sys_pc::ffi::IceConnectionState::IceConnectionDisconnected => {
|
||||
IceConnectionState::Disconnected
|
||||
}
|
||||
sys_pc::ffi::IceConnectionState::IceConnectionClosed => IceConnectionState::Closed,
|
||||
sys_pc::ffi::IceConnectionState::IceConnectionMax => IceConnectionState::Max,
|
||||
_ => panic!("unknown IceConnectionState"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_pc::ffi::IceGatheringState> for IceGatheringState {
|
||||
fn from(state: sys_pc::ffi::IceGatheringState) -> Self {
|
||||
match state {
|
||||
sys_pc::ffi::IceGatheringState::IceGatheringNew => IceGatheringState::New,
|
||||
sys_pc::ffi::IceGatheringState::IceGatheringGathering => IceGatheringState::Gathering,
|
||||
sys_pc::ffi::IceGatheringState::IceGatheringComplete => IceGatheringState::Complete,
|
||||
_ => panic!("unknown IceGatheringState"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_pc::ffi::SignalingState> for SignalingState {
|
||||
fn from(state: sys_pc::ffi::SignalingState) -> Self {
|
||||
match state {
|
||||
sys_pc::ffi::SignalingState::Stable => SignalingState::Stable,
|
||||
sys_pc::ffi::SignalingState::HaveLocalOffer => SignalingState::HaveLocalOffer,
|
||||
sys_pc::ffi::SignalingState::HaveRemoteOffer => SignalingState::HaveRemoteOffer,
|
||||
sys_pc::ffi::SignalingState::HaveLocalPrAnswer => SignalingState::HaveLocalPrAnswer,
|
||||
sys_pc::ffi::SignalingState::HaveRemotePrAnswer => SignalingState::HaveRemotePrAnswer,
|
||||
sys_pc::ffi::SignalingState::Closed => SignalingState::Closed,
|
||||
_ => panic!("unknown SignalingState"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<IceServer> for sys_pc::ffi::IceServer {
|
||||
fn from(value: IceServer) -> Self {
|
||||
sys_pc::ffi::IceServer {
|
||||
urls: value.urls,
|
||||
username: value.username,
|
||||
password: value.password,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ContinualGatheringPolicy> for sys_pc::ffi::ContinualGatheringPolicy {
|
||||
fn from(value: ContinualGatheringPolicy) -> Self {
|
||||
match value {
|
||||
ContinualGatheringPolicy::GatherOnce => {
|
||||
sys_pc::ffi::ContinualGatheringPolicy::GatherOnce
|
||||
}
|
||||
ContinualGatheringPolicy::GatherContinually => {
|
||||
sys_pc::ffi::ContinualGatheringPolicy::GatherContinually
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<IceTransportsType> for sys_pc::ffi::IceTransportsType {
|
||||
fn from(value: IceTransportsType) -> Self {
|
||||
match value {
|
||||
IceTransportsType::Relay => sys_pc::ffi::IceTransportsType::Relay,
|
||||
IceTransportsType::NoHost => sys_pc::ffi::IceTransportsType::NoHost,
|
||||
IceTransportsType::All => sys_pc::ffi::IceTransportsType::All,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtcConfiguration> for sys_pc::ffi::RtcConfiguration {
|
||||
fn from(value: RtcConfiguration) -> Self {
|
||||
Self {
|
||||
ice_servers: value.ice_servers.into_iter().map(Into::into).collect(),
|
||||
continual_gathering_policy: value.continual_gathering_policy.into(),
|
||||
ice_transport_type: value.ice_transport_type.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PeerConnection {
|
||||
observer: Arc<PeerObserver>,
|
||||
pub(crate) sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
|
||||
}
|
||||
|
||||
impl PeerConnection {
|
||||
pub fn configure(
|
||||
sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
|
||||
observer: Arc<PeerObserver>,
|
||||
) -> Self {
|
||||
Self { sys_handle, observer }
|
||||
}
|
||||
|
||||
pub fn set_configuration(&self, config: RtcConfiguration) -> Result<(), RtcError> {
|
||||
let res = self.sys_handle.set_configuration(config.into());
|
||||
|
||||
match res {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn create_offer(
|
||||
&self,
|
||||
options: OfferOptions,
|
||||
) -> Result<SessionDescription, RtcError> {
|
||||
let (tx, mut rx) = mpsc::channel::<Result<SessionDescription, RtcError>>(1);
|
||||
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
|
||||
type CtxType = mpsc::Sender<Result<SessionDescription, RtcError>>;
|
||||
|
||||
self.sys_handle.create_offer(
|
||||
options.into(),
|
||||
ctx,
|
||||
|ctx, sdp| {
|
||||
let tx = *ctx.0.downcast::<CtxType>().unwrap();
|
||||
let _ = tx.blocking_send(Ok(SessionDescription {
|
||||
handle: imp_sdp::SessionDescription { sys_handle: sdp },
|
||||
}));
|
||||
},
|
||||
|ctx, error| {
|
||||
let tx = *ctx.0.downcast::<CtxType>().unwrap();
|
||||
let _ = tx.blocking_send(Err(error.into()));
|
||||
},
|
||||
);
|
||||
|
||||
rx.recv().await.unwrap()
|
||||
}
|
||||
|
||||
pub async fn create_answer(
|
||||
&self,
|
||||
options: AnswerOptions,
|
||||
) -> Result<SessionDescription, RtcError> {
|
||||
let (tx, mut rx) = mpsc::channel::<Result<SessionDescription, RtcError>>(1);
|
||||
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
|
||||
type CtxType = mpsc::Sender<Result<SessionDescription, RtcError>>;
|
||||
|
||||
self.sys_handle.create_answer(
|
||||
options.into(),
|
||||
ctx,
|
||||
|ctx, sdp| {
|
||||
let tx = *ctx.0.downcast::<CtxType>().unwrap();
|
||||
let _ = tx.blocking_send(Ok(SessionDescription {
|
||||
handle: imp_sdp::SessionDescription { sys_handle: sdp },
|
||||
}));
|
||||
},
|
||||
|ctx, error| {
|
||||
let tx = *ctx.0.downcast::<CtxType>().unwrap();
|
||||
let _ = tx.blocking_send(Err(error.into()));
|
||||
},
|
||||
);
|
||||
|
||||
rx.recv().await.unwrap()
|
||||
}
|
||||
|
||||
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
|
||||
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
|
||||
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
|
||||
|
||||
self.sys_handle.set_local_description(desc.handle.sys_handle, ctx, |ctx, err| {
|
||||
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
|
||||
|
||||
if err.ok() {
|
||||
let _ = tx.send(Ok(()));
|
||||
} else {
|
||||
let _ = tx.send(Err(err.into()));
|
||||
}
|
||||
});
|
||||
|
||||
rx.await.unwrap()
|
||||
}
|
||||
|
||||
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
|
||||
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
|
||||
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
|
||||
|
||||
self.sys_handle.set_remote_description(desc.handle.sys_handle, ctx, |ctx, err| {
|
||||
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
|
||||
|
||||
if err.ok() {
|
||||
let _ = tx.send(Ok(()));
|
||||
} else {
|
||||
let _ = tx.send(Err(err.into()));
|
||||
}
|
||||
});
|
||||
|
||||
rx.await.map_err(|_| RtcError {
|
||||
error_type: RtcErrorType::Internal,
|
||||
message: "set_remote_description cancelled".to_owned(),
|
||||
})?
|
||||
}
|
||||
|
||||
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
|
||||
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
|
||||
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
|
||||
|
||||
self.sys_handle.add_ice_candidate(candidate.handle.sys_handle, ctx, |ctx, err| {
|
||||
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
|
||||
|
||||
if err.ok() {
|
||||
let _ = tx.send(Ok(()));
|
||||
} else {
|
||||
let _ = tx.send(Err(err.into()));
|
||||
}
|
||||
});
|
||||
|
||||
rx.await.map_err(|_| RtcError {
|
||||
error_type: RtcErrorType::Internal,
|
||||
message: "add_ice_candidate cancelled".to_owned(),
|
||||
})?
|
||||
}
|
||||
|
||||
pub fn create_data_channel(
|
||||
&self,
|
||||
label: &str,
|
||||
init: DataChannelInit,
|
||||
) -> Result<DataChannel, RtcError> {
|
||||
let res = self.sys_handle.create_data_channel(label.to_string(), init.into());
|
||||
|
||||
match res {
|
||||
Ok(sys_handle) => {
|
||||
Ok(DataChannel { handle: imp_dc::DataChannel::configure(sys_handle) })
|
||||
}
|
||||
Err(e) => Err(unsafe { sys_err::ffi::RtcError::from(e.what()).into() }),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_track<T: AsRef<str>>(
|
||||
&self,
|
||||
track: MediaStreamTrack,
|
||||
stream_ids: &[T],
|
||||
) -> Result<RtpSender, RtcError> {
|
||||
let stream_ids = stream_ids.iter().map(|s| s.as_ref().to_owned()).collect();
|
||||
let res = self.sys_handle.add_track(track.sys_handle(), &stream_ids);
|
||||
|
||||
match res {
|
||||
Ok(sys_handle) => Ok(RtpSender { handle: imp_rs::RtpSender { sys_handle } }),
|
||||
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_transceiver(
|
||||
&self,
|
||||
track: MediaStreamTrack,
|
||||
init: RtpTransceiverInit,
|
||||
) -> Result<RtpTransceiver, RtcError> {
|
||||
let res = self.sys_handle.add_transceiver(track.sys_handle(), init.into());
|
||||
|
||||
match res {
|
||||
Ok(sys_handle) => Ok(RtpTransceiver { handle: imp_rt::RtpTransceiver { sys_handle } }),
|
||||
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_transceiver_for_media(
|
||||
&self,
|
||||
media_type: MediaType,
|
||||
init: RtpTransceiverInit,
|
||||
) -> Result<RtpTransceiver, RtcError> {
|
||||
let res = self.sys_handle.add_transceiver_for_media(media_type.into(), init.into());
|
||||
|
||||
match res {
|
||||
Ok(cxx_handle) => {
|
||||
Ok(RtpTransceiver { handle: imp_rt::RtpTransceiver { sys_handle: cxx_handle } })
|
||||
}
|
||||
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
|
||||
}
|
||||
}
|
||||
|
||||
pub fn restart_ice(&self) {
|
||||
self.sys_handle.restart_ice();
|
||||
}
|
||||
|
||||
pub fn close(&self) {
|
||||
self.sys_handle.close();
|
||||
}
|
||||
|
||||
pub fn connection_state(&self) -> PeerConnectionState {
|
||||
self.sys_handle.connection_state().into()
|
||||
}
|
||||
|
||||
pub fn ice_connection_state(&self) -> IceConnectionState {
|
||||
self.sys_handle.ice_connection_state().into()
|
||||
}
|
||||
|
||||
pub fn ice_gathering_state(&self) -> IceGatheringState {
|
||||
self.sys_handle.ice_gathering_state().into()
|
||||
}
|
||||
|
||||
pub fn signaling_state(&self) -> SignalingState {
|
||||
self.sys_handle.signaling_state().into()
|
||||
}
|
||||
|
||||
pub fn current_local_description(&self) -> Option<SessionDescription> {
|
||||
let sdp = self.sys_handle.current_local_description();
|
||||
if sdp.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(SessionDescription { handle: imp_sdp::SessionDescription { sys_handle: sdp } })
|
||||
}
|
||||
|
||||
pub fn current_remote_description(&self) -> Option<SessionDescription> {
|
||||
let sdp = self.sys_handle.current_remote_description();
|
||||
if sdp.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(SessionDescription { handle: imp_sdp::SessionDescription { sys_handle: sdp } })
|
||||
}
|
||||
|
||||
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
|
||||
self.sys_handle
|
||||
.remove_track(sender.handle.sys_handle)
|
||||
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
|
||||
}
|
||||
|
||||
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
|
||||
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
|
||||
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
|
||||
|
||||
self.sys_handle.get_stats(ctx, |ctx, stats| {
|
||||
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
|
||||
|
||||
if stats.is_empty() {
|
||||
let _ = tx.send(Ok(vec![]));
|
||||
return;
|
||||
}
|
||||
|
||||
// Unwrap because it should not happens
|
||||
let vec = serde_json::from_str(&stats).unwrap();
|
||||
let _ = tx.send(Ok(vec));
|
||||
});
|
||||
|
||||
rx.await.map_err(|_| RtcError {
|
||||
error_type: RtcErrorType::Internal,
|
||||
message: "get_stats cancelled".to_owned(),
|
||||
})?
|
||||
}
|
||||
|
||||
pub fn senders(&self) -> Vec<RtpSender> {
|
||||
self.sys_handle
|
||||
.get_senders()
|
||||
.into_iter()
|
||||
.map(|sender| RtpSender { handle: imp_rs::RtpSender { sys_handle: sender.ptr } })
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn receivers(&self) -> Vec<RtpReceiver> {
|
||||
self.sys_handle
|
||||
.get_receivers()
|
||||
.into_iter()
|
||||
.map(|receiver| RtpReceiver {
|
||||
handle: imp_rr::RtpReceiver { sys_handle: receiver.ptr },
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
|
||||
self.sys_handle
|
||||
.get_transceivers()
|
||||
.into_iter()
|
||||
.map(|transceiver| RtpTransceiver {
|
||||
handle: imp_rt::RtpTransceiver { sys_handle: transceiver.ptr },
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
|
||||
*self.observer.connection_change_handler.lock() = f;
|
||||
}
|
||||
|
||||
pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
|
||||
*self.observer.data_channel_handler.lock() = f;
|
||||
}
|
||||
|
||||
pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
|
||||
*self.observer.ice_candidate_handler.lock() = f;
|
||||
}
|
||||
|
||||
pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
|
||||
*self.observer.ice_candidate_error_handler.lock() = f;
|
||||
}
|
||||
|
||||
pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
|
||||
*self.observer.ice_connection_change_handler.lock() = f;
|
||||
}
|
||||
|
||||
pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
|
||||
*self.observer.ice_gathering_change_handler.lock() = f;
|
||||
}
|
||||
|
||||
pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
|
||||
*self.observer.negotiation_needed_handler.lock() = f;
|
||||
}
|
||||
|
||||
pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
|
||||
*self.observer.signaling_change_handler.lock() = f;
|
||||
}
|
||||
|
||||
pub fn on_track(&self, f: Option<OnTrack>) {
|
||||
*self.observer.track_handler.lock() = f;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct PeerObserver {
|
||||
pub connection_change_handler: Mutex<Option<OnConnectionChange>>,
|
||||
pub data_channel_handler: Mutex<Option<OnDataChannel>>,
|
||||
pub ice_candidate_handler: Mutex<Option<OnIceCandidate>>,
|
||||
pub ice_candidate_error_handler: Mutex<Option<OnIceCandidateError>>,
|
||||
pub ice_connection_change_handler: Mutex<Option<OnIceConnectionChange>>,
|
||||
pub ice_gathering_change_handler: Mutex<Option<OnIceGatheringChange>>,
|
||||
pub negotiation_needed_handler: Mutex<Option<OnNegotiationNeeded>>,
|
||||
pub signaling_change_handler: Mutex<Option<OnSignalingChange>>,
|
||||
pub track_handler: Mutex<Option<OnTrack>>,
|
||||
}
|
||||
|
||||
impl sys_pcf::PeerConnectionObserver for PeerObserver {
|
||||
fn on_signaling_change(&self, new_state: sys_pc::ffi::SignalingState) {
|
||||
if let Some(f) = self.signaling_change_handler.lock().as_mut() {
|
||||
f(new_state.into());
|
||||
}
|
||||
}
|
||||
|
||||
fn on_add_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
|
||||
|
||||
fn on_remove_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
|
||||
|
||||
fn on_data_channel(&self, data_channel: SharedPtr<sys_dc::ffi::DataChannel>) {
|
||||
if let Some(f) = self.data_channel_handler.lock().as_mut() {
|
||||
f(DataChannel { handle: imp_dc::DataChannel::configure(data_channel) });
|
||||
}
|
||||
}
|
||||
|
||||
fn on_renegotiation_needed(&self) {}
|
||||
|
||||
fn on_negotiation_needed_event(&self, event: u32) {
|
||||
if let Some(f) = self.negotiation_needed_handler.lock().as_mut() {
|
||||
f(event);
|
||||
}
|
||||
}
|
||||
|
||||
fn on_ice_connection_change(&self, _new_state: sys_pc::ffi::IceConnectionState) {}
|
||||
|
||||
fn on_standardized_ice_connection_change(&self, new_state: sys_pc::ffi::IceConnectionState) {
|
||||
if let Some(f) = self.ice_connection_change_handler.lock().as_mut() {
|
||||
f(new_state.into());
|
||||
}
|
||||
}
|
||||
|
||||
fn on_connection_change(&self, new_state: sys_pc::ffi::PeerConnectionState) {
|
||||
if let Some(f) = self.connection_change_handler.lock().as_mut() {
|
||||
f(new_state.into());
|
||||
}
|
||||
}
|
||||
|
||||
fn on_ice_gathering_change(&self, new_state: sys_pc::ffi::IceGatheringState) {
|
||||
if let Some(f) = self.ice_gathering_change_handler.lock().as_mut() {
|
||||
f(new_state.into());
|
||||
}
|
||||
}
|
||||
|
||||
fn on_ice_candidate(&self, candidate: SharedPtr<sys_jsep::ffi::IceCandidate>) {
|
||||
if let Some(f) = self.ice_candidate_handler.lock().as_mut() {
|
||||
f(IceCandidate { handle: imp_ic::IceCandidate { sys_handle: candidate } });
|
||||
}
|
||||
}
|
||||
|
||||
fn on_ice_candidate_error(
|
||||
&self,
|
||||
address: String,
|
||||
port: i32,
|
||||
url: String,
|
||||
error_code: i32,
|
||||
error_text: String,
|
||||
) {
|
||||
if let Some(f) = self.ice_candidate_error_handler.lock().as_mut() {
|
||||
f(IceCandidateError { address, port, url, error_code, error_text });
|
||||
}
|
||||
}
|
||||
|
||||
fn on_ice_candidates_removed(
|
||||
&self,
|
||||
_removed: Vec<SharedPtr<webrtc_sys::candidate::ffi::Candidate>>,
|
||||
) {
|
||||
}
|
||||
|
||||
fn on_ice_connection_receiving_change(&self, _receiving: bool) {}
|
||||
|
||||
fn on_ice_selected_candidate_pair_changed(
|
||||
&self,
|
||||
_event: sys_pcf::ffi::CandidatePairChangeEvent,
|
||||
) {
|
||||
}
|
||||
|
||||
fn on_add_track(
|
||||
&self,
|
||||
_receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>,
|
||||
_streams: Vec<SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>>,
|
||||
) {
|
||||
}
|
||||
|
||||
fn on_track(&self, transceiver: SharedPtr<webrtc_sys::rtp_transceiver::ffi::RtpTransceiver>) {
|
||||
if let Some(f) = self.track_handler.lock().as_mut() {
|
||||
let receiver = transceiver.receiver();
|
||||
let streams = receiver.streams();
|
||||
let track = receiver.track();
|
||||
|
||||
f(TrackEvent {
|
||||
receiver: RtpReceiver { handle: imp_rr::RtpReceiver { sys_handle: receiver } },
|
||||
streams: streams
|
||||
.into_iter()
|
||||
.map(|s| MediaStream { handle: imp_ms::MediaStream { sys_handle: s.ptr } })
|
||||
.collect(),
|
||||
track: imp_mst::new_media_stream_track(track),
|
||||
transceiver: RtpTransceiver {
|
||||
handle: imp_rt::RtpTransceiver { sys_handle: transceiver },
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn on_remove_track(&self, _receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>) {}
|
||||
|
||||
fn on_interesting_usage(&self, _usage_pattern: i32) {}
|
||||
}
|
||||
+361
@@ -0,0 +1,361 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use cxx::{SharedPtr, UniquePtr};
|
||||
use lazy_static::lazy_static;
|
||||
use parking_lot::Mutex;
|
||||
use webrtc_sys::{peer_connection_factory as sys_pcf, rtc_error as sys_err, webrtc as sys_rtc};
|
||||
|
||||
use crate::{
|
||||
audio_source::native::NativeAudioSource,
|
||||
audio_track::RtcAudioTrack,
|
||||
imp::{audio_track as imp_at, peer_connection as imp_pc, video_track as imp_vt},
|
||||
peer_connection::PeerConnection,
|
||||
peer_connection_factory::RtcConfiguration,
|
||||
rtp_parameters::RtpCapabilities,
|
||||
video_source::native::NativeVideoSource,
|
||||
video_track::RtcVideoTrack,
|
||||
MediaType, RtcError,
|
||||
};
|
||||
|
||||
lazy_static! {
|
||||
static ref LOG_SINK: Mutex<Option<UniquePtr<sys_rtc::ffi::LogSink>>> = Default::default();
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PeerConnectionFactory {
|
||||
pub(crate) sys_handle: SharedPtr<sys_pcf::ffi::PeerConnectionFactory>,
|
||||
}
|
||||
|
||||
impl Default for PeerConnectionFactory {
|
||||
fn default() -> Self {
|
||||
let mut log_sink = LOG_SINK.lock();
|
||||
if log_sink.is_none() {
|
||||
*log_sink = Some(sys_rtc::ffi::new_log_sink(|msg, _| {
|
||||
let msg = msg.strip_suffix("\r\n").or(msg.strip_suffix('\n')).unwrap_or(&msg);
|
||||
log::debug!(target: "libwebrtc", "{}", msg);
|
||||
}));
|
||||
}
|
||||
|
||||
let sys_handle = sys_pcf::ffi::create_peer_connection_factory();
|
||||
Self { sys_handle }
|
||||
}
|
||||
}
|
||||
|
||||
impl PeerConnectionFactory {
|
||||
pub fn create_peer_connection(
|
||||
&self,
|
||||
config: RtcConfiguration,
|
||||
) -> Result<PeerConnection, RtcError> {
|
||||
let observer = Arc::new(imp_pc::PeerObserver::default());
|
||||
let res = self.sys_handle.create_peer_connection(
|
||||
config.into(),
|
||||
Box::new(sys_pcf::PeerConnectionObserverWrapper::new(observer.clone())),
|
||||
);
|
||||
|
||||
match res {
|
||||
Ok(sys_handle) => Ok(PeerConnection {
|
||||
handle: imp_pc::PeerConnection::configure(sys_handle, observer),
|
||||
}),
|
||||
Err(e) => Err(unsafe { sys_err::ffi::RtcError::from(e.what()).into() }),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
|
||||
RtcVideoTrack {
|
||||
handle: imp_vt::RtcVideoTrack::new(
|
||||
self.sys_handle.create_video_track(label.to_string(), source.handle.sys_handle()),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
|
||||
RtcAudioTrack {
|
||||
handle: imp_at::RtcAudioTrack {
|
||||
sys_handle: self
|
||||
.sys_handle
|
||||
.create_audio_track(label.to_string(), source.handle.sys_handle()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Create an audio track that uses the Platform ADM for capture.
|
||||
///
|
||||
/// This requires that `enable_platform_adm()` was called first.
|
||||
/// The track will capture audio from the selected recording device.
|
||||
pub fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack {
|
||||
RtcAudioTrack {
|
||||
handle: imp_at::RtcAudioTrack {
|
||||
sys_handle: self.sys_handle.create_device_audio_track(label.to_string()),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
|
||||
self.sys_handle.rtp_sender_capabilities(media_type.into()).into()
|
||||
}
|
||||
|
||||
pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
|
||||
self.sys_handle.rtp_receiver_capabilities(media_type.into()).into()
|
||||
}
|
||||
|
||||
// ===== Device Management Methods =====
|
||||
|
||||
/// Get the number of playout (output) devices
|
||||
pub fn playout_devices(&self) -> i16 {
|
||||
self.sys_handle.audio_device().playout_devices()
|
||||
}
|
||||
|
||||
/// Get the number of recording (input) devices
|
||||
pub fn recording_devices(&self) -> i16 {
|
||||
self.sys_handle.audio_device().recording_devices()
|
||||
}
|
||||
|
||||
/// Get the name of a playout device by index
|
||||
pub fn playout_device_name(&self, index: u16) -> String {
|
||||
self.sys_handle.audio_device().playout_device_name(index)
|
||||
}
|
||||
|
||||
/// Get the name of a recording device by index
|
||||
pub fn recording_device_name(&self, index: u16) -> String {
|
||||
self.sys_handle.audio_device().recording_device_name(index)
|
||||
}
|
||||
|
||||
/// Get the GUID of a playout device by index
|
||||
/// The GUID is a platform-specific unique identifier that is stable across device hot-plug events.
|
||||
pub fn playout_device_guid(&self, index: u16) -> String {
|
||||
self.sys_handle.audio_device().playout_device_guid(index)
|
||||
}
|
||||
|
||||
/// Get the GUID of a recording device by index
|
||||
/// The GUID is a platform-specific unique identifier that is stable across device hot-plug events.
|
||||
pub fn recording_device_guid(&self, index: u16) -> String {
|
||||
self.sys_handle.audio_device().recording_device_guid(index)
|
||||
}
|
||||
|
||||
/// Set the playout device by index
|
||||
pub fn set_playout_device(&self, index: u16) -> bool {
|
||||
self.sys_handle.audio_device().set_playout_device(index)
|
||||
}
|
||||
|
||||
/// Set the recording device by index
|
||||
pub fn set_recording_device(&self, index: u16) -> bool {
|
||||
self.sys_handle.audio_device().set_recording_device(index)
|
||||
}
|
||||
|
||||
/// Set the playout device by GUID
|
||||
/// This is preferred over index as GUIDs are stable across device hot-plug events.
|
||||
pub fn set_playout_device_by_guid(&self, guid: &str) -> bool {
|
||||
self.sys_handle.audio_device().set_playout_device_by_guid(guid.to_string())
|
||||
}
|
||||
|
||||
/// Set the recording device by GUID
|
||||
/// This is preferred over index as GUIDs are stable across device hot-plug events.
|
||||
pub fn set_recording_device_by_guid(&self, guid: &str) -> bool {
|
||||
self.sys_handle.audio_device().set_recording_device_by_guid(guid.to_string())
|
||||
}
|
||||
|
||||
/// Stop recording (clears initialized state, allowing device switch)
|
||||
pub fn stop_recording(&self) -> bool {
|
||||
self.sys_handle.audio_device().stop_recording()
|
||||
}
|
||||
|
||||
/// Initialize recording
|
||||
pub fn init_recording(&self) -> bool {
|
||||
self.sys_handle.audio_device().init_recording()
|
||||
}
|
||||
|
||||
/// Start recording
|
||||
pub fn start_recording(&self) -> bool {
|
||||
self.sys_handle.audio_device().start_recording()
|
||||
}
|
||||
|
||||
/// Check if recording is initialized
|
||||
pub fn recording_is_initialized(&self) -> bool {
|
||||
self.sys_handle.audio_device().recording_is_initialized()
|
||||
}
|
||||
|
||||
/// Stop playout (clears initialized state, allowing device switch)
|
||||
pub fn stop_playout(&self) -> bool {
|
||||
self.sys_handle.audio_device().stop_playout()
|
||||
}
|
||||
|
||||
/// Initialize playout
|
||||
pub fn init_playout(&self) -> bool {
|
||||
self.sys_handle.audio_device().init_playout()
|
||||
}
|
||||
|
||||
/// Start playout
|
||||
pub fn start_playout(&self) -> bool {
|
||||
self.sys_handle.audio_device().start_playout()
|
||||
}
|
||||
|
||||
/// Check if playout is initialized
|
||||
pub fn playout_is_initialized(&self) -> bool {
|
||||
self.sys_handle.audio_device().playout_is_initialized()
|
||||
}
|
||||
|
||||
// ===== Built-in Audio Processing Methods =====
|
||||
// These control hardware AEC/AGC/NS on platforms that support it (iOS, some Android)
|
||||
|
||||
/// Check if built-in (hardware) AEC is available on this device.
|
||||
///
|
||||
/// Returns true on iOS (VPIO) and some Android devices.
|
||||
/// Returns false on desktop platforms (macOS, Windows, Linux).
|
||||
pub fn builtin_aec_is_available(&self) -> bool {
|
||||
self.sys_handle.audio_device().builtin_aec_is_available()
|
||||
}
|
||||
|
||||
/// Check if built-in (hardware) AGC is available on this device.
|
||||
///
|
||||
/// Returns true on iOS (VPIO) and some Android devices.
|
||||
/// Returns false on desktop platforms (macOS, Windows, Linux).
|
||||
pub fn builtin_agc_is_available(&self) -> bool {
|
||||
self.sys_handle.audio_device().builtin_agc_is_available()
|
||||
}
|
||||
|
||||
/// Check if built-in (hardware) NS is available on this device.
|
||||
///
|
||||
/// Returns true on iOS (VPIO) and some Android devices.
|
||||
/// Returns false on desktop platforms (macOS, Windows, Linux).
|
||||
pub fn builtin_ns_is_available(&self) -> bool {
|
||||
self.sys_handle.audio_device().builtin_ns_is_available()
|
||||
}
|
||||
|
||||
/// Enable or disable built-in (hardware) AEC.
|
||||
///
|
||||
/// When disabled on platforms that support it, WebRTC's software AEC
|
||||
/// will be used instead.
|
||||
pub fn enable_builtin_aec(&self, enable: bool) -> bool {
|
||||
self.sys_handle.audio_device().enable_builtin_aec(enable)
|
||||
}
|
||||
|
||||
/// Enable or disable built-in (hardware) AGC.
|
||||
///
|
||||
/// When disabled on platforms that support it, WebRTC's software AGC
|
||||
/// will be used instead.
|
||||
pub fn enable_builtin_agc(&self, enable: bool) -> bool {
|
||||
self.sys_handle.audio_device().enable_builtin_agc(enable)
|
||||
}
|
||||
|
||||
/// Enable or disable built-in (hardware) NS.
|
||||
///
|
||||
/// When disabled on platforms that support it, WebRTC's software NS
|
||||
/// will be used instead.
|
||||
pub fn enable_builtin_ns(&self, enable: bool) -> bool {
|
||||
self.sys_handle.audio_device().enable_builtin_ns(enable)
|
||||
}
|
||||
|
||||
/// Control whether ADM recording (microphone) is enabled.
|
||||
///
|
||||
/// When disabled, WebRTC's calls to InitRecording/StartRecording will be no-ops.
|
||||
/// Use this when only using NativeAudioSource (no microphone capture needed).
|
||||
/// This prevents the microphone from interfering with the audio pipeline.
|
||||
pub fn set_adm_recording_enabled(&self, enabled: bool) {
|
||||
self.sys_handle.audio_device().set_adm_recording_enabled(enabled)
|
||||
}
|
||||
|
||||
/// Check if ADM recording (microphone) is enabled.
|
||||
pub fn adm_recording_enabled(&self) -> bool {
|
||||
self.sys_handle.audio_device().adm_recording_enabled()
|
||||
}
|
||||
|
||||
/// Control whether ADM playout (speakers) is enabled.
|
||||
///
|
||||
/// When disabled (default), playout uses synthetic mode - remote audio is
|
||||
/// delivered via FFI callbacks to the application (e.g., Unity AudioSource).
|
||||
/// When enabled, remote audio plays through the platform speakers with AEC.
|
||||
pub fn set_adm_playout_enabled(&self, enabled: bool) {
|
||||
self.sys_handle.audio_device().set_adm_playout_enabled(enabled)
|
||||
}
|
||||
|
||||
/// Check if ADM playout (speakers) is enabled.
|
||||
pub fn adm_playout_enabled(&self) -> bool {
|
||||
self.sys_handle.audio_device().adm_playout_enabled()
|
||||
}
|
||||
|
||||
// ===== Platform ADM Lifecycle Management =====
|
||||
|
||||
/// Acquires a reference to the Platform ADM.
|
||||
///
|
||||
/// On first call, creates and initializes the Platform ADM. On subsequent
|
||||
/// calls, just increments the reference count.
|
||||
///
|
||||
/// Returns true if Platform ADM is ready for use, false if initialization failed.
|
||||
pub fn acquire_platform_adm(&self) -> bool {
|
||||
self.sys_handle.audio_device().acquire_platform_adm()
|
||||
}
|
||||
|
||||
/// Releases a reference to the Platform ADM.
|
||||
///
|
||||
/// When the reference count reaches zero, the Platform ADM is terminated
|
||||
/// and the proxy returns to synthetic mode.
|
||||
pub fn release_platform_adm(&self) {
|
||||
self.sys_handle.audio_device().release_platform_adm()
|
||||
}
|
||||
|
||||
/// Returns the current reference count for the Platform ADM.
|
||||
pub fn platform_adm_ref_count(&self) -> i32 {
|
||||
self.sys_handle.audio_device().platform_adm_ref_count()
|
||||
}
|
||||
|
||||
/// Returns true if Platform ADM is currently active (ref_count > 0).
|
||||
pub fn is_platform_adm_active(&self) -> bool {
|
||||
self.sys_handle.audio_device().is_platform_adm_active()
|
||||
}
|
||||
|
||||
/// Ensures the Platform ADM exists, retrying creation if an earlier
|
||||
/// attempt failed (e.g. the OS audio stack was still starting up).
|
||||
///
|
||||
/// Returns true if the Platform ADM is available after the call.
|
||||
pub fn ensure_platform_adm(&self) -> bool {
|
||||
self.sys_handle.audio_device().ensure_platform_adm()
|
||||
}
|
||||
|
||||
/// Returns true if the Platform ADM has been created and initialized.
|
||||
/// Distinguishes "audio stack unavailable" from "zero audio devices".
|
||||
pub fn platform_adm_available(&self) -> bool {
|
||||
self.sys_handle.audio_device().platform_adm_available()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
static TEST_MUTEX: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_peer_connection_factory_and_audio_device_controller_bridge() {
|
||||
let _guard = TEST_MUTEX.lock().expect("test mutex poisoned");
|
||||
let _ = env_logger::builder().is_test(true).try_init();
|
||||
|
||||
let factory = PeerConnectionFactory::default();
|
||||
let source = NativeVideoSource::default();
|
||||
let _track = factory.create_video_track("test", source);
|
||||
let recording_count = factory.recording_devices();
|
||||
let playout_count = factory.playout_devices();
|
||||
|
||||
assert!(recording_count >= 0);
|
||||
assert!(playout_count >= 0);
|
||||
|
||||
let initial_recording = factory.adm_recording_enabled();
|
||||
factory.set_adm_recording_enabled(!initial_recording);
|
||||
assert_eq!(factory.adm_recording_enabled(), !initial_recording);
|
||||
factory.set_adm_recording_enabled(initial_recording);
|
||||
}
|
||||
}
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use webrtc_sys::{rtp_parameters as sys_rp, webrtc as sys_webrtc};
|
||||
|
||||
use crate::rtp_parameters::*;
|
||||
|
||||
impl From<sys_webrtc::ffi::Priority> for Priority {
|
||||
fn from(value: sys_webrtc::ffi::Priority) -> Self {
|
||||
match value {
|
||||
sys_webrtc::ffi::Priority::VeryLow => Self::VeryLow,
|
||||
sys_webrtc::ffi::Priority::Low => Self::Low,
|
||||
sys_webrtc::ffi::Priority::Medium => Self::Medium,
|
||||
sys_webrtc::ffi::Priority::High => Self::High,
|
||||
_ => panic!("unknown Priority"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_rp::ffi::RtpExtension> for RtpHeaderExtensionParameters {
|
||||
fn from(value: sys_rp::ffi::RtpExtension) -> Self {
|
||||
Self { uri: value.uri, id: value.id, encrypted: value.encrypt }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_rp::ffi::RtpParameters> for RtpParameters {
|
||||
fn from(value: sys_rp::ffi::RtpParameters) -> Self {
|
||||
Self {
|
||||
codecs: value.codecs.into_iter().map(Into::into).collect(),
|
||||
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
|
||||
rtcp: value.rtcp.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_rp::ffi::RtpCodecParameters> for RtpCodecParameters {
|
||||
fn from(value: sys_rp::ffi::RtpCodecParameters) -> Self {
|
||||
Self {
|
||||
mime_type: value.mime_type,
|
||||
payload_type: value.payload_type as u8,
|
||||
clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
|
||||
channels: value.has_num_channels.then_some(value.num_channels as u16),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_rp::ffi::RtcpParameters> for RtcpParameters {
|
||||
fn from(value: sys_rp::ffi::RtcpParameters) -> Self {
|
||||
Self { cname: value.cname, reduced_size: value.reduced_size }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_rp::ffi::RtpEncodingParameters> for RtpEncodingParameters {
|
||||
fn from(value: sys_rp::ffi::RtpEncodingParameters) -> Self {
|
||||
Self {
|
||||
active: value.active,
|
||||
max_bitrate: value.has_max_bitrate_bps.then_some(value.max_bitrate_bps as u64),
|
||||
max_framerate: value.has_max_framerate.then_some(value.max_framerate),
|
||||
priority: value.network_priority.into(),
|
||||
rid: value.rid,
|
||||
scale_resolution_down_by: value
|
||||
.has_scale_resolution_down_by
|
||||
.then_some(value.scale_resolution_down_by),
|
||||
scalability_mode: value.has_scalability_mode.then_some(value.scalability_mode),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_rp::ffi::RtpCodecCapability> for RtpCodecCapability {
|
||||
fn from(value: sys_rp::ffi::RtpCodecCapability) -> Self {
|
||||
Self {
|
||||
channels: value.has_num_channels.then_some(value.num_channels as u16),
|
||||
mime_type: value.mime_type,
|
||||
clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
|
||||
sdp_fmtp_line: {
|
||||
let parameters: Vec<String> = value
|
||||
.parameters
|
||||
.into_iter()
|
||||
.map(|key_value| {
|
||||
if !key_value.key.is_empty() {
|
||||
format!("{}={}", key_value.key, key_value.value)
|
||||
} else {
|
||||
key_value.value
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
if !parameters.is_empty() {
|
||||
Some(parameters.join(";"))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_rp::ffi::RtpHeaderExtensionCapability> for RtpHeaderExtensionCapability {
|
||||
fn from(value: sys_rp::ffi::RtpHeaderExtensionCapability) -> Self {
|
||||
Self { direction: value.direction.into(), uri: value.uri }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_rp::ffi::RtpCapabilities> for RtpCapabilities {
|
||||
fn from(value: sys_rp::ffi::RtpCapabilities) -> Self {
|
||||
Self {
|
||||
codecs: value.codecs.into_iter().map(Into::into).collect(),
|
||||
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Priority> for sys_webrtc::ffi::Priority {
|
||||
fn from(value: Priority) -> Self {
|
||||
match value {
|
||||
Priority::VeryLow => Self::VeryLow,
|
||||
Priority::Low => Self::Low,
|
||||
Priority::Medium => Self::Medium,
|
||||
Priority::High => Self::High,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtpHeaderExtensionParameters> for sys_rp::ffi::RtpExtension {
|
||||
fn from(value: RtpHeaderExtensionParameters) -> Self {
|
||||
Self { uri: value.uri, id: value.id, encrypt: value.encrypted }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtpParameters> for sys_rp::ffi::RtpParameters {
|
||||
fn from(value: RtpParameters) -> Self {
|
||||
Self {
|
||||
codecs: value.codecs.into_iter().map(Into::into).collect(),
|
||||
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
|
||||
encodings: Vec::new(),
|
||||
rtcp: value.rtcp.into(),
|
||||
transaction_id: "".to_string(),
|
||||
mid: "".to_string(),
|
||||
has_degradation_preference: false,
|
||||
degradation_preference: sys_rp::ffi::DegradationPreference::Balanced,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtpCodecParameters> for sys_rp::ffi::RtpCodecParameters {
|
||||
fn from(value: RtpCodecParameters) -> Self {
|
||||
Self {
|
||||
payload_type: value.payload_type as i32,
|
||||
mime_type: value.mime_type,
|
||||
has_clock_rate: value.clock_rate.is_some(),
|
||||
clock_rate: value.clock_rate.unwrap_or_default() as i32,
|
||||
has_num_channels: value.channels.is_some(),
|
||||
num_channels: value.channels.unwrap_or_default() as i32,
|
||||
name: "".to_string(),
|
||||
kind: sys_rp::ffi::MediaType::Audio,
|
||||
has_max_ptime: false,
|
||||
max_ptime: 0,
|
||||
has_ptime: false,
|
||||
ptime: 0,
|
||||
rtcp_feedback: Vec::new(),
|
||||
parameters: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtcpParameters> for sys_rp::ffi::RtcpParameters {
|
||||
fn from(value: RtcpParameters) -> Self {
|
||||
Self {
|
||||
cname: value.cname,
|
||||
reduced_size: value.reduced_size,
|
||||
has_ssrc: false,
|
||||
ssrc: 0,
|
||||
mux: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtpEncodingParameters> for sys_rp::ffi::RtpEncodingParameters {
|
||||
fn from(value: RtpEncodingParameters) -> Self {
|
||||
Self {
|
||||
active: value.active,
|
||||
has_max_bitrate_bps: value.max_bitrate.is_some(),
|
||||
max_bitrate_bps: value.max_bitrate.unwrap_or_default() as i32,
|
||||
has_max_framerate: value.max_framerate.is_some(),
|
||||
max_framerate: value.max_framerate.unwrap_or_default(),
|
||||
network_priority: value.priority.into(),
|
||||
rid: value.rid,
|
||||
has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(),
|
||||
scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or_default(),
|
||||
adaptive_ptime: false,
|
||||
bitrate_priority: sys_rp::DEFAULT_BITRATE_PRIORITY,
|
||||
has_min_bitrate_bps: false,
|
||||
min_bitrate_bps: 0,
|
||||
has_num_temporal_layers: false,
|
||||
num_temporal_layers: 0,
|
||||
has_scalability_mode: value.scalability_mode.is_some(),
|
||||
scalability_mode: value.scalability_mode.unwrap_or_default(),
|
||||
has_ssrc: false,
|
||||
ssrc: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtpCodecCapability> for sys_rp::ffi::RtpCodecCapability {
|
||||
fn from(value: RtpCodecCapability) -> Self {
|
||||
let mime_type: Vec<&str> = value.mime_type.split('/').collect();
|
||||
let kind = match mime_type[0] {
|
||||
"audio" => sys_webrtc::ffi::MediaType::Audio,
|
||||
"video" => sys_webrtc::ffi::MediaType::Video,
|
||||
_ => panic!("invalid media type"),
|
||||
};
|
||||
let name = mime_type[1].to_string();
|
||||
|
||||
Self {
|
||||
name,
|
||||
kind,
|
||||
has_clock_rate: value.clock_rate.is_some(),
|
||||
clock_rate: value.clock_rate.unwrap_or_default() as i32,
|
||||
has_num_channels: value.channels.is_some(),
|
||||
num_channels: value.channels.unwrap_or_default() as i32,
|
||||
parameters: {
|
||||
value
|
||||
.sdp_fmtp_line
|
||||
.map(|sdp_fmtp_line| {
|
||||
sdp_fmtp_line
|
||||
.split(';')
|
||||
.map(|v| {
|
||||
let key_value: Vec<&str> = v.split('=').collect();
|
||||
if key_value.len() == 2 {
|
||||
sys_rp::ffi::StringKeyValue {
|
||||
key: key_value[0].to_string(),
|
||||
value: key_value[1].to_string(),
|
||||
}
|
||||
} else {
|
||||
sys_rp::ffi::StringKeyValue {
|
||||
key: "".to_string(),
|
||||
value: key_value[0].to_string(),
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
},
|
||||
// Ignore
|
||||
mime_type: String::default(), // !!
|
||||
has_preferred_payload_type: false,
|
||||
preferred_payload_type: 0,
|
||||
rtcp_feedback: Vec::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use tokio::sync::oneshot;
|
||||
use webrtc_sys::rtp_receiver as sys_rr;
|
||||
|
||||
use crate::{
|
||||
imp::media_stream_track::new_media_stream_track, media_stream_track::MediaStreamTrack,
|
||||
rtp_parameters::RtpParameters, stats::RtcStats, RtcError, RtcErrorType,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtpReceiver {
|
||||
pub(crate) sys_handle: SharedPtr<sys_rr::ffi::RtpReceiver>,
|
||||
}
|
||||
|
||||
impl RtpReceiver {
|
||||
pub fn track(&self) -> Option<MediaStreamTrack> {
|
||||
let track_handle = self.sys_handle.track();
|
||||
if track_handle.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(new_media_stream_track(track_handle))
|
||||
}
|
||||
|
||||
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
|
||||
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
|
||||
let ctx = Box::new(sys_rr::ReceiverContext(Box::new(tx)));
|
||||
|
||||
self.sys_handle.get_stats(ctx, |ctx, stats| {
|
||||
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
|
||||
|
||||
if stats.is_empty() {
|
||||
let _ = tx.send(Ok(vec![]));
|
||||
return;
|
||||
}
|
||||
|
||||
// Unwrap because it should not happens
|
||||
let vec = serde_json::from_str(&stats).unwrap();
|
||||
let _ = tx.send(Ok(vec));
|
||||
});
|
||||
|
||||
rx.await.map_err(|_| RtcError {
|
||||
error_type: RtcErrorType::Internal,
|
||||
message: "get_stats cancelled".to_owned(),
|
||||
})?
|
||||
}
|
||||
|
||||
pub fn parameters(&self) -> RtpParameters {
|
||||
self.sys_handle.get_parameters().into()
|
||||
}
|
||||
}
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use tokio::sync::oneshot;
|
||||
use webrtc_sys::{rtc_error as sys_err, rtp_sender as sys_rs};
|
||||
|
||||
use super::media_stream_track::new_media_stream_track;
|
||||
use crate::{
|
||||
media_stream_track::MediaStreamTrack, rtp_parameters::RtpParameters, stats::RtcStats, RtcError,
|
||||
RtcErrorType,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtpSender {
|
||||
pub(crate) sys_handle: SharedPtr<sys_rs::ffi::RtpSender>,
|
||||
}
|
||||
|
||||
impl RtpSender {
|
||||
pub fn track(&self) -> Option<MediaStreamTrack> {
|
||||
let track_handle = self.sys_handle.track();
|
||||
if track_handle.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(new_media_stream_track(track_handle))
|
||||
}
|
||||
|
||||
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
|
||||
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
|
||||
let ctx = Box::new(sys_rs::SenderContext(Box::new(tx)));
|
||||
|
||||
self.sys_handle.get_stats(ctx, |ctx, stats| {
|
||||
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
|
||||
|
||||
if stats.is_empty() {
|
||||
let _ = tx.send(Ok(vec![]));
|
||||
return;
|
||||
}
|
||||
|
||||
// Unwrap because it should not happens
|
||||
let vec = serde_json::from_str(&stats).unwrap();
|
||||
let _ = tx.send(Ok(vec));
|
||||
});
|
||||
|
||||
rx.await.map_err(|_| RtcError {
|
||||
error_type: RtcErrorType::Internal,
|
||||
message: "get_stats cancelled".to_owned(),
|
||||
})?
|
||||
}
|
||||
|
||||
pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
|
||||
if !self.sys_handle.set_track(track.map_or(SharedPtr::null(), |t| t.sys_handle())) {
|
||||
return Err(RtcError {
|
||||
error_type: RtcErrorType::InvalidState,
|
||||
message: "Failed to set track".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn parameters(&self) -> RtpParameters {
|
||||
self.sys_handle.get_parameters().into()
|
||||
}
|
||||
|
||||
pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
|
||||
self.sys_handle
|
||||
.set_parameters(parameters.into())
|
||||
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use webrtc_sys::{rtc_error as sys_err, rtp_transceiver as sys_rt, webrtc as sys_webrtc};
|
||||
|
||||
use crate::{
|
||||
imp::{rtp_receiver::RtpReceiver, rtp_sender::RtpSender},
|
||||
rtp_parameters::RtpCodecCapability,
|
||||
rtp_receiver, rtp_sender,
|
||||
rtp_transceiver::{RtpTransceiverDirection, RtpTransceiverInit},
|
||||
RtcError,
|
||||
};
|
||||
|
||||
impl From<sys_webrtc::ffi::RtpTransceiverDirection> for RtpTransceiverDirection {
|
||||
fn from(value: sys_webrtc::ffi::RtpTransceiverDirection) -> Self {
|
||||
match value {
|
||||
sys_webrtc::ffi::RtpTransceiverDirection::SendRecv => Self::SendRecv,
|
||||
sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly,
|
||||
sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
|
||||
sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive,
|
||||
sys_webrtc::ffi::RtpTransceiverDirection::Stopped => Self::Stopped,
|
||||
_ => panic!("unknown RtpTransceiverDirection"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtpTransceiverDirection> for sys_webrtc::ffi::RtpTransceiverDirection {
|
||||
fn from(value: RtpTransceiverDirection) -> Self {
|
||||
match value {
|
||||
RtpTransceiverDirection::SendRecv => Self::SendRecv,
|
||||
RtpTransceiverDirection::SendOnly => Self::SendOnly,
|
||||
RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
|
||||
RtpTransceiverDirection::Inactive => Self::Inactive,
|
||||
RtpTransceiverDirection::Stopped => Self::Stopped,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RtpTransceiverInit> for sys_rt::ffi::RtpTransceiverInit {
|
||||
fn from(value: RtpTransceiverInit) -> Self {
|
||||
Self {
|
||||
direction: value.direction.into(),
|
||||
stream_ids: value.stream_ids,
|
||||
send_encodings: value.send_encodings.into_iter().map(Into::into).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtpTransceiver {
|
||||
pub(crate) sys_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>,
|
||||
}
|
||||
|
||||
impl RtpTransceiver {
|
||||
pub fn mid(&self) -> Option<String> {
|
||||
self.sys_handle.mid().ok()
|
||||
}
|
||||
|
||||
pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
|
||||
self.sys_handle.current_direction().ok().map(Into::into)
|
||||
}
|
||||
|
||||
pub fn direction(&self) -> RtpTransceiverDirection {
|
||||
self.sys_handle.direction().into()
|
||||
}
|
||||
|
||||
pub fn sender(&self) -> rtp_sender::RtpSender {
|
||||
rtp_sender::RtpSender { handle: RtpSender { sys_handle: self.sys_handle.sender() } }
|
||||
}
|
||||
|
||||
pub fn receiver(&self) -> rtp_receiver::RtpReceiver {
|
||||
rtp_receiver::RtpReceiver { handle: RtpReceiver { sys_handle: self.sys_handle.receiver() } }
|
||||
}
|
||||
|
||||
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
|
||||
self.sys_handle
|
||||
.set_codec_preferences(codecs.into_iter().map(Into::into).collect())
|
||||
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
|
||||
}
|
||||
|
||||
pub fn stop(&self) -> Result<(), RtcError> {
|
||||
self.sys_handle
|
||||
.stop_standard()
|
||||
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use cxx::UniquePtr;
|
||||
use webrtc_sys::jsep as sys_jsep;
|
||||
|
||||
use crate::session_description::{self, SdpParseError, SdpType};
|
||||
|
||||
impl From<sys_jsep::ffi::SdpType> for SdpType {
|
||||
fn from(sdp_type: sys_jsep::ffi::SdpType) -> Self {
|
||||
match sdp_type {
|
||||
sys_jsep::ffi::SdpType::Offer => SdpType::Offer,
|
||||
sys_jsep::ffi::SdpType::PrAnswer => SdpType::PrAnswer,
|
||||
sys_jsep::ffi::SdpType::Answer => SdpType::Answer,
|
||||
sys_jsep::ffi::SdpType::Rollback => SdpType::Rollback,
|
||||
_ => panic!("unknown SdpType"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SdpType> for sys_jsep::ffi::SdpType {
|
||||
fn from(sdp_type: SdpType) -> Self {
|
||||
match sdp_type {
|
||||
SdpType::Offer => sys_jsep::ffi::SdpType::Offer,
|
||||
SdpType::PrAnswer => sys_jsep::ffi::SdpType::PrAnswer,
|
||||
SdpType::Answer => sys_jsep::ffi::SdpType::Answer,
|
||||
SdpType::Rollback => sys_jsep::ffi::SdpType::Rollback,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<sys_jsep::ffi::SdpParseError> for SdpParseError {
|
||||
fn from(e: sys_jsep::ffi::SdpParseError) -> Self {
|
||||
Self { line: e.line, description: e.description }
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SessionDescription {
|
||||
pub(crate) sys_handle: UniquePtr<sys_jsep::ffi::SessionDescription>,
|
||||
}
|
||||
|
||||
impl SessionDescription {
|
||||
pub fn parse(
|
||||
sdp: &str,
|
||||
sdp_type: SdpType,
|
||||
) -> Result<session_description::SessionDescription, SdpParseError> {
|
||||
let res = sys_jsep::ffi::create_session_description(sdp_type.into(), sdp.to_owned());
|
||||
match res {
|
||||
Ok(sys_handle) => Ok(session_description::SessionDescription {
|
||||
handle: SessionDescription { sys_handle },
|
||||
}),
|
||||
Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sdp_type(&self) -> SdpType {
|
||||
self.sys_handle.sdp_type().into()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToString for SessionDescription {
|
||||
fn to_string(&self) -> String {
|
||||
self.sys_handle.stringify()
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for SessionDescription {
|
||||
fn clone(&self) -> Self {
|
||||
SessionDescription { sys_handle: self.sys_handle.clone() }
|
||||
}
|
||||
}
|
||||
+956
@@ -0,0 +1,956 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::slice;
|
||||
|
||||
use cxx::UniquePtr;
|
||||
use webrtc_sys::{video_frame as vf_sys, video_frame_buffer as vfb_sys};
|
||||
|
||||
use super::yuv_helper;
|
||||
use crate::video_frame::{self as vf, VideoFormatType, VideoRotation};
|
||||
|
||||
/// We don't use vf::VideoFrameBuffer trait for the types inside this module to avoid confusion
|
||||
/// because directly using platform specific types is not valid (e.g user callback)
|
||||
/// All the types inside this module are only used internally. For public types, see the top level
|
||||
/// video_frame.rs
|
||||
|
||||
pub fn new_video_frame_buffer(
|
||||
mut sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
|
||||
) -> Box<dyn vf::VideoBuffer + Send + Sync> {
|
||||
unsafe {
|
||||
match sys_handle.buffer_type() {
|
||||
vfb_sys::ffi::VideoFrameBufferType::Native => {
|
||||
Box::new(vf::native::NativeBuffer { handle: NativeBuffer { sys_handle } })
|
||||
}
|
||||
vfb_sys::ffi::VideoFrameBufferType::I420 => Box::new(vf::I420Buffer {
|
||||
handle: I420Buffer { sys_handle: sys_handle.pin_mut().get_i420() },
|
||||
}),
|
||||
vfb_sys::ffi::VideoFrameBufferType::I420A => Box::new(vf::I420ABuffer {
|
||||
handle: I420ABuffer { sys_handle: sys_handle.pin_mut().get_i420a() },
|
||||
}),
|
||||
vfb_sys::ffi::VideoFrameBufferType::I422 => Box::new(vf::I422Buffer {
|
||||
handle: I422Buffer { sys_handle: sys_handle.pin_mut().get_i422() },
|
||||
}),
|
||||
vfb_sys::ffi::VideoFrameBufferType::I444 => Box::new(vf::I444Buffer {
|
||||
handle: I444Buffer { sys_handle: sys_handle.pin_mut().get_i444() },
|
||||
}),
|
||||
vfb_sys::ffi::VideoFrameBufferType::I010 => Box::new(vf::I010Buffer {
|
||||
handle: I010Buffer { sys_handle: sys_handle.pin_mut().get_i010() },
|
||||
}),
|
||||
vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer {
|
||||
handle: NV12Buffer { sys_handle: sys_handle.pin_mut().get_nv12() },
|
||||
}),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<vf_sys::ffi::VideoRotation> for VideoRotation {
|
||||
fn from(rotation: vf_sys::ffi::VideoRotation) -> Self {
|
||||
match rotation {
|
||||
vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0,
|
||||
vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90,
|
||||
vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180,
|
||||
vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270,
|
||||
_ => panic!("invalid VideoRotation"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<VideoRotation> for vf_sys::ffi::VideoRotation {
|
||||
fn from(rotation: VideoRotation) -> Self {
|
||||
match rotation {
|
||||
VideoRotation::VideoRotation0 => Self::VideoRotation0,
|
||||
VideoRotation::VideoRotation90 => Self::VideoRotation90,
|
||||
VideoRotation::VideoRotation180 => Self::VideoRotation180,
|
||||
VideoRotation::VideoRotation270 => Self::VideoRotation270,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! recursive_cast {
|
||||
($ptr:expr $(, $fnc:ident)*) => {
|
||||
{
|
||||
let ptr = $ptr;
|
||||
$(
|
||||
let ptr = vfb_sys::ffi::$fnc(ptr);
|
||||
)*
|
||||
ptr
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub struct NativeBuffer {
|
||||
sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
|
||||
}
|
||||
|
||||
pub struct I420Buffer {
|
||||
sys_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
|
||||
}
|
||||
|
||||
pub struct I420ABuffer {
|
||||
sys_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>,
|
||||
}
|
||||
|
||||
pub struct I422Buffer {
|
||||
sys_handle: UniquePtr<vfb_sys::ffi::I422Buffer>,
|
||||
}
|
||||
|
||||
pub struct I444Buffer {
|
||||
sys_handle: UniquePtr<vfb_sys::ffi::I444Buffer>,
|
||||
}
|
||||
|
||||
pub struct I010Buffer {
|
||||
sys_handle: UniquePtr<vfb_sys::ffi::I010Buffer>,
|
||||
}
|
||||
|
||||
pub struct NV12Buffer {
|
||||
sys_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>,
|
||||
}
|
||||
|
||||
macro_rules! impl_to_argb {
|
||||
(I420Buffer [$($variant:ident: $fnc:ident),+], $format:ident, $self:ident, $dst:ident, $dst_stride:ident, $dst_width:ident, $dst_height:ident) => {
|
||||
match $format {
|
||||
$(
|
||||
VideoFormatType::$variant => {
|
||||
let (data_y, data_u, data_v) = $self.data();
|
||||
yuv_helper::$fnc(
|
||||
data_y,
|
||||
$self.stride_y(),
|
||||
data_u,
|
||||
$self.stride_u(),
|
||||
data_v,
|
||||
$self.stride_v(),
|
||||
$dst,
|
||||
$dst_stride,
|
||||
$dst_width,
|
||||
$dst_height,
|
||||
)
|
||||
}
|
||||
)+
|
||||
}
|
||||
};
|
||||
(I420ABuffer) => {
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused_unsafe)]
|
||||
impl NativeBuffer {
|
||||
pub fn from_fluxer_d3d11_texture(
|
||||
handle: u64,
|
||||
width: u32,
|
||||
height: u32,
|
||||
dxgi_format: u32,
|
||||
) -> Option<vf::native::NativeBuffer> {
|
||||
let sys_handle = vfb_sys::ffi::new_fluxer_d3d11_texture_buffer(
|
||||
handle,
|
||||
width,
|
||||
height,
|
||||
dxgi_format,
|
||||
);
|
||||
if sys_handle.is_null() {
|
||||
return None;
|
||||
}
|
||||
Some(vf::native::NativeBuffer {
|
||||
handle: NativeBuffer { sys_handle },
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn from_fluxer_dmabuf_texture(
|
||||
fds: [i32; 4],
|
||||
plane_count: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
drm_format: u32,
|
||||
modifier: u64,
|
||||
strides: [u32; 4],
|
||||
offsets: [u32; 4],
|
||||
device_uuid_hi: u64,
|
||||
device_uuid_lo: u64,
|
||||
) -> Option<vf::native::NativeBuffer> {
|
||||
let sys_handle = vfb_sys::ffi::new_fluxer_dmabuf_texture_buffer(
|
||||
fds[0],
|
||||
fds[1],
|
||||
fds[2],
|
||||
fds[3],
|
||||
plane_count,
|
||||
width,
|
||||
height,
|
||||
drm_format,
|
||||
modifier,
|
||||
strides[0],
|
||||
strides[1],
|
||||
strides[2],
|
||||
strides[3],
|
||||
offsets[0],
|
||||
offsets[1],
|
||||
offsets[2],
|
||||
offsets[3],
|
||||
device_uuid_hi,
|
||||
device_uuid_lo,
|
||||
);
|
||||
if sys_handle.is_null() {
|
||||
return None;
|
||||
}
|
||||
Some(vf::native::NativeBuffer {
|
||||
handle: NativeBuffer { sys_handle },
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "macos", target_os = "ios"))]
|
||||
pub unsafe fn from_cv_pixel_buffer(
|
||||
cv_pixel_buffer: *mut std::ffi::c_void,
|
||||
) -> vf::native::NativeBuffer {
|
||||
vf::native::NativeBuffer {
|
||||
handle: NativeBuffer {
|
||||
sys_handle: vfb_sys::ffi::new_native_buffer_from_platform_image_buffer(
|
||||
cv_pixel_buffer as *mut _,
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "macos", target_os = "ios"))]
|
||||
pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void {
|
||||
unsafe { vfb_sys::ffi::native_buffer_to_platform_image_buffer(&self.sys_handle) as *mut _ }
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
|
||||
&self.sys_handle
|
||||
}
|
||||
|
||||
pub fn width(&self) -> u32 {
|
||||
self.sys_handle.width()
|
||||
}
|
||||
|
||||
pub fn height(&self) -> u32 {
|
||||
self.sys_handle.height()
|
||||
}
|
||||
|
||||
pub fn to_i420(&self) -> I420Buffer {
|
||||
I420Buffer { sys_handle: unsafe { self.sys_handle.to_i420() } }
|
||||
}
|
||||
|
||||
pub fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
) {
|
||||
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
|
||||
}
|
||||
}
|
||||
|
||||
impl I420Buffer {
|
||||
pub fn new(
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_u: u32,
|
||||
stride_v: u32,
|
||||
) -> vf::I420Buffer {
|
||||
vf::I420Buffer {
|
||||
handle: I420Buffer {
|
||||
sys_handle: vfb_sys::ffi::new_i420_buffer(
|
||||
width.try_into().unwrap(),
|
||||
height.try_into().unwrap(),
|
||||
stride_y.try_into().unwrap(),
|
||||
stride_u.try_into().unwrap(),
|
||||
stride_v.try_into().unwrap(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
|
||||
unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
|
||||
}
|
||||
|
||||
pub fn width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).chroma_width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).chroma_height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_y(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_y()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_u(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_u()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_v(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_v()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_i420(&self) -> I420Buffer {
|
||||
I420Buffer {
|
||||
sys_handle: unsafe {
|
||||
// We make a copy of the buffer because internally, when calling ToI420()
|
||||
// if the buffer is of type I420, libwebrtc will reuse the same underlying pointer
|
||||
// for the new created type
|
||||
let copy = vfb_sys::ffi::copy_i420_buffer(&self.sys_handle);
|
||||
let ptr = recursive_cast!(&*copy, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).to_i420()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
) {
|
||||
impl_to_argb!(
|
||||
I420Buffer
|
||||
[
|
||||
ARGB: i420_to_argb,
|
||||
BGRA: i420_to_bgra,
|
||||
ABGR: i420_to_abgr,
|
||||
RGBA: i420_to_rgba
|
||||
],
|
||||
format, self, dst, dst_stride, dst_width, dst_height
|
||||
)
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8);
|
||||
let chroma_height = (self.height() + 1) / 2;
|
||||
(
|
||||
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
|
||||
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
|
||||
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420Buffer {
|
||||
vf::I420Buffer {
|
||||
handle: I420Buffer {
|
||||
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl I420ABuffer {
|
||||
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
|
||||
unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
|
||||
}
|
||||
|
||||
pub fn width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).chroma_width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).chroma_height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_y(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_y()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_u(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_u()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_v(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_v()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_a(&self) -> u32 {
|
||||
self.sys_handle.stride_a()
|
||||
}
|
||||
|
||||
pub fn to_i420(&self) -> I420Buffer {
|
||||
I420Buffer {
|
||||
sys_handle: unsafe {
|
||||
let ptr =
|
||||
recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).to_i420()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
) {
|
||||
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8);
|
||||
let chroma_height = (self.height() + 1) / 2;
|
||||
let data_a = self.sys_handle.data_a();
|
||||
let has_data_a = !data_a.is_null();
|
||||
(
|
||||
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
|
||||
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
|
||||
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
|
||||
has_data_a.then_some(slice::from_raw_parts(
|
||||
data_a,
|
||||
(self.stride_a() * self.height()) as usize,
|
||||
)),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420ABuffer {
|
||||
vf::I420ABuffer {
|
||||
handle: I420ABuffer {
|
||||
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl I422Buffer {
|
||||
pub fn new(
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_u: u32,
|
||||
stride_v: u32,
|
||||
) -> vf::I422Buffer {
|
||||
vf::I422Buffer {
|
||||
handle: I422Buffer {
|
||||
sys_handle: vfb_sys::ffi::new_i422_buffer(
|
||||
width.try_into().unwrap(),
|
||||
height.try_into().unwrap(),
|
||||
stride_y.try_into().unwrap(),
|
||||
stride_u.try_into().unwrap(),
|
||||
stride_v.try_into().unwrap(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
|
||||
unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
|
||||
}
|
||||
|
||||
pub fn width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).chroma_width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).chroma_height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_y(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_y()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_u(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_u()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_v(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_v()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_i420(&self) -> I420Buffer {
|
||||
I420Buffer {
|
||||
sys_handle: unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).to_i420()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
) {
|
||||
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8);
|
||||
(
|
||||
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
|
||||
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
|
||||
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I422Buffer {
|
||||
vf::I422Buffer {
|
||||
handle: I422Buffer {
|
||||
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
impl I444Buffer {
|
||||
pub fn new(
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_u: u32,
|
||||
stride_v: u32,
|
||||
) -> vf::I444Buffer {
|
||||
vf::I444Buffer {
|
||||
handle: I444Buffer {
|
||||
sys_handle: vfb_sys::ffi::new_i444_buffer(
|
||||
width.try_into().unwrap(),
|
||||
height.try_into().unwrap(),
|
||||
stride_y.try_into().unwrap(),
|
||||
stride_u.try_into().unwrap(),
|
||||
stride_v.try_into().unwrap(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
|
||||
unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
|
||||
}
|
||||
|
||||
pub fn width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).chroma_width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).chroma_height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_y(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_y()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_u(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_u()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_v(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
|
||||
(*ptr).stride_v()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_i420(&self) -> I420Buffer {
|
||||
I420Buffer {
|
||||
sys_handle: unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
|
||||
(*ptr).to_i420()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
) {
|
||||
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8);
|
||||
(
|
||||
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
|
||||
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
|
||||
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I444Buffer {
|
||||
vf::I444Buffer {
|
||||
handle: I444Buffer {
|
||||
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl I010Buffer {
|
||||
pub fn new(
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_u: u32,
|
||||
stride_v: u32,
|
||||
) -> vf::I010Buffer {
|
||||
vf::I010Buffer {
|
||||
handle: I010Buffer {
|
||||
sys_handle: vfb_sys::ffi::new_i010_buffer(
|
||||
width.try_into().unwrap(),
|
||||
height.try_into().unwrap(),
|
||||
stride_y.try_into().unwrap(),
|
||||
stride_u.try_into().unwrap(),
|
||||
stride_v.try_into().unwrap(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
|
||||
unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) }
|
||||
}
|
||||
|
||||
pub fn width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
|
||||
(*ptr).width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
|
||||
(*ptr).height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
|
||||
(*ptr).chroma_width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
|
||||
(*ptr).chroma_height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_y(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
|
||||
(*ptr).stride_y()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_u(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
|
||||
(*ptr).stride_u()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_v(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
|
||||
(*ptr).stride_v()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_i420(&self) -> I420Buffer {
|
||||
I420Buffer {
|
||||
sys_handle: unsafe {
|
||||
let ptr =
|
||||
recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
|
||||
(*ptr).to_i420()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
) {
|
||||
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b);
|
||||
let chroma_height = (self.height() + 1) / 2;
|
||||
(
|
||||
slice::from_raw_parts(
|
||||
(*ptr).data_y(),
|
||||
(self.stride_y() * self.height()) as usize / 2,
|
||||
),
|
||||
slice::from_raw_parts(
|
||||
(*ptr).data_u(),
|
||||
(self.stride_u() * chroma_height) as usize / 2,
|
||||
),
|
||||
slice::from_raw_parts(
|
||||
(*ptr).data_v(),
|
||||
(self.stride_v() * chroma_height) as usize / 2,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I010Buffer {
|
||||
vf::I010Buffer {
|
||||
handle: I010Buffer {
|
||||
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NV12Buffer {
|
||||
pub fn new(width: u32, height: u32, stride_y: u32, stride_uv: u32) -> vf::NV12Buffer {
|
||||
vf::NV12Buffer {
|
||||
handle: NV12Buffer {
|
||||
sys_handle: vfb_sys::ffi::new_nv12_buffer(
|
||||
width.try_into().unwrap(),
|
||||
height.try_into().unwrap(),
|
||||
stride_y.try_into().unwrap(),
|
||||
stride_uv.try_into().unwrap(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
|
||||
unsafe {
|
||||
&*recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr =
|
||||
recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb);
|
||||
(*ptr).width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr =
|
||||
recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb);
|
||||
(*ptr).height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
|
||||
(*ptr).chroma_width()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
|
||||
(*ptr).chroma_height()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_y(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
|
||||
(*ptr).stride_y()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stride_uv(&self) -> u32 {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
|
||||
(*ptr).stride_uv()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_i420(&self) -> I420Buffer {
|
||||
I420Buffer {
|
||||
sys_handle: unsafe {
|
||||
let ptr = recursive_cast!(
|
||||
&*self.sys_handle,
|
||||
nv12_to_biyuv8,
|
||||
biyuv8_to_biyuv,
|
||||
biyuv_to_vfb
|
||||
);
|
||||
(*ptr).to_i420()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
) {
|
||||
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8]) {
|
||||
unsafe {
|
||||
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8);
|
||||
let chroma_height = (self.height() + 1) / 2;
|
||||
|
||||
(
|
||||
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
|
||||
slice::from_raw_parts(
|
||||
(*ptr).data_uv(),
|
||||
(self.stride_uv() * chroma_height) as usize,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::NV12Buffer {
|
||||
vf::NV12Buffer {
|
||||
handle: NV12Buffer {
|
||||
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use livekit_runtime::interval;
|
||||
use parking_lot::Mutex;
|
||||
use webrtc_sys::{video_frame as vf_sys, video_frame::ffi::VideoRotation, video_track as vt_sys};
|
||||
|
||||
use crate::{
|
||||
native::packet_trailer::PacketTrailerHandler,
|
||||
video_frame::{I420Buffer, VideoBuffer, VideoFrame},
|
||||
video_source::VideoResolution,
|
||||
};
|
||||
|
||||
impl From<vt_sys::ffi::VideoResolution> for VideoResolution {
|
||||
fn from(res: vt_sys::ffi::VideoResolution) -> Self {
|
||||
Self { width: res.width, height: res.height }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<VideoResolution> for vt_sys::ffi::VideoResolution {
|
||||
fn from(res: VideoResolution) -> Self {
|
||||
Self { width: res.width, height: res.height }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NativeVideoSource {
|
||||
sys_handle: SharedPtr<vt_sys::ffi::VideoTrackSource>,
|
||||
inner: Arc<Mutex<VideoSourceInner>>,
|
||||
}
|
||||
|
||||
struct VideoSourceInner {
|
||||
captured_frames: usize,
|
||||
}
|
||||
|
||||
impl NativeVideoSource {
|
||||
pub fn new(resolution: VideoResolution, is_screencast: bool) -> NativeVideoSource {
|
||||
let source = Self {
|
||||
sys_handle: vt_sys::ffi::new_video_track_source(
|
||||
&vt_sys::ffi::VideoResolution::from(resolution.clone()),
|
||||
is_screencast,
|
||||
),
|
||||
inner: Arc::new(Mutex::new(VideoSourceInner { captured_frames: 0 })),
|
||||
};
|
||||
|
||||
livekit_runtime::spawn({
|
||||
let source = source.clone();
|
||||
let i420 = I420Buffer::new(resolution.width, resolution.height);
|
||||
async move {
|
||||
let mut interval = interval(Duration::from_millis(100)); // 10 fps
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
let inner = source.inner.lock();
|
||||
if inner.captured_frames > 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let mut builder = vf_sys::ffi::new_video_frame_builder();
|
||||
builder.pin_mut().set_rotation(VideoRotation::VideoRotation0);
|
||||
builder.pin_mut().set_video_frame_buffer(i420.as_ref().sys_handle());
|
||||
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
|
||||
builder.pin_mut().set_timestamp_us(now.as_micros() as i64);
|
||||
|
||||
source.sys_handle.on_captured_frame(
|
||||
&builder.pin_mut().build(),
|
||||
&vt_sys::ffi::FrameMetadata {
|
||||
has_packet_trailer: false,
|
||||
user_timestamp: 0,
|
||||
frame_id: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
source
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> SharedPtr<vt_sys::ffi::VideoTrackSource> {
|
||||
self.sys_handle.clone()
|
||||
}
|
||||
|
||||
pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {
|
||||
let mut builder = vf_sys::ffi::new_video_frame_builder();
|
||||
builder.pin_mut().set_rotation(frame.rotation.into());
|
||||
builder.pin_mut().set_video_frame_buffer(frame.buffer.as_ref().sys_handle());
|
||||
|
||||
let capture_ts = if frame.timestamp_us == 0 {
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
|
||||
now.as_micros() as i64
|
||||
} else {
|
||||
frame.timestamp_us
|
||||
};
|
||||
builder.pin_mut().set_timestamp_us(capture_ts);
|
||||
|
||||
let (has_trailer, user_ts, fid) = match frame.frame_metadata {
|
||||
Some(meta) => (true, meta.user_timestamp.unwrap_or(0), meta.frame_id.unwrap_or(0)),
|
||||
None => (false, 0, 0),
|
||||
};
|
||||
|
||||
self.inner.lock().captured_frames += 1;
|
||||
|
||||
self.sys_handle.on_captured_frame(
|
||||
&builder.pin_mut().build(),
|
||||
&vt_sys::ffi::FrameMetadata {
|
||||
has_packet_trailer: has_trailer,
|
||||
user_timestamp: user_ts,
|
||||
frame_id: fid,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Set the packet trailer handler used by this source.
|
||||
///
|
||||
/// When set, any frame captured with a `user_timestamp` value will
|
||||
/// automatically have its timestamp stored in the handler so the
|
||||
/// `PacketTrailerTransformer` can embed it into the encoded frame.
|
||||
/// The handler is set on the C++ VideoTrackSource so it has access to
|
||||
/// the TimestampAligner-adjusted capture timestamp for correct keying.
|
||||
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
|
||||
self.sys_handle.set_packet_trailer_handler(handler.sys_handle());
|
||||
}
|
||||
|
||||
pub fn video_resolution(&self) -> VideoResolution {
|
||||
self.sys_handle.video_resolution().into()
|
||||
}
|
||||
}
|
||||
+350
@@ -0,0 +1,350 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
task::{Context, Poll, Waker},
|
||||
};
|
||||
|
||||
use cxx::{SharedPtr, UniquePtr};
|
||||
use livekit_runtime::Stream;
|
||||
use parking_lot::Mutex;
|
||||
use rtrb::{Consumer, Producer, PushError, RingBuffer};
|
||||
use webrtc_sys::video_track as sys_vt;
|
||||
|
||||
use super::{packet_trailer::SubscribeTimingStage, video_frame::new_video_frame_buffer};
|
||||
use crate::{
|
||||
native::packet_trailer::PacketTrailerHandler,
|
||||
video_frame::{BoxVideoFrame, FrameMetadata, VideoFrame},
|
||||
video_track::RtcVideoTrack,
|
||||
};
|
||||
|
||||
pub struct NativeVideoStream {
|
||||
native_sink: SharedPtr<sys_vt::ffi::NativeVideoSink>,
|
||||
observer: Arc<VideoTrackObserver>,
|
||||
video_track: RtcVideoTrack,
|
||||
frame_queue: Arc<VideoFrameQueue>,
|
||||
}
|
||||
|
||||
impl NativeVideoStream {
|
||||
pub fn new(video_track: RtcVideoTrack, queue_size_frames: Option<usize>) -> Self {
|
||||
let frame_queue = Arc::new(VideoFrameQueue::new(queue_size_frames));
|
||||
// Auto-wire the packet trailer handler from the track if one is set.
|
||||
let handler = video_track.handle.packet_trailer_handler();
|
||||
let observer = Arc::new(VideoTrackObserver {
|
||||
frame_queue: frame_queue.clone(),
|
||||
packet_trailer_handler: parking_lot::Mutex::new(handler),
|
||||
});
|
||||
let native_sink = sys_vt::ffi::new_native_video_sink(Box::new(
|
||||
sys_vt::VideoSinkWrapper::new(observer.clone()),
|
||||
));
|
||||
|
||||
let video = unsafe { sys_vt::ffi::media_to_video(video_track.sys_handle()) };
|
||||
video.add_sink(&native_sink);
|
||||
|
||||
Self { native_sink, observer, video_track, frame_queue }
|
||||
}
|
||||
|
||||
/// Set the packet trailer handler for this stream.
|
||||
///
|
||||
/// When set, each frame produced by this stream will have its
|
||||
/// `user_timestamp` field populated from the handler's receive
|
||||
/// map (looked up by RTP timestamp).
|
||||
///
|
||||
/// Note: If the handler was already set on the `RtcVideoTrack` before
|
||||
/// creating this stream, it is automatically wired up. This method is
|
||||
/// only needed if you want to override or set the handler after
|
||||
/// construction.
|
||||
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
|
||||
*self.observer.packet_trailer_handler.lock() = Some(handler);
|
||||
}
|
||||
|
||||
pub fn track(&self) -> RtcVideoTrack {
|
||||
self.video_track.clone()
|
||||
}
|
||||
|
||||
pub fn close(&mut self) {
|
||||
let video = unsafe { sys_vt::ffi::media_to_video(self.video_track.sys_handle()) };
|
||||
video.remove_sink(&self.native_sink);
|
||||
self.frame_queue.close();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for NativeVideoStream {
|
||||
fn drop(&mut self) {
|
||||
self.close();
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for NativeVideoStream {
|
||||
type Item = BoxVideoFrame;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
self.frame_queue.poll_recv(cx)
|
||||
}
|
||||
}
|
||||
|
||||
struct VideoTrackObserver {
|
||||
frame_queue: Arc<VideoFrameQueue>,
|
||||
packet_trailer_handler: parking_lot::Mutex<Option<PacketTrailerHandler>>,
|
||||
}
|
||||
|
||||
impl sys_vt::VideoSink for VideoTrackObserver {
|
||||
fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
|
||||
let rtp_timestamp = frame.timestamp();
|
||||
let packet_trailer_handler = self.packet_trailer_handler.lock().clone();
|
||||
let frame_metadata = packet_trailer_handler
|
||||
.as_ref()
|
||||
.and_then(|handler| {
|
||||
handler.lookup_frame_metadata(rtp_timestamp).map(|(ts, fid)| {
|
||||
handler.emit_subscribe_timing(SubscribeTimingStage::DecoderOutput, ts, fid);
|
||||
(ts, fid)
|
||||
})
|
||||
})
|
||||
.map(|(ts, fid)| FrameMetadata {
|
||||
user_timestamp: Some(ts),
|
||||
frame_id: if fid != 0 { Some(fid) } else { None },
|
||||
});
|
||||
|
||||
self.frame_queue.push(VideoFrame {
|
||||
rotation: frame.rotation().into(),
|
||||
timestamp_us: frame.timestamp_us(),
|
||||
frame_metadata,
|
||||
buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }),
|
||||
});
|
||||
}
|
||||
|
||||
fn on_discarded_frame(&self) {}
|
||||
|
||||
fn on_constraints_changed(&self, _constraints: sys_vt::ffi::VideoTrackSourceConstraints) {}
|
||||
}
|
||||
|
||||
struct VideoFrameQueue {
|
||||
kind: VideoFrameQueueKind,
|
||||
closed: AtomicBool,
|
||||
dropped_frames: AtomicU64,
|
||||
waker: Mutex<Option<Waker>>,
|
||||
}
|
||||
|
||||
enum VideoFrameQueueKind {
|
||||
Bounded(BoundedVideoFrameQueue),
|
||||
Unbounded(UnboundedVideoFrameQueue),
|
||||
}
|
||||
|
||||
struct BoundedVideoFrameQueue {
|
||||
producer: Mutex<Producer<BoxVideoFrame>>,
|
||||
consumer: Mutex<Consumer<BoxVideoFrame>>,
|
||||
}
|
||||
|
||||
struct UnboundedVideoFrameQueue {
|
||||
frames: Mutex<VecDeque<BoxVideoFrame>>,
|
||||
}
|
||||
|
||||
impl VideoFrameQueue {
|
||||
fn new(capacity: Option<usize>) -> Self {
|
||||
let kind = match capacity.filter(|capacity| *capacity > 0) {
|
||||
Some(capacity) => {
|
||||
let (producer, consumer) = RingBuffer::new(capacity);
|
||||
VideoFrameQueueKind::Bounded(BoundedVideoFrameQueue {
|
||||
producer: Mutex::new(producer),
|
||||
consumer: Mutex::new(consumer),
|
||||
})
|
||||
}
|
||||
None => VideoFrameQueueKind::Unbounded(UnboundedVideoFrameQueue {
|
||||
frames: Mutex::new(VecDeque::new()),
|
||||
}),
|
||||
};
|
||||
|
||||
Self {
|
||||
kind,
|
||||
closed: AtomicBool::new(false),
|
||||
dropped_frames: AtomicU64::new(0),
|
||||
waker: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn push(&self, frame: BoxVideoFrame) {
|
||||
if self.closed.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
|
||||
match &self.kind {
|
||||
VideoFrameQueueKind::Bounded(queue) => self.push_bounded(queue, frame),
|
||||
VideoFrameQueueKind::Unbounded(queue) => {
|
||||
queue.frames.lock().push_back(frame);
|
||||
}
|
||||
}
|
||||
|
||||
self.wake_receiver();
|
||||
}
|
||||
|
||||
fn push_bounded(&self, queue: &BoundedVideoFrameQueue, mut frame: BoxVideoFrame) {
|
||||
loop {
|
||||
let push_result = queue.producer.lock().push(frame);
|
||||
match push_result {
|
||||
Ok(()) => return,
|
||||
Err(PushError::Full(returned_frame)) => {
|
||||
frame = returned_frame;
|
||||
|
||||
let dropped = queue.consumer.lock().pop().is_ok();
|
||||
if dropped {
|
||||
self.record_drop();
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&self) {
|
||||
self.closed.store(true, Ordering::Release);
|
||||
self.wake_receiver();
|
||||
|
||||
match &self.kind {
|
||||
VideoFrameQueueKind::Bounded(queue) => {
|
||||
let mut consumer = queue.consumer.lock();
|
||||
while consumer.pop().is_ok() {}
|
||||
}
|
||||
VideoFrameQueueKind::Unbounded(queue) => {
|
||||
queue.frames.lock().clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Option<BoxVideoFrame>> {
|
||||
if let Some(frame) = self.try_pop() {
|
||||
return Poll::Ready(Some(frame));
|
||||
}
|
||||
|
||||
if self.closed.load(Ordering::Acquire) {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
|
||||
*self.waker.lock() = Some(cx.waker().clone());
|
||||
|
||||
if let Some(frame) = self.try_pop() {
|
||||
self.waker.lock().take();
|
||||
Poll::Ready(Some(frame))
|
||||
} else if self.closed.load(Ordering::Acquire) {
|
||||
Poll::Ready(None)
|
||||
} else {
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
fn try_pop(&self) -> Option<BoxVideoFrame> {
|
||||
match &self.kind {
|
||||
VideoFrameQueueKind::Bounded(queue) => queue.consumer.lock().pop().ok(),
|
||||
VideoFrameQueueKind::Unbounded(queue) => queue.frames.lock().pop_front(),
|
||||
}
|
||||
}
|
||||
|
||||
fn wake_receiver(&self) {
|
||||
let waker = self.waker.lock().take();
|
||||
if let Some(waker) = waker {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
|
||||
fn record_drop(&self) {
|
||||
let dropped_frames = self.dropped_frames.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if dropped_frames == 1 || dropped_frames % 100 == 0 {
|
||||
log::warn!(
|
||||
"native video stream queue overflow; dropped {} queued frames",
|
||||
dropped_frames
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use super::VideoFrameQueue;
|
||||
use crate::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation};
|
||||
|
||||
fn test_frame(timestamp_us: i64) -> BoxVideoFrame {
|
||||
VideoFrame {
|
||||
rotation: VideoRotation::VideoRotation0,
|
||||
timestamp_us,
|
||||
frame_metadata: None,
|
||||
buffer: Box::new(I420Buffer::new(2, 2)),
|
||||
}
|
||||
}
|
||||
|
||||
fn pop_timestamp(queue: &VideoFrameQueue) -> Option<i64> {
|
||||
queue.try_pop().map(|frame| frame.timestamp_us)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_queue_preserves_fifo_order_under_capacity() {
|
||||
let queue = VideoFrameQueue::new(Some(3));
|
||||
|
||||
queue.push(test_frame(1));
|
||||
queue.push(test_frame(2));
|
||||
queue.push(test_frame(3));
|
||||
|
||||
assert_eq!(pop_timestamp(&queue), Some(1));
|
||||
assert_eq!(pop_timestamp(&queue), Some(2));
|
||||
assert_eq!(pop_timestamp(&queue), Some(3));
|
||||
assert_eq!(pop_timestamp(&queue), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_queue_drops_oldest_when_full() {
|
||||
let queue = VideoFrameQueue::new(Some(2));
|
||||
|
||||
queue.push(test_frame(1));
|
||||
queue.push(test_frame(2));
|
||||
queue.push(test_frame(3));
|
||||
|
||||
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(pop_timestamp(&queue), Some(2));
|
||||
assert_eq!(pop_timestamp(&queue), Some(3));
|
||||
assert_eq!(pop_timestamp(&queue), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unbounded_queue_retains_all_frames() {
|
||||
let queue = VideoFrameQueue::new(None);
|
||||
|
||||
for timestamp_us in 1..=4 {
|
||||
queue.push(test_frame(timestamp_us));
|
||||
}
|
||||
|
||||
for timestamp_us in 1..=4 {
|
||||
assert_eq!(pop_timestamp(&queue), Some(timestamp_us));
|
||||
}
|
||||
assert_eq!(pop_timestamp(&queue), None);
|
||||
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn close_clears_buffer_and_rejects_future_pushes() {
|
||||
let queue = VideoFrameQueue::new(Some(2));
|
||||
|
||||
queue.push(test_frame(1));
|
||||
queue.close();
|
||||
queue.push(test_frame(2));
|
||||
|
||||
assert_eq!(pop_timestamp(&queue), None);
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use cxx::SharedPtr;
|
||||
use parking_lot::Mutex;
|
||||
use sys_vt::ffi::video_to_media;
|
||||
use webrtc_sys::video_track as sys_vt;
|
||||
|
||||
use super::media_stream_track::impl_media_stream_track;
|
||||
use super::packet_trailer::PacketTrailerHandler;
|
||||
use crate::media_stream_track::RtcTrackState;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtcVideoTrack {
|
||||
pub(crate) sys_handle: SharedPtr<sys_vt::ffi::VideoTrack>,
|
||||
packet_trailer_handler: Arc<Mutex<Option<PacketTrailerHandler>>>,
|
||||
}
|
||||
|
||||
impl RtcVideoTrack {
|
||||
impl_media_stream_track!(video_to_media);
|
||||
|
||||
pub(crate) fn new(sys_handle: SharedPtr<sys_vt::ffi::VideoTrack>) -> Self {
|
||||
Self { sys_handle, packet_trailer_handler: Arc::new(Mutex::new(None)) }
|
||||
}
|
||||
|
||||
pub fn sys_handle(&self) -> SharedPtr<sys_vt::ffi::MediaStreamTrack> {
|
||||
video_to_media(self.sys_handle.clone())
|
||||
}
|
||||
|
||||
/// Set the packet trailer handler for this track.
|
||||
///
|
||||
/// When set, any `NativeVideoStream` created from this track will
|
||||
/// automatically use this handler to populate `user_timestamp`
|
||||
/// on each decoded frame.
|
||||
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
|
||||
self.packet_trailer_handler.lock().replace(handler);
|
||||
}
|
||||
|
||||
/// Get the packet trailer handler, if one has been set.
|
||||
pub fn packet_trailer_handler(&self) -> Option<PacketTrailerHandler> {
|
||||
self.packet_trailer_handler.lock().clone()
|
||||
}
|
||||
}
|
||||
+868
@@ -0,0 +1,868 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![allow(clippy::too_many_arguments)]
|
||||
|
||||
use webrtc_sys::yuv_helper as yuv_sys;
|
||||
|
||||
fn argb_assert_safety(src: &[u8], src_stride: u32, _width: i32, height: i32) {
|
||||
let height_abs = height.unsigned_abs();
|
||||
let min = (src_stride * height_abs) as usize;
|
||||
assert!(src.len() >= min, "src isn't large enough");
|
||||
}
|
||||
|
||||
fn i420_assert_safety(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
_width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
let height_abs = height.unsigned_abs();
|
||||
let chroma_height = (height_abs + 1) / 2;
|
||||
let min_y = (src_stride_y * height_abs) as usize;
|
||||
let min_u = (src_stride_u * chroma_height) as usize;
|
||||
let min_v = (src_stride_v * chroma_height) as usize;
|
||||
|
||||
assert!(src_y.len() >= min_y, "src_y isn't large enough");
|
||||
assert!(src_u.len() >= min_u, "src_u isn't large enough");
|
||||
assert!(src_v.len() >= min_v, "src_v isn't large enough");
|
||||
}
|
||||
|
||||
fn nv12_assert_safety(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_uv: &[u8],
|
||||
src_stride_uv: u32,
|
||||
_width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
let height_abs = height.unsigned_abs();
|
||||
let chroma_height = (height_abs + 1) / 2;
|
||||
|
||||
let min_y = (src_stride_y * height_abs) as usize;
|
||||
let min_uv = (src_stride_uv * chroma_height) as usize;
|
||||
|
||||
assert!(src_y.len() >= min_y, "src_y isn't large enough");
|
||||
assert!(src_uv.len() >= min_uv, "src_uv isn't large enough");
|
||||
}
|
||||
|
||||
fn i444_assert_safety(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
_width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
let height_abs = height.unsigned_abs();
|
||||
let min_y = (src_stride_y * height_abs) as usize;
|
||||
let min_u = (src_stride_u * height_abs) as usize;
|
||||
let min_v = (src_stride_v * height_abs) as usize;
|
||||
|
||||
assert!(src_y.len() >= min_y, "src_y isn't large enough");
|
||||
assert!(src_u.len() >= min_u, "src_u isn't large enough");
|
||||
assert!(src_v.len() >= min_v, "src_v isn't large enough");
|
||||
}
|
||||
|
||||
fn i422_assert_safety(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
_width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
let height_abs = height.unsigned_abs();
|
||||
let min_y = (src_stride_y * height_abs) as usize;
|
||||
let min_u = (src_stride_u * height_abs) as usize;
|
||||
let min_v = (src_stride_v * height_abs) as usize;
|
||||
|
||||
assert!(src_y.len() >= min_y, "src_y isn't large enough");
|
||||
assert!(src_u.len() >= min_u, "src_u isn't large enough");
|
||||
assert!(src_v.len() >= min_v, "src_v isn't large enough");
|
||||
}
|
||||
|
||||
fn i010_assert_safety(
|
||||
src_y: &[u16],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u16],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u16],
|
||||
src_stride_v: u32,
|
||||
_width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
let height_abs: u32 = height.unsigned_abs();
|
||||
let chroma_height = height_abs / 2;
|
||||
let min_y = (src_stride_y * height_abs) as usize / 2;
|
||||
let min_u = (src_stride_u * chroma_height) as usize / 2;
|
||||
let min_v = (src_stride_v * chroma_height) as usize / 2;
|
||||
|
||||
assert!(src_y.len() >= min_y, "src_y isn't large enough");
|
||||
assert!(src_u.len() >= min_u, "src_u isn't large enough");
|
||||
assert!(src_v.len() >= min_v, "src_v isn't large enough");
|
||||
}
|
||||
|
||||
macro_rules! i420_to_rgba {
|
||||
($x:ident) => {
|
||||
pub fn $x(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i420_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
argb_assert_safety(dst, dst_stride, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::$x(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst.as_mut_ptr(),
|
||||
dst_stride as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! rgba_to_i420 {
|
||||
($x:ident) => {
|
||||
pub fn $x(
|
||||
src_argb: &[u8],
|
||||
src_stride_argb: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_u: &mut [u8],
|
||||
dst_stride_u: u32,
|
||||
dst_v: &mut [u8],
|
||||
dst_stride_v: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i420_assert_safety(
|
||||
dst_y,
|
||||
dst_stride_y,
|
||||
dst_u,
|
||||
dst_stride_u,
|
||||
dst_v,
|
||||
dst_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
argb_assert_safety(src_argb, src_stride_argb, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::$x(
|
||||
src_argb.as_ptr(),
|
||||
src_stride_argb as i32,
|
||||
dst_y.as_mut_ptr(),
|
||||
dst_stride_y as i32,
|
||||
dst_u.as_mut_ptr(),
|
||||
dst_stride_u as i32,
|
||||
dst_v.as_mut_ptr(),
|
||||
dst_stride_v as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub fn argb_to_rgb24(
|
||||
src_argb: &[u8],
|
||||
src_stride_argb: u32,
|
||||
dst_rgb24: &mut [u8],
|
||||
dst_stride_rgb24: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
argb_assert_safety(src_argb, src_stride_argb, width, height);
|
||||
argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::argb_to_rgb24(
|
||||
src_argb.as_ptr(),
|
||||
src_stride_argb as i32,
|
||||
dst_rgb24.as_mut_ptr(),
|
||||
dst_stride_rgb24 as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// I420 <> RGB conversion
|
||||
rgba_to_i420!(argb_to_i420);
|
||||
rgba_to_i420!(abgr_to_i420);
|
||||
|
||||
i420_to_rgba!(i420_to_argb);
|
||||
i420_to_rgba!(i420_to_bgra);
|
||||
i420_to_rgba!(i420_to_abgr);
|
||||
i420_to_rgba!(i420_to_rgba);
|
||||
|
||||
pub fn i420_to_nv12(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_uv: &mut [u8],
|
||||
dst_stride_uv: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i420_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i420_to_nv12(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_y.as_mut_ptr(),
|
||||
dst_stride_y as i32,
|
||||
dst_uv.as_mut_ptr(),
|
||||
dst_stride_uv as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn nv12_to_i420(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_uv: &[u8],
|
||||
src_stride_uv: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_u: &mut [u8],
|
||||
dst_stride_u: u32,
|
||||
dst_v: &mut [u8],
|
||||
dst_stride_v: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
|
||||
i420_assert_safety(
|
||||
dst_y,
|
||||
dst_stride_y,
|
||||
dst_u,
|
||||
dst_stride_u,
|
||||
dst_v,
|
||||
dst_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::nv12_to_i420(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_uv.as_ptr(),
|
||||
src_stride_uv as i32,
|
||||
dst_y.as_mut_ptr(),
|
||||
dst_stride_y as i32,
|
||||
dst_u.as_mut_ptr(),
|
||||
dst_stride_u as i32,
|
||||
dst_v.as_mut_ptr(),
|
||||
dst_stride_v as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i444_to_i420(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_u: &mut [u8],
|
||||
dst_stride_u: u32,
|
||||
dst_v: &mut [u8],
|
||||
dst_stride_v: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i444_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
i420_assert_safety(
|
||||
dst_y,
|
||||
dst_stride_y,
|
||||
dst_u,
|
||||
dst_stride_u,
|
||||
dst_v,
|
||||
dst_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i444_to_i420(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_y.as_mut_ptr(),
|
||||
dst_stride_y as i32,
|
||||
dst_u.as_mut_ptr(),
|
||||
dst_stride_u as i32,
|
||||
dst_v.as_mut_ptr(),
|
||||
dst_stride_v as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i422_to_i420(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_u: &mut [u8],
|
||||
dst_stride_u: u32,
|
||||
dst_v: &mut [u8],
|
||||
dst_stride_v: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i422_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
i420_assert_safety(
|
||||
dst_y,
|
||||
dst_stride_y,
|
||||
dst_u,
|
||||
dst_stride_u,
|
||||
dst_v,
|
||||
dst_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i422_to_i420(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_y.as_mut_ptr(),
|
||||
dst_stride_y as i32,
|
||||
dst_u.as_mut_ptr(),
|
||||
dst_stride_u as i32,
|
||||
dst_v.as_mut_ptr(),
|
||||
dst_stride_v as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i010_to_i420(
|
||||
src_y: &[u16],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u16],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u16],
|
||||
src_stride_v: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_u: &mut [u8],
|
||||
dst_stride_u: u32,
|
||||
dst_v: &mut [u8],
|
||||
dst_stride_v: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i010_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
i420_assert_safety(
|
||||
dst_y,
|
||||
dst_stride_y,
|
||||
dst_u,
|
||||
dst_stride_u,
|
||||
dst_v,
|
||||
dst_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i010_to_i420(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_y.as_mut_ptr(),
|
||||
dst_stride_y as i32,
|
||||
dst_u.as_mut_ptr(),
|
||||
dst_stride_u as i32,
|
||||
dst_v.as_mut_ptr(),
|
||||
dst_stride_v as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn nv12_to_argb(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_uv: &[u8],
|
||||
src_stride_uv: u32,
|
||||
dst_argb: &mut [u8],
|
||||
dst_stride_argb: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
|
||||
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::nv12_to_argb(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_uv.as_ptr(),
|
||||
src_stride_uv as i32,
|
||||
dst_argb.as_mut_ptr(),
|
||||
dst_stride_argb as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn nv12_to_abgr(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_uv: &[u8],
|
||||
src_stride_uv: u32,
|
||||
dst_abgr: &mut [u8],
|
||||
dst_stride_abgr: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
|
||||
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::nv12_to_abgr(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_uv.as_ptr(),
|
||||
src_stride_uv as i32,
|
||||
dst_abgr.as_mut_ptr(),
|
||||
dst_stride_abgr as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i444_to_argb(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
dst_argb: &mut [u8],
|
||||
dst_stride_argb: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i444_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i444_to_argb(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_argb.as_mut_ptr(),
|
||||
dst_stride_argb as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i444_to_abgr(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
dst_abgr: &mut [u8],
|
||||
dst_stride_abgr: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i444_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i444_to_abgr(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_abgr.as_mut_ptr(),
|
||||
dst_stride_abgr as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i422_to_argb(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
dst_argb: &mut [u8],
|
||||
dst_stride_argb: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i422_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i422_to_argb(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_argb.as_mut_ptr(),
|
||||
dst_stride_argb as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i422_to_abgr(
|
||||
src_y: &[u8],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u8],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u8],
|
||||
src_stride_v: u32,
|
||||
dst_abgr: &mut [u8],
|
||||
dst_stride_abgr: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i422_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i422_to_abgr(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_abgr.as_mut_ptr(),
|
||||
dst_stride_abgr as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i010_to_argb(
|
||||
src_y: &[u16],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u16],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u16],
|
||||
src_stride_v: u32,
|
||||
dst_argb: &mut [u8],
|
||||
dst_stride_argb: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i010_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i010_to_argb(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_argb.as_mut_ptr(),
|
||||
dst_stride_argb as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn i010_to_abgr(
|
||||
src_y: &[u16],
|
||||
src_stride_y: u32,
|
||||
src_u: &[u16],
|
||||
src_stride_u: u32,
|
||||
src_v: &[u16],
|
||||
src_stride_v: u32,
|
||||
dst_abgr: &mut [u8],
|
||||
dst_stride_abgr: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
i010_assert_safety(
|
||||
src_y,
|
||||
src_stride_y,
|
||||
src_u,
|
||||
src_stride_u,
|
||||
src_v,
|
||||
src_stride_v,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::i010_to_abgr(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as i32,
|
||||
src_u.as_ptr(),
|
||||
src_stride_u as i32,
|
||||
src_v.as_ptr(),
|
||||
src_stride_v as i32,
|
||||
dst_abgr.as_mut_ptr(),
|
||||
dst_stride_abgr as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn abgr_to_nv12(
|
||||
src_abgr: &[u8],
|
||||
src_stride_abgr: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_uv: &mut [u8],
|
||||
dst_stride_uv: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
argb_assert_safety(src_abgr, src_stride_abgr, width, height);
|
||||
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::abgr_to_nv12(
|
||||
src_abgr.as_ptr(),
|
||||
src_stride_abgr as i32,
|
||||
dst_y.as_mut_ptr(),
|
||||
dst_stride_y as i32,
|
||||
dst_uv.as_mut_ptr(),
|
||||
dst_stride_uv as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn argb_to_nv12(
|
||||
src_argb: &[u8],
|
||||
src_stride_argb: u32,
|
||||
dst_y: &mut [u8],
|
||||
dst_stride_y: u32,
|
||||
dst_uv: &mut [u8],
|
||||
dst_stride_uv: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
argb_assert_safety(src_argb, src_stride_argb, width, height);
|
||||
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
|
||||
|
||||
unsafe {
|
||||
yuv_sys::ffi::argb_to_nv12(
|
||||
src_argb.as_ptr(),
|
||||
src_stride_argb as i32,
|
||||
dst_y.as_mut_ptr(),
|
||||
dst_stride_y as i32,
|
||||
dst_uv.as_mut_ptr(),
|
||||
dst_stride_uv as i32,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{
|
||||
data_channel::{DataChannel, DataChannelInit},
|
||||
ice_candidate::IceCandidate,
|
||||
imp::peer_connection as imp_pc,
|
||||
media_stream::MediaStream,
|
||||
media_stream_track::MediaStreamTrack,
|
||||
peer_connection_factory::RtcConfiguration,
|
||||
rtp_receiver::RtpReceiver,
|
||||
rtp_sender::RtpSender,
|
||||
rtp_transceiver::{RtpTransceiver, RtpTransceiverInit},
|
||||
session_description::SessionDescription,
|
||||
stats::RtcStats,
|
||||
MediaType, RtcError,
|
||||
};
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum PeerConnectionState {
|
||||
New,
|
||||
Connecting,
|
||||
Connected,
|
||||
Disconnected,
|
||||
Failed,
|
||||
Closed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum IceConnectionState {
|
||||
New,
|
||||
Checking,
|
||||
Connected,
|
||||
Completed,
|
||||
Failed,
|
||||
Disconnected,
|
||||
Closed,
|
||||
Max,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum IceGatheringState {
|
||||
New,
|
||||
Gathering,
|
||||
Complete,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum SignalingState {
|
||||
Stable,
|
||||
HaveLocalOffer,
|
||||
HaveLocalPrAnswer,
|
||||
HaveRemoteOffer,
|
||||
HaveRemotePrAnswer,
|
||||
Closed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct OfferOptions {
|
||||
pub ice_restart: bool,
|
||||
pub offer_to_receive_audio: bool,
|
||||
pub offer_to_receive_video: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AnswerOptions {}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IceCandidateError {
|
||||
pub address: String,
|
||||
pub port: i32,
|
||||
pub url: String,
|
||||
pub error_code: i32,
|
||||
pub error_text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TrackEvent {
|
||||
pub receiver: RtpReceiver,
|
||||
pub streams: Vec<MediaStream>,
|
||||
pub track: MediaStreamTrack,
|
||||
pub transceiver: RtpTransceiver,
|
||||
}
|
||||
|
||||
pub type OnConnectionChange = Box<dyn FnMut(PeerConnectionState) + Send + Sync>;
|
||||
pub type OnDataChannel = Box<dyn FnMut(DataChannel) + Send + Sync>;
|
||||
pub type OnIceCandidate = Box<dyn FnMut(IceCandidate) + Send + Sync>;
|
||||
pub type OnIceCandidateError = Box<dyn FnMut(IceCandidateError) + Send + Sync>;
|
||||
pub type OnIceConnectionChange = Box<dyn FnMut(IceConnectionState) + Send + Sync>;
|
||||
pub type OnIceGatheringChange = Box<dyn FnMut(IceGatheringState) + Send + Sync>;
|
||||
pub type OnNegotiationNeeded = Box<dyn FnMut(u32) + Send + Sync>;
|
||||
pub type OnSignalingChange = Box<dyn FnMut(SignalingState) + Send + Sync>;
|
||||
pub type OnTrack = Box<dyn FnMut(TrackEvent) + Send + Sync>;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PeerConnection {
|
||||
pub(crate) handle: imp_pc::PeerConnection,
|
||||
}
|
||||
|
||||
impl PeerConnection {
|
||||
pub fn set_configuration(&self, config: RtcConfiguration) -> Result<(), RtcError> {
|
||||
self.handle.set_configuration(config)
|
||||
}
|
||||
|
||||
pub async fn create_offer(
|
||||
&self,
|
||||
options: OfferOptions,
|
||||
) -> Result<SessionDescription, RtcError> {
|
||||
self.handle.create_offer(options).await
|
||||
}
|
||||
|
||||
pub async fn create_answer(
|
||||
&self,
|
||||
options: AnswerOptions,
|
||||
) -> Result<SessionDescription, RtcError> {
|
||||
self.handle.create_answer(options).await
|
||||
}
|
||||
|
||||
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
|
||||
self.handle.set_local_description(desc).await
|
||||
}
|
||||
|
||||
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
|
||||
self.handle.set_remote_description(desc).await
|
||||
}
|
||||
|
||||
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
|
||||
self.handle.add_ice_candidate(candidate).await
|
||||
}
|
||||
|
||||
pub fn create_data_channel(
|
||||
&self,
|
||||
label: &str,
|
||||
init: DataChannelInit,
|
||||
) -> Result<DataChannel, RtcError> {
|
||||
self.handle.create_data_channel(label, init)
|
||||
}
|
||||
|
||||
pub fn add_track<T: AsRef<str>>(
|
||||
&self,
|
||||
track: MediaStreamTrack,
|
||||
streams_ids: &[T],
|
||||
) -> Result<RtpSender, RtcError> {
|
||||
self.handle.add_track(track, streams_ids)
|
||||
}
|
||||
|
||||
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
|
||||
self.handle.remove_track(sender)
|
||||
}
|
||||
|
||||
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
|
||||
self.handle.get_stats().await
|
||||
}
|
||||
|
||||
pub fn add_transceiver(
|
||||
&self,
|
||||
track: MediaStreamTrack,
|
||||
init: RtpTransceiverInit,
|
||||
) -> Result<RtpTransceiver, RtcError> {
|
||||
self.handle.add_transceiver(track, init)
|
||||
}
|
||||
|
||||
pub fn add_transceiver_for_media(
|
||||
&self,
|
||||
media_type: MediaType,
|
||||
init: RtpTransceiverInit,
|
||||
) -> Result<RtpTransceiver, RtcError> {
|
||||
self.handle.add_transceiver_for_media(media_type, init)
|
||||
}
|
||||
|
||||
pub fn close(&self) {
|
||||
self.handle.close()
|
||||
}
|
||||
|
||||
pub fn restart_ice(&self) {
|
||||
self.handle.restart_ice()
|
||||
}
|
||||
|
||||
pub fn connection_state(&self) -> PeerConnectionState {
|
||||
self.handle.connection_state()
|
||||
}
|
||||
|
||||
pub fn ice_connection_state(&self) -> IceConnectionState {
|
||||
self.handle.ice_connection_state()
|
||||
}
|
||||
|
||||
pub fn ice_gathering_state(&self) -> IceGatheringState {
|
||||
self.handle.ice_gathering_state()
|
||||
}
|
||||
|
||||
pub fn signaling_state(&self) -> SignalingState {
|
||||
self.handle.signaling_state()
|
||||
}
|
||||
|
||||
pub fn current_local_description(&self) -> Option<SessionDescription> {
|
||||
self.handle.current_local_description()
|
||||
}
|
||||
|
||||
pub fn current_remote_description(&self) -> Option<SessionDescription> {
|
||||
self.handle.current_remote_description()
|
||||
}
|
||||
|
||||
pub fn senders(&self) -> Vec<RtpSender> {
|
||||
self.handle.senders()
|
||||
}
|
||||
|
||||
pub fn receivers(&self) -> Vec<RtpReceiver> {
|
||||
self.handle.receivers()
|
||||
}
|
||||
|
||||
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
|
||||
self.handle.transceivers()
|
||||
}
|
||||
|
||||
pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
|
||||
self.handle.on_connection_state_change(f)
|
||||
}
|
||||
|
||||
pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
|
||||
self.handle.on_data_channel(f)
|
||||
}
|
||||
|
||||
pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
|
||||
self.handle.on_ice_candidate(f)
|
||||
}
|
||||
|
||||
pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
|
||||
self.handle.on_ice_candidate_error(f)
|
||||
}
|
||||
|
||||
pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
|
||||
self.handle.on_ice_connection_state_change(f)
|
||||
}
|
||||
|
||||
pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
|
||||
self.handle.on_ice_gathering_state_change(f)
|
||||
}
|
||||
|
||||
pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
|
||||
self.handle.on_negotiation_needed(f)
|
||||
}
|
||||
|
||||
pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
|
||||
self.handle.on_signaling_state_change(f)
|
||||
}
|
||||
|
||||
pub fn on_track(&self, f: Option<OnTrack>) {
|
||||
self.handle.on_track(f)
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for PeerConnection {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("PeerConnection")
|
||||
.field("state", &self.connection_state())
|
||||
.field("ice_state", &self.ice_connection_state())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use log::trace;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::{peer_connection::*, peer_connection_factory::*};
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_pc() {
|
||||
let _ = env_logger::builder().is_test(true).try_init();
|
||||
|
||||
let factory = PeerConnectionFactory::default();
|
||||
let config = RtcConfiguration {
|
||||
ice_servers: vec![IceServer {
|
||||
urls: vec!["stun:stun1.l.google.com:19302".to_string()],
|
||||
username: "".into(),
|
||||
password: "".into(),
|
||||
}],
|
||||
continual_gathering_policy: ContinualGatheringPolicy::GatherOnce,
|
||||
ice_transport_type: IceTransportsType::All,
|
||||
};
|
||||
|
||||
let bob = factory.create_peer_connection(config.clone()).unwrap();
|
||||
let alice = factory.create_peer_connection(config.clone()).unwrap();
|
||||
|
||||
let (bob_ice_tx, mut bob_ice_rx) = mpsc::unbounded_channel::<IceCandidate>();
|
||||
let (alice_ice_tx, mut alice_ice_rx) = mpsc::unbounded_channel::<IceCandidate>();
|
||||
let (alice_dc_tx, mut alice_dc_rx) = mpsc::unbounded_channel::<DataChannel>();
|
||||
|
||||
bob.on_ice_candidate(Some(Box::new(move |candidate| {
|
||||
bob_ice_tx.send(candidate).unwrap();
|
||||
})));
|
||||
|
||||
alice.on_ice_candidate(Some(Box::new(move |candidate| {
|
||||
alice_ice_tx.send(candidate).unwrap();
|
||||
})));
|
||||
|
||||
alice.on_data_channel(Some(Box::new(move |dc| {
|
||||
alice_dc_tx.send(dc).unwrap();
|
||||
})));
|
||||
|
||||
let bob_dc = bob.create_data_channel("test_dc", DataChannelInit::default()).unwrap();
|
||||
|
||||
let offer = bob.create_offer(OfferOptions::default()).await.unwrap();
|
||||
trace!("Bob offer: {:?}", offer);
|
||||
bob.set_local_description(offer.clone()).await.unwrap();
|
||||
alice.set_remote_description(offer).await.unwrap();
|
||||
|
||||
let answer = alice.create_answer(AnswerOptions::default()).await.unwrap();
|
||||
trace!("Alice answer: {:?}", answer);
|
||||
alice.set_local_description(answer.clone()).await.unwrap();
|
||||
bob.set_remote_description(answer).await.unwrap();
|
||||
|
||||
let bob_ice = bob_ice_rx.recv().await.unwrap();
|
||||
let alice_ice = alice_ice_rx.recv().await.unwrap();
|
||||
|
||||
bob.add_ice_candidate(alice_ice).await.unwrap();
|
||||
alice.add_ice_candidate(bob_ice).await.unwrap();
|
||||
|
||||
let (data_tx, mut data_rx) = mpsc::unbounded_channel::<String>();
|
||||
let alice_dc = alice_dc_rx.recv().await.unwrap();
|
||||
alice_dc.on_message(Some(Box::new(move |buffer| {
|
||||
data_tx.send(String::from_utf8_lossy(buffer.data).to_string()).unwrap();
|
||||
})));
|
||||
|
||||
bob_dc.send(b"This is a test", true).unwrap();
|
||||
assert_eq!(data_rx.recv().await.unwrap(), "This is a test");
|
||||
|
||||
alice.close();
|
||||
bob.close();
|
||||
}
|
||||
}
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{
|
||||
imp::peer_connection_factory as imp_pcf, peer_connection::PeerConnection,
|
||||
rtp_parameters::RtpCapabilities, MediaType, RtcError,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IceServer {
|
||||
pub urls: Vec<String>,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum ContinualGatheringPolicy {
|
||||
GatherOnce,
|
||||
GatherContinually,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum IceTransportsType {
|
||||
Relay,
|
||||
NoHost,
|
||||
All,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RtcConfiguration {
|
||||
pub ice_servers: Vec<IceServer>,
|
||||
pub continual_gathering_policy: ContinualGatheringPolicy,
|
||||
pub ice_transport_type: IceTransportsType,
|
||||
}
|
||||
|
||||
impl Default for RtcConfiguration {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
ice_servers: vec![],
|
||||
continual_gathering_policy: ContinualGatheringPolicy::GatherContinually,
|
||||
ice_transport_type: IceTransportsType::All,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct PeerConnectionFactory {
|
||||
pub(crate) handle: imp_pcf::PeerConnectionFactory,
|
||||
}
|
||||
|
||||
impl Debug for PeerConnectionFactory {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
f.debug_struct("PeerConnectionFactory").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl PeerConnectionFactory {
|
||||
pub fn create_peer_connection(
|
||||
&self,
|
||||
config: RtcConfiguration,
|
||||
) -> Result<PeerConnection, RtcError> {
|
||||
self.handle.create_peer_connection(config)
|
||||
}
|
||||
|
||||
pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
|
||||
self.handle.get_rtp_sender_capabilities(media_type)
|
||||
}
|
||||
|
||||
pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
|
||||
self.handle.get_rtp_receiver_capabilities(media_type)
|
||||
}
|
||||
}
|
||||
|
||||
pub mod native {
|
||||
use super::PeerConnectionFactory;
|
||||
use crate::{
|
||||
audio_source::native::NativeAudioSource, audio_track::RtcAudioTrack,
|
||||
video_source::native::NativeVideoSource, video_track::RtcVideoTrack,
|
||||
};
|
||||
|
||||
pub trait PeerConnectionFactoryExt {
|
||||
fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack;
|
||||
fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack;
|
||||
|
||||
/// Create an audio track that uses the Platform ADM for capture.
|
||||
/// The track will capture audio from the selected recording device.
|
||||
fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack;
|
||||
|
||||
// Device enumeration
|
||||
fn playout_devices(&self) -> i16;
|
||||
fn recording_devices(&self) -> i16;
|
||||
fn playout_device_name(&self, index: u16) -> String;
|
||||
fn recording_device_name(&self, index: u16) -> String;
|
||||
/// Get device GUID (platform-specific unique identifier, stable across hot-plug)
|
||||
fn playout_device_guid(&self, index: u16) -> String;
|
||||
fn recording_device_guid(&self, index: u16) -> String;
|
||||
|
||||
// Device selection by index
|
||||
fn set_playout_device(&self, index: u16) -> bool;
|
||||
fn set_recording_device(&self, index: u16) -> bool;
|
||||
/// Device selection by GUID (preferred - stable across device changes)
|
||||
fn set_playout_device_by_guid(&self, guid: &str) -> bool;
|
||||
fn set_recording_device_by_guid(&self, guid: &str) -> bool;
|
||||
|
||||
// Recording control (for device switching while active)
|
||||
fn stop_recording(&self) -> bool;
|
||||
fn init_recording(&self) -> bool;
|
||||
fn start_recording(&self) -> bool;
|
||||
fn recording_is_initialized(&self) -> bool;
|
||||
|
||||
// Playout control (for device switching while active)
|
||||
fn stop_playout(&self) -> bool;
|
||||
fn init_playout(&self) -> bool;
|
||||
fn start_playout(&self) -> bool;
|
||||
fn playout_is_initialized(&self) -> bool;
|
||||
|
||||
// Built-in audio processing (hardware AEC/AGC/NS)
|
||||
// Only available on iOS and some Android devices
|
||||
fn builtin_aec_is_available(&self) -> bool;
|
||||
fn builtin_agc_is_available(&self) -> bool;
|
||||
fn builtin_ns_is_available(&self) -> bool;
|
||||
fn enable_builtin_aec(&self, enable: bool) -> bool;
|
||||
fn enable_builtin_agc(&self, enable: bool) -> bool;
|
||||
fn enable_builtin_ns(&self, enable: bool) -> bool;
|
||||
|
||||
// ADM recording control
|
||||
// Use this to disable microphone when only using NativeAudioSource
|
||||
fn set_adm_recording_enabled(&self, enabled: bool);
|
||||
fn adm_recording_enabled(&self) -> bool;
|
||||
|
||||
// ADM playout control
|
||||
// When disabled (default), playout uses synthetic mode - remote audio is
|
||||
// delivered via FFI callbacks. When enabled, plays through platform speakers.
|
||||
fn set_adm_playout_enabled(&self, enabled: bool);
|
||||
fn adm_playout_enabled(&self) -> bool;
|
||||
|
||||
// Platform ADM lifecycle management
|
||||
// Call acquire_platform_adm when creating PlatformAudio.
|
||||
// Call release_platform_adm when disposing PlatformAudio.
|
||||
// The Platform ADM is only created when first acquired, and terminated
|
||||
// when the last reference is released.
|
||||
fn acquire_platform_adm(&self) -> bool;
|
||||
fn release_platform_adm(&self);
|
||||
fn platform_adm_ref_count(&self) -> i32;
|
||||
fn is_platform_adm_active(&self) -> bool;
|
||||
|
||||
// Ensures the Platform ADM exists, retrying creation if an earlier
|
||||
// attempt failed (e.g. the OS audio stack was still starting up).
|
||||
fn ensure_platform_adm(&self) -> bool;
|
||||
// Distinguishes "audio stack unavailable" from "zero audio devices".
|
||||
fn platform_adm_available(&self) -> bool;
|
||||
}
|
||||
|
||||
impl PeerConnectionFactoryExt for PeerConnectionFactory {
|
||||
fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
|
||||
self.handle.create_video_track(label, source)
|
||||
}
|
||||
|
||||
fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
|
||||
self.handle.create_audio_track(label, source)
|
||||
}
|
||||
|
||||
fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack {
|
||||
self.handle.create_device_audio_track(label)
|
||||
}
|
||||
|
||||
fn playout_devices(&self) -> i16 {
|
||||
self.handle.playout_devices()
|
||||
}
|
||||
|
||||
fn recording_devices(&self) -> i16 {
|
||||
self.handle.recording_devices()
|
||||
}
|
||||
|
||||
fn playout_device_name(&self, index: u16) -> String {
|
||||
self.handle.playout_device_name(index)
|
||||
}
|
||||
|
||||
fn recording_device_name(&self, index: u16) -> String {
|
||||
self.handle.recording_device_name(index)
|
||||
}
|
||||
|
||||
fn playout_device_guid(&self, index: u16) -> String {
|
||||
self.handle.playout_device_guid(index)
|
||||
}
|
||||
|
||||
fn recording_device_guid(&self, index: u16) -> String {
|
||||
self.handle.recording_device_guid(index)
|
||||
}
|
||||
|
||||
fn set_playout_device(&self, index: u16) -> bool {
|
||||
self.handle.set_playout_device(index)
|
||||
}
|
||||
|
||||
fn set_recording_device(&self, index: u16) -> bool {
|
||||
self.handle.set_recording_device(index)
|
||||
}
|
||||
|
||||
fn set_playout_device_by_guid(&self, guid: &str) -> bool {
|
||||
self.handle.set_playout_device_by_guid(guid)
|
||||
}
|
||||
|
||||
fn set_recording_device_by_guid(&self, guid: &str) -> bool {
|
||||
self.handle.set_recording_device_by_guid(guid)
|
||||
}
|
||||
|
||||
fn stop_recording(&self) -> bool {
|
||||
self.handle.stop_recording()
|
||||
}
|
||||
|
||||
fn init_recording(&self) -> bool {
|
||||
self.handle.init_recording()
|
||||
}
|
||||
|
||||
fn start_recording(&self) -> bool {
|
||||
self.handle.start_recording()
|
||||
}
|
||||
|
||||
fn recording_is_initialized(&self) -> bool {
|
||||
self.handle.recording_is_initialized()
|
||||
}
|
||||
|
||||
fn stop_playout(&self) -> bool {
|
||||
self.handle.stop_playout()
|
||||
}
|
||||
|
||||
fn init_playout(&self) -> bool {
|
||||
self.handle.init_playout()
|
||||
}
|
||||
|
||||
fn start_playout(&self) -> bool {
|
||||
self.handle.start_playout()
|
||||
}
|
||||
|
||||
fn playout_is_initialized(&self) -> bool {
|
||||
self.handle.playout_is_initialized()
|
||||
}
|
||||
|
||||
fn builtin_aec_is_available(&self) -> bool {
|
||||
self.handle.builtin_aec_is_available()
|
||||
}
|
||||
|
||||
fn builtin_agc_is_available(&self) -> bool {
|
||||
self.handle.builtin_agc_is_available()
|
||||
}
|
||||
|
||||
fn builtin_ns_is_available(&self) -> bool {
|
||||
self.handle.builtin_ns_is_available()
|
||||
}
|
||||
|
||||
fn enable_builtin_aec(&self, enable: bool) -> bool {
|
||||
self.handle.enable_builtin_aec(enable)
|
||||
}
|
||||
|
||||
fn enable_builtin_agc(&self, enable: bool) -> bool {
|
||||
self.handle.enable_builtin_agc(enable)
|
||||
}
|
||||
|
||||
fn enable_builtin_ns(&self, enable: bool) -> bool {
|
||||
self.handle.enable_builtin_ns(enable)
|
||||
}
|
||||
|
||||
fn set_adm_recording_enabled(&self, enabled: bool) {
|
||||
self.handle.set_adm_recording_enabled(enabled)
|
||||
}
|
||||
|
||||
fn adm_recording_enabled(&self) -> bool {
|
||||
self.handle.adm_recording_enabled()
|
||||
}
|
||||
|
||||
fn set_adm_playout_enabled(&self, enabled: bool) {
|
||||
self.handle.set_adm_playout_enabled(enabled)
|
||||
}
|
||||
|
||||
fn adm_playout_enabled(&self) -> bool {
|
||||
self.handle.adm_playout_enabled()
|
||||
}
|
||||
|
||||
fn acquire_platform_adm(&self) -> bool {
|
||||
self.handle.acquire_platform_adm()
|
||||
}
|
||||
|
||||
fn release_platform_adm(&self) {
|
||||
self.handle.release_platform_adm()
|
||||
}
|
||||
|
||||
fn platform_adm_ref_count(&self) -> i32 {
|
||||
self.handle.platform_adm_ref_count()
|
||||
}
|
||||
|
||||
fn is_platform_adm_active(&self) -> bool {
|
||||
self.handle.is_platform_adm_active()
|
||||
}
|
||||
|
||||
fn ensure_platform_adm(&self) -> bool {
|
||||
self.handle.ensure_platform_adm()
|
||||
}
|
||||
|
||||
fn platform_adm_available(&self) -> bool {
|
||||
self.handle.platform_adm_available()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub use crate::{
|
||||
audio_frame::AudioFrame,
|
||||
audio_source::{AudioSourceOptions, RtcAudioSource},
|
||||
audio_track::RtcAudioTrack,
|
||||
data_channel::{DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataChannelState},
|
||||
ice_candidate::IceCandidate,
|
||||
media_stream::MediaStream,
|
||||
media_stream_track::{MediaStreamTrack, RtcTrackState},
|
||||
peer_connection::{
|
||||
AnswerOptions, IceConnectionState, IceGatheringState, OfferOptions, PeerConnection,
|
||||
PeerConnectionState, SignalingState,
|
||||
},
|
||||
peer_connection_factory::{
|
||||
ContinualGatheringPolicy, IceServer, IceTransportsType, PeerConnectionFactory,
|
||||
RtcConfiguration,
|
||||
},
|
||||
rtp_parameters::*,
|
||||
rtp_receiver::RtpReceiver,
|
||||
rtp_sender::RtpSender,
|
||||
rtp_transceiver::{RtpTransceiver, RtpTransceiverDirection, RtpTransceiverInit},
|
||||
session_description::{SdpType, SessionDescription},
|
||||
video_frame::{
|
||||
BoxVideoBuffer, BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer,
|
||||
NV12Buffer, VideoBuffer, VideoBufferType, VideoFormatType, VideoFrame, VideoRotation,
|
||||
},
|
||||
video_source::{RtcVideoSource, VideoResolution},
|
||||
video_track::RtcVideoTrack,
|
||||
MediaType, RtcError, RtcErrorType,
|
||||
};
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2026 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod native {
|
||||
use webrtc_sys::recorded_audio_tap::ffi as sys;
|
||||
use webrtc_sys::recorded_audio_tap::RecordedAudioSinkWrapper;
|
||||
|
||||
/// Handle to an installed recorded-audio sink. Dropping it does not clear
|
||||
/// the sink; call [`clear_recorded_audio_sink`] with this generation.
|
||||
pub type RecordedAudioSinkGeneration = u64;
|
||||
|
||||
/// Installs a process-global tap on platform-ADM recorded microphone audio.
|
||||
///
|
||||
/// `callback` is invoked on the ADM capture thread with one 48kHz mono
|
||||
/// 10ms frame (480 samples) per call: `(samples, sample_rate_hz,
|
||||
/// num_channels, samples_per_channel)`. It must be wait-free: do no
|
||||
/// allocation or blocking work, only hand the frame to a bounded queue.
|
||||
/// Returns a generation token to pass to [`clear_recorded_audio_sink`].
|
||||
pub fn set_recorded_audio_sink<F>(callback: F) -> RecordedAudioSinkGeneration
|
||||
where
|
||||
F: Fn(&[i16], i32, usize, usize) + Send + Sync + 'static,
|
||||
{
|
||||
sys::set_recorded_audio_sink(Box::new(RecordedAudioSinkWrapper::new(Box::new(callback))))
|
||||
}
|
||||
|
||||
/// Removes the recorded-audio sink, but only if `generation` is still the
|
||||
/// installed one. A stale token is a no-op, so a late teardown cannot
|
||||
/// clobber a sink a newer caller installed.
|
||||
pub fn clear_recorded_audio_sink(generation: RecordedAudioSinkGeneration) {
|
||||
sys::clear_recorded_audio_sink(generation);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::rtp_transceiver::RtpTransceiverDirection;
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum Priority {
|
||||
VeryLow,
|
||||
Low,
|
||||
Medium,
|
||||
High,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RtpHeaderExtensionParameters {
|
||||
pub uri: String,
|
||||
pub id: i32,
|
||||
pub encrypted: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct RtpParameters {
|
||||
pub codecs: Vec<RtpCodecParameters>,
|
||||
pub header_extensions: Vec<RtpHeaderExtensionParameters>,
|
||||
pub rtcp: RtcpParameters,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct RtpCodecParameters {
|
||||
pub payload_type: u8,
|
||||
pub mime_type: String, // read-only
|
||||
pub clock_rate: Option<u64>,
|
||||
pub channels: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct RtcpParameters {
|
||||
pub cname: String,
|
||||
pub reduced_size: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RtpEncodingParameters {
|
||||
pub active: bool,
|
||||
pub max_bitrate: Option<u64>,
|
||||
pub max_framerate: Option<f64>,
|
||||
pub priority: Priority,
|
||||
pub rid: String,
|
||||
pub scale_resolution_down_by: Option<f64>,
|
||||
/// RTP scalability mode (e.g. "L3T3_KEY"). Required to enable true
|
||||
/// SVC for codecs that support it (VP9, AV1).
|
||||
pub scalability_mode: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RtpCodecCapability {
|
||||
pub channels: Option<u16>,
|
||||
pub clock_rate: Option<u64>,
|
||||
pub mime_type: String,
|
||||
pub sdp_fmtp_line: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RtpHeaderExtensionCapability {
|
||||
pub uri: String,
|
||||
pub direction: RtpTransceiverDirection,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RtpCapabilities {
|
||||
pub codecs: Vec<RtpCodecCapability>,
|
||||
pub header_extensions: Vec<RtpHeaderExtensionCapability>,
|
||||
}
|
||||
|
||||
impl Default for RtpEncodingParameters {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
active: true,
|
||||
max_bitrate: None,
|
||||
max_framerate: None,
|
||||
priority: Priority::Low,
|
||||
rid: String::default(),
|
||||
scale_resolution_down_by: None,
|
||||
scalability_mode: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{
|
||||
imp::rtp_receiver as imp_rr, media_stream_track::MediaStreamTrack,
|
||||
rtp_parameters::RtpParameters, stats::RtcStats, RtcError,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtpReceiver {
|
||||
pub(crate) handle: imp_rr::RtpReceiver,
|
||||
}
|
||||
|
||||
impl RtpReceiver {
|
||||
pub fn track(&self) -> Option<MediaStreamTrack> {
|
||||
self.handle.track()
|
||||
}
|
||||
|
||||
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
|
||||
self.handle.get_stats().await
|
||||
}
|
||||
|
||||
pub fn parameters(&self) -> RtpParameters {
|
||||
self.handle.parameters()
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for RtpReceiver {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RtpReceiver")
|
||||
.field("track", &self.track())
|
||||
.field("cname", &self.parameters().rtcp.cname)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{
|
||||
imp::rtp_sender as imp_rs, media_stream_track::MediaStreamTrack, rtp_parameters::RtpParameters,
|
||||
stats::RtcStats, RtcError,
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtpSender {
|
||||
pub(crate) handle: imp_rs::RtpSender,
|
||||
}
|
||||
|
||||
impl RtpSender {
|
||||
pub fn track(&self) -> Option<MediaStreamTrack> {
|
||||
self.handle.track()
|
||||
}
|
||||
|
||||
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
|
||||
self.handle.get_stats().await
|
||||
}
|
||||
|
||||
pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
|
||||
self.handle.set_track(track)
|
||||
}
|
||||
|
||||
pub fn parameters(&self) -> RtpParameters {
|
||||
self.handle.parameters()
|
||||
}
|
||||
|
||||
pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
|
||||
self.handle.set_parameters(parameters)
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for RtpSender {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RtpReceiver").field("cname", &self.parameters().rtcp.cname).finish()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{
|
||||
imp::rtp_transceiver as imp_rt,
|
||||
rtp_parameters::{RtpCodecCapability, RtpEncodingParameters},
|
||||
rtp_receiver::RtpReceiver,
|
||||
rtp_sender::RtpSender,
|
||||
RtcError,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RtpTransceiverInit {
|
||||
pub direction: RtpTransceiverDirection,
|
||||
pub stream_ids: Vec<String>,
|
||||
pub send_encodings: Vec<RtpEncodingParameters>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum RtpTransceiverDirection {
|
||||
SendRecv,
|
||||
SendOnly,
|
||||
RecvOnly,
|
||||
Inactive,
|
||||
Stopped,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtpTransceiver {
|
||||
pub(crate) handle: imp_rt::RtpTransceiver,
|
||||
}
|
||||
|
||||
impl RtpTransceiver {
|
||||
pub fn mid(&self) -> Option<String> {
|
||||
self.handle.mid()
|
||||
}
|
||||
|
||||
pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
|
||||
self.handle.current_direction()
|
||||
}
|
||||
|
||||
pub fn direction(&self) -> RtpTransceiverDirection {
|
||||
self.handle.direction()
|
||||
}
|
||||
|
||||
pub fn sender(&self) -> RtpSender {
|
||||
self.handle.sender()
|
||||
}
|
||||
|
||||
pub fn receiver(&self) -> RtpReceiver {
|
||||
self.handle.receiver()
|
||||
}
|
||||
|
||||
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
|
||||
self.handle.set_codec_preferences(codecs)
|
||||
}
|
||||
|
||||
pub fn stop(&self) -> Result<(), RtcError> {
|
||||
self.handle.stop()
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for RtpTransceiver {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RtpTransceiver")
|
||||
.field("mid", &self.mid())
|
||||
.field("direction", &self.direction())
|
||||
.field("sender", &self.sender())
|
||||
.field("receiver", &self.receiver())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
fmt::{Debug, Display},
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::imp::session_description as sd_imp;
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum SdpType {
|
||||
Offer,
|
||||
PrAnswer,
|
||||
Answer,
|
||||
Rollback,
|
||||
}
|
||||
|
||||
impl FromStr for SdpType {
|
||||
type Err = &'static str;
|
||||
|
||||
fn from_str(sdp_type: &str) -> Result<Self, Self::Err> {
|
||||
match sdp_type {
|
||||
"offer" => Ok(Self::Offer),
|
||||
"pranswer" => Ok(Self::PrAnswer),
|
||||
"answer" => Ok(Self::Answer),
|
||||
"rollback" => Ok(Self::Rollback),
|
||||
_ => Err("invalid SdpType"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for SdpType {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let s = match self {
|
||||
SdpType::Offer => "offer",
|
||||
SdpType::PrAnswer => "pranswer",
|
||||
SdpType::Answer => "answer",
|
||||
SdpType::Rollback => "rollback",
|
||||
};
|
||||
write!(f, "{}", s)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SessionDescription {
|
||||
pub(crate) handle: sd_imp::SessionDescription,
|
||||
}
|
||||
|
||||
#[derive(Clone, Error, Debug)]
|
||||
#[error("Failed to parse sdp: {line} - {description}")]
|
||||
pub struct SdpParseError {
|
||||
pub line: String,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
impl SessionDescription {
|
||||
pub fn parse(sdp: &str, sdp_type: SdpType) -> Result<Self, SdpParseError> {
|
||||
sd_imp::SessionDescription::parse(sdp, sdp_type)
|
||||
}
|
||||
|
||||
pub fn sdp_type(&self) -> SdpType {
|
||||
self.handle.sdp_type()
|
||||
}
|
||||
}
|
||||
|
||||
impl ToString for SessionDescription {
|
||||
fn to_string(&self) -> String {
|
||||
self.handle.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for SessionDescription {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("SessionDescription").field("sdp_type", &self.sdp_type()).finish()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,624 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::data_channel::DataChannelState;
|
||||
|
||||
/// Values from https://www.w3.org/TR/webrtc-stats/ (NOTE: Some of the structs are not in the SPEC
|
||||
/// but inside libwebrtc)
|
||||
/// serde will handle the magic of correctly deserializing the json into our structs.
|
||||
/// The enums values are inside encapsulated inside option because we're not sure about their
|
||||
/// default values (So we default to None instead of an arbitrary value)
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum RtcStats {
|
||||
Codec(CodecStats),
|
||||
InboundRtp(InboundRtpStats),
|
||||
OutboundRtp(OutboundRtpStats),
|
||||
RemoteInboundRtp(RemoteInboundRtpStats),
|
||||
RemoteOutboundRtp(RemoteOutboundRtpStats),
|
||||
MediaSource(MediaSourceStats),
|
||||
MediaPlayout(MediaPlayoutStats),
|
||||
PeerConnection(PeerConnectionStats),
|
||||
DataChannel(DataChannelStats),
|
||||
Transport(TransportStats),
|
||||
CandidatePair(CandidatePairStats),
|
||||
LocalCandidate(LocalCandidateStats),
|
||||
RemoteCandidate(RemoteCandidateStats),
|
||||
Certificate(CertificateStats),
|
||||
Stream(StreamStats),
|
||||
Track, // Deprecated
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum QualityLimitationReason {
|
||||
#[default]
|
||||
None,
|
||||
Cpu,
|
||||
Bandwidth,
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum IceRole {
|
||||
#[default]
|
||||
Unknown,
|
||||
Controlling,
|
||||
Controlled,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DtlsTransportState {
|
||||
New,
|
||||
Connecting,
|
||||
Connected,
|
||||
Closed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum IceTransportState {
|
||||
New,
|
||||
Checking,
|
||||
Connected,
|
||||
Completed,
|
||||
Disconnected,
|
||||
Failed,
|
||||
Closed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DtlsRole {
|
||||
Client,
|
||||
Server,
|
||||
#[default]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum IceCandidatePairState {
|
||||
Frozen,
|
||||
Waiting,
|
||||
InProgress, // in-progress
|
||||
Failed,
|
||||
Succeeded,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum IceCandidateType {
|
||||
Host,
|
||||
Srflx,
|
||||
Prflx,
|
||||
Relay,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum IceServerTransportProtocol {
|
||||
Udp,
|
||||
Tcp,
|
||||
Tls,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum IceTcpCandidateType {
|
||||
Active,
|
||||
Passive,
|
||||
So,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct CodecStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub codec: dictionaries::CodecStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct InboundRtpStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub stream: dictionaries::RtpStreamStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub received: dictionaries::ReceivedRtpStreamStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub inbound: dictionaries::InboundRtpStreamStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct OutboundRtpStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub stream: dictionaries::RtpStreamStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub sent: dictionaries::SentRtpStreamStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub outbound: dictionaries::OutboundRtpStreamStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct RemoteInboundRtpStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub stream: dictionaries::RtpStreamStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub received: dictionaries::ReceivedRtpStreamStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub remote_inbound: dictionaries::RemoteInboundRtpStreamStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct RemoteOutboundRtpStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub stream: dictionaries::RtpStreamStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub sent: dictionaries::SentRtpStreamStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub remote_outbound: dictionaries::RemoteOutboundRtpStreamStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct MediaSourceStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub source: dictionaries::MediaSourceStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub audio: dictionaries::AudioSourceStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub video: dictionaries::VideoSourceStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct MediaPlayoutStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub audio_playout: dictionaries::AudioPlayoutStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct PeerConnectionStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub pc: dictionaries::PeerConnectionStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct DataChannelStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub dc: dictionaries::DataChannelStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct TransportStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub transport: dictionaries::TransportStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct CandidatePairStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub candidate_pair: dictionaries::CandidatePairStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct LocalCandidateStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub local_candidate: dictionaries::IceCandidateStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct RemoteCandidateStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub remote_candidate: dictionaries::IceCandidateStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct CertificateStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub certificate: dictionaries::CertificateStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct StreamStats {
|
||||
#[serde(flatten)]
|
||||
pub rtc: dictionaries::RtcStats,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub stream: dictionaries::StreamStats,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
pub struct TrackStats {}
|
||||
|
||||
pub mod dictionaries {
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct RtcStats {
|
||||
pub id: String,
|
||||
pub timestamp: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct CodecStats {
|
||||
pub payload_type: u32,
|
||||
pub transport_id: String,
|
||||
pub mime_type: String,
|
||||
pub clock_rate: u32,
|
||||
pub channels: u32,
|
||||
pub sdp_fmtp_line: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct RtpStreamStats {
|
||||
pub ssrc: u32,
|
||||
pub kind: String,
|
||||
pub transport_id: String,
|
||||
pub codec_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct ReceivedRtpStreamStats {
|
||||
pub packets_received: u64,
|
||||
pub packets_lost: i64,
|
||||
pub jitter: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct InboundRtpStreamStats {
|
||||
pub track_identifier: String,
|
||||
pub mid: String,
|
||||
pub remote_id: String,
|
||||
pub frames_decoded: u32,
|
||||
pub key_frames_decoded: u32,
|
||||
pub frames_rendered: u32,
|
||||
pub frames_dropped: u32,
|
||||
pub frame_width: u32,
|
||||
pub frame_height: u32,
|
||||
pub frames_per_second: f64,
|
||||
pub qp_sum: u64,
|
||||
pub total_decode_time: f64,
|
||||
pub total_inter_frame_delay: f64,
|
||||
pub total_squared_inter_frame_delay: f64,
|
||||
pub pause_count: u32,
|
||||
pub total_pause_duration: f64,
|
||||
pub freeze_count: u32,
|
||||
pub total_freeze_duration: f64,
|
||||
pub last_packet_received_timestamp: f64,
|
||||
pub header_bytes_received: u64,
|
||||
pub packets_discarded: u64,
|
||||
pub fec_bytes_received: u64,
|
||||
pub fec_packets_received: u64,
|
||||
pub fec_packets_discarded: u64,
|
||||
pub bytes_received: u64,
|
||||
pub nack_count: u32,
|
||||
pub fir_count: u32,
|
||||
pub pli_count: u32,
|
||||
pub total_processing_delay: f64,
|
||||
pub estimated_playout_timestamp: f64,
|
||||
pub jitter_buffer_delay: f64,
|
||||
pub jitter_buffer_target_delay: f64,
|
||||
pub jitter_buffer_emitted_count: u64,
|
||||
pub jitter_buffer_minimum_delay: f64,
|
||||
pub total_samples_received: u64,
|
||||
pub concealed_samples: u64,
|
||||
pub silent_concealed_samples: u64,
|
||||
pub concealment_events: u64,
|
||||
pub inserted_samples_for_deceleration: u64,
|
||||
pub removed_samples_for_acceleration: u64,
|
||||
pub audio_level: f64,
|
||||
pub total_audio_energy: f64,
|
||||
pub total_samples_duration: f64,
|
||||
pub frames_received: u64,
|
||||
pub decoder_implementation: String,
|
||||
pub playout_id: String,
|
||||
pub power_efficient_decoder: bool,
|
||||
pub frames_assembled_from_multiple_packets: u64,
|
||||
pub total_assembly_time: f64,
|
||||
pub retransmitted_packets_received: u64,
|
||||
pub retransmitted_bytes_received: u64,
|
||||
pub rtx_ssrc: u32,
|
||||
pub fec_ssrc: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct SentRtpStreamStats {
|
||||
pub packets_sent: u64,
|
||||
pub bytes_sent: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct OutboundRtpStreamStats {
|
||||
pub mid: String,
|
||||
pub media_source_id: String,
|
||||
pub remote_id: String,
|
||||
pub rid: String,
|
||||
pub header_bytes_sent: u64,
|
||||
pub retransmitted_packets_sent: u64,
|
||||
pub retransmitted_bytes_sent: u64,
|
||||
pub rtx_ssrc: u32,
|
||||
pub target_bitrate: f64,
|
||||
pub total_encoded_bytes_target: u64,
|
||||
pub frame_width: u32,
|
||||
pub frame_height: u32,
|
||||
pub frames_per_second: f64,
|
||||
pub frames_sent: u32,
|
||||
pub huge_frames_sent: u32,
|
||||
pub frames_encoded: u32,
|
||||
pub key_frames_encoded: u32,
|
||||
pub qp_sum: u64,
|
||||
pub total_encode_time: f64,
|
||||
pub total_packet_send_delay: f64,
|
||||
pub quality_limitation_reason: QualityLimitationReason,
|
||||
pub quality_limitation_durations: HashMap<String, f64>,
|
||||
pub quality_limitation_resolution_changes: u32,
|
||||
pub nack_count: u32,
|
||||
pub fir_count: u32,
|
||||
pub pli_count: u32,
|
||||
pub encoder_implementation: String,
|
||||
pub power_efficient_encoder: bool,
|
||||
pub active: bool,
|
||||
pub scalibility_mode: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct RemoteInboundRtpStreamStats {
|
||||
pub local_id: String,
|
||||
pub round_trip_time: f64,
|
||||
pub total_round_trip_time: f64,
|
||||
pub fraction_lost: f64,
|
||||
pub round_trip_time_measurements: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct RemoteOutboundRtpStreamStats {
|
||||
pub local_id: String,
|
||||
pub remote_timestamp: f64,
|
||||
pub reports_sent: u64,
|
||||
pub round_trip_time: f64,
|
||||
pub total_round_trip_time: f64,
|
||||
pub round_trip_time_measurements: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct MediaSourceStats {
|
||||
pub track_identifier: String,
|
||||
pub kind: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct AudioSourceStats {
|
||||
pub audio_level: f64,
|
||||
pub total_audio_energy: f64,
|
||||
pub total_samples_duration: f64,
|
||||
pub echo_return_loss: f64,
|
||||
pub echo_return_loss_enhancement: f64,
|
||||
pub dropped_samples_duration: f64,
|
||||
pub dropped_samples_events: u32,
|
||||
pub total_capture_delay: f64,
|
||||
pub total_samples_captured: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct VideoSourceStats {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub frames: u32,
|
||||
pub frames_per_second: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct AudioPlayoutStats {
|
||||
pub kind: String,
|
||||
pub synthesized_samples_duration: f64,
|
||||
pub synthesized_samples_events: u32,
|
||||
pub total_samples_duration: f64,
|
||||
pub total_playout_delay: f64,
|
||||
pub total_samples_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct PeerConnectionStats {
|
||||
pub data_channels_opened: u32,
|
||||
pub data_channels_closed: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct DataChannelStats {
|
||||
pub label: String,
|
||||
pub protocol: String,
|
||||
pub data_channel_identifier: i32,
|
||||
pub state: Option<DataChannelState>,
|
||||
pub messages_sent: u32,
|
||||
pub bytes_sent: u64,
|
||||
pub messages_received: u32,
|
||||
pub bytes_received: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct TransportStats {
|
||||
pub packets_sent: u64,
|
||||
pub packets_received: u64,
|
||||
pub bytes_sent: u64,
|
||||
pub bytes_received: u64,
|
||||
pub ice_role: IceRole,
|
||||
pub ice_local_username_fragment: String,
|
||||
pub dtls_state: Option<DtlsTransportState>,
|
||||
pub ice_state: Option<IceTransportState>,
|
||||
pub selected_candidate_pair_id: String,
|
||||
pub local_certificate_id: String,
|
||||
pub remote_certificate_id: String,
|
||||
pub tls_version: String,
|
||||
pub dtls_cipher: String,
|
||||
pub dtls_role: DtlsRole,
|
||||
pub srtp_cipher: String,
|
||||
pub selected_candidate_pair_changes: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct CandidatePairStats {
|
||||
pub transport_id: String,
|
||||
pub local_candidate_id: String,
|
||||
pub remote_candidate_id: String,
|
||||
pub state: Option<IceCandidatePairState>,
|
||||
pub nominated: bool,
|
||||
pub packets_sent: u64,
|
||||
pub packets_received: u64,
|
||||
pub bytes_sent: u64,
|
||||
pub bytes_received: u64,
|
||||
pub last_packet_sent_timestamp: f64,
|
||||
pub last_packet_received_timestamp: f64,
|
||||
pub total_round_trip_time: f64,
|
||||
pub current_round_trip_time: f64,
|
||||
pub available_outgoing_bitrate: f64,
|
||||
pub available_incoming_bitrate: f64,
|
||||
pub requests_received: u64,
|
||||
pub requests_sent: u64,
|
||||
pub responses_received: u64,
|
||||
pub responses_sent: u64,
|
||||
pub consent_requests_sent: u64,
|
||||
pub packets_discarded_on_send: u32,
|
||||
pub bytes_discarded_on_send: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct IceCandidateStats {
|
||||
pub transport_id: String,
|
||||
pub address: String,
|
||||
pub port: i32,
|
||||
pub protocol: String,
|
||||
pub candidate_type: Option<IceCandidateType>,
|
||||
pub priority: i32,
|
||||
pub url: String,
|
||||
pub relay_protocol: Option<IceServerTransportProtocol>,
|
||||
pub foundation: String,
|
||||
pub related_address: String,
|
||||
pub related_port: i32,
|
||||
pub username_fragment: String,
|
||||
pub tcp_type: Option<IceTcpCandidateType>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct CertificateStats {
|
||||
pub fingerprint: String,
|
||||
pub fingerprint_algorithm: String,
|
||||
pub base64_certificate: String,
|
||||
pub issuer_certificate_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(default)]
|
||||
pub struct StreamStats {
|
||||
pub id: String,
|
||||
pub stream_identifier: String,
|
||||
// pub timestamp: i64,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::imp::video_frame as vf_imp;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SinkError {
|
||||
#[error("platform error: {0}")]
|
||||
Platform(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum VideoRotation {
|
||||
VideoRotation0 = 0,
|
||||
VideoRotation90 = 90,
|
||||
VideoRotation180 = 180,
|
||||
VideoRotation270 = 270,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum VideoFormatType {
|
||||
ARGB,
|
||||
BGRA,
|
||||
ABGR,
|
||||
RGBA,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum VideoBufferType {
|
||||
Native,
|
||||
I420,
|
||||
I420A,
|
||||
I422,
|
||||
I444,
|
||||
I010,
|
||||
NV12,
|
||||
}
|
||||
|
||||
/// Metadata carried alongside a video frame via the packet trailer mechanism.
|
||||
///
|
||||
/// Each field corresponds to an independently negotiable packet trailer feature
|
||||
/// (`PTF_USER_TIMESTAMP`, `PTF_FRAME_ID`), so individual fields are `Option`.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct FrameMetadata {
|
||||
/// Wall-clock capture time in microseconds, when `PTF_USER_TIMESTAMP` is enabled.
|
||||
pub user_timestamp: Option<u64>,
|
||||
/// Monotonically increasing frame identifier, when `PTF_FRAME_ID` is enabled.
|
||||
pub frame_id: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct VideoFrame<T>
|
||||
where
|
||||
T: AsRef<dyn VideoBuffer>,
|
||||
{
|
||||
pub rotation: VideoRotation,
|
||||
pub timestamp_us: i64, // When the frame was captured in microseconds
|
||||
/// Packet-trailer metadata, if any trailer features are active.
|
||||
pub frame_metadata: Option<FrameMetadata>,
|
||||
pub buffer: T,
|
||||
}
|
||||
|
||||
impl<T: AsRef<dyn VideoBuffer>> VideoFrame<T> {
|
||||
pub fn new(rotation: VideoRotation, buffer: T) -> Self {
|
||||
Self { rotation, timestamp_us: 0, frame_metadata: None, buffer }
|
||||
}
|
||||
}
|
||||
|
||||
pub type BoxVideoBuffer = Box<dyn VideoBuffer>;
|
||||
pub type BoxVideoFrame = VideoFrame<BoxVideoBuffer>;
|
||||
|
||||
pub(crate) mod internal {
|
||||
use super::{I420Buffer, VideoFormatType};
|
||||
|
||||
pub trait BufferSealed: Send + Sync {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn to_i420(&self) -> I420Buffer;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub trait VideoBuffer: internal::BufferSealed + Debug {
|
||||
fn width(&self) -> u32;
|
||||
fn height(&self) -> u32;
|
||||
fn buffer_type(&self) -> VideoBufferType;
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn as_native(&self) -> Option<&native::NativeBuffer> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_i420(&self) -> Option<&I420Buffer> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_i420a(&self) -> Option<&I420ABuffer> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_i422(&self) -> Option<&I422Buffer> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_i444(&self) -> Option<&I444Buffer> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_i010(&self) -> Option<&I010Buffer> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_nv12(&self) -> Option<&NV12Buffer> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! new_buffer_type {
|
||||
($type:ident, $variant:ident, $as:ident) => {
|
||||
pub struct $type {
|
||||
pub(crate) handle: vf_imp::$type,
|
||||
}
|
||||
|
||||
impl $crate::video_frame::internal::BufferSealed for $type {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer {
|
||||
self.handle.sys_handle()
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn to_i420(&self) -> I420Buffer {
|
||||
I420Buffer { handle: self.handle.to_i420() }
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
stride: u32,
|
||||
width: i32,
|
||||
height: i32,
|
||||
) {
|
||||
self.handle.to_argb(format, dst, stride, width, height)
|
||||
}
|
||||
}
|
||||
|
||||
impl VideoBuffer for $type {
|
||||
fn width(&self) -> u32 {
|
||||
self.handle.width()
|
||||
}
|
||||
|
||||
fn height(&self) -> u32 {
|
||||
self.handle.height()
|
||||
}
|
||||
|
||||
fn buffer_type(&self) -> VideoBufferType {
|
||||
VideoBufferType::$variant
|
||||
}
|
||||
|
||||
fn $as(&self) -> Option<&$type> {
|
||||
Some(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for $type {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct(stringify!($type))
|
||||
.field("width", &self.width())
|
||||
.field("height", &self.height())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<dyn VideoBuffer> for $type {
|
||||
fn as_ref(&self) -> &(dyn VideoBuffer + 'static) {
|
||||
self
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
new_buffer_type!(I420Buffer, I420, as_i420);
|
||||
new_buffer_type!(I420ABuffer, I420A, as_i420a);
|
||||
new_buffer_type!(I422Buffer, I422, as_i422);
|
||||
new_buffer_type!(I444Buffer, I444, as_i444);
|
||||
new_buffer_type!(I010Buffer, I010, as_i010);
|
||||
new_buffer_type!(NV12Buffer, NV12, as_nv12);
|
||||
|
||||
impl I420Buffer {
|
||||
pub fn with_strides(
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_u: u32,
|
||||
stride_v: u32,
|
||||
) -> I420Buffer {
|
||||
vf_imp::I420Buffer::new(width, height, stride_y, stride_u, stride_v)
|
||||
}
|
||||
|
||||
pub fn new(width: u32, height: u32) -> I420Buffer {
|
||||
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
self.handle.chroma_width()
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
self.handle.chroma_height()
|
||||
}
|
||||
|
||||
pub fn strides(&self) -> (u32, u32, u32) {
|
||||
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
|
||||
self.handle.data()
|
||||
}
|
||||
|
||||
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
|
||||
let (data_y, data_u, data_v) = self.handle.data();
|
||||
unsafe {
|
||||
(
|
||||
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
|
||||
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
|
||||
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420Buffer {
|
||||
self.handle.scale(scaled_width, scaled_height)
|
||||
}
|
||||
}
|
||||
|
||||
impl I420ABuffer {
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
self.handle.chroma_width()
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
self.handle.chroma_height()
|
||||
}
|
||||
|
||||
pub fn strides(&self) -> (u32, u32, u32, u32) {
|
||||
(
|
||||
self.handle.stride_y(),
|
||||
self.handle.stride_u(),
|
||||
self.handle.stride_v(),
|
||||
self.handle.stride_a(),
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
|
||||
self.handle.data()
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub fn data_mut(&self) -> (&mut [u8], &mut [u8], &mut [u8], Option<&mut [u8]>) {
|
||||
let (data_y, data_u, data_v, data_a) = self.handle.data();
|
||||
unsafe {
|
||||
(
|
||||
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
|
||||
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
|
||||
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
|
||||
data_a.map(|data_a| {
|
||||
std::slice::from_raw_parts_mut(data_a.as_ptr() as *mut u8, data_a.len())
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420ABuffer {
|
||||
self.handle.scale(scaled_width, scaled_height)
|
||||
}
|
||||
}
|
||||
|
||||
impl I422Buffer {
|
||||
pub fn with_strides(
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_u: u32,
|
||||
stride_v: u32,
|
||||
) -> I422Buffer {
|
||||
vf_imp::I422Buffer::new(width, height, stride_y, stride_u, stride_v)
|
||||
}
|
||||
|
||||
pub fn new(width: u32, height: u32) -> I422Buffer {
|
||||
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
self.handle.chroma_width()
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
self.handle.chroma_height()
|
||||
}
|
||||
|
||||
pub fn strides(&self) -> (u32, u32, u32) {
|
||||
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
|
||||
self.handle.data()
|
||||
}
|
||||
|
||||
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
|
||||
let (data_y, data_u, data_v) = self.handle.data();
|
||||
unsafe {
|
||||
(
|
||||
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
|
||||
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
|
||||
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I422Buffer {
|
||||
self.handle.scale(scaled_width, scaled_height)
|
||||
}
|
||||
}
|
||||
|
||||
impl I444Buffer {
|
||||
pub fn with_strides(
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_u: u32,
|
||||
stride_v: u32,
|
||||
) -> I444Buffer {
|
||||
vf_imp::I444Buffer::new(width, height, stride_y, stride_u, stride_v)
|
||||
}
|
||||
|
||||
pub fn new(width: u32, height: u32) -> I444Buffer {
|
||||
Self::with_strides(width, height, width, width, width)
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
self.handle.chroma_width()
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
self.handle.chroma_height()
|
||||
}
|
||||
|
||||
pub fn strides(&self) -> (u32, u32, u32) {
|
||||
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
|
||||
self.handle.data()
|
||||
}
|
||||
|
||||
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
|
||||
let (data_y, data_u, data_v) = self.handle.data();
|
||||
unsafe {
|
||||
(
|
||||
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
|
||||
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
|
||||
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I444Buffer {
|
||||
self.handle.scale(scaled_width, scaled_height)
|
||||
}
|
||||
}
|
||||
|
||||
impl I010Buffer {
|
||||
pub fn with_strides(
|
||||
width: u32,
|
||||
height: u32,
|
||||
stride_y: u32,
|
||||
stride_u: u32,
|
||||
stride_v: u32,
|
||||
) -> I010Buffer {
|
||||
vf_imp::I010Buffer::new(width, height, stride_y, stride_u, stride_v)
|
||||
}
|
||||
|
||||
pub fn new(width: u32, height: u32) -> I010Buffer {
|
||||
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
self.handle.chroma_width()
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
self.handle.chroma_height()
|
||||
}
|
||||
|
||||
pub fn strides(&self) -> (u32, u32, u32) {
|
||||
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
|
||||
self.handle.data()
|
||||
}
|
||||
|
||||
pub fn data_mut(&mut self) -> (&mut [u16], &mut [u16], &mut [u16]) {
|
||||
let (data_y, data_u, data_v) = self.handle.data();
|
||||
unsafe {
|
||||
(
|
||||
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u16, data_y.len()),
|
||||
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u16, data_u.len()),
|
||||
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u16, data_v.len()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I010Buffer {
|
||||
self.handle.scale(scaled_width, scaled_height)
|
||||
}
|
||||
}
|
||||
|
||||
impl NV12Buffer {
|
||||
pub fn with_strides(width: u32, height: u32, stride_y: u32, stride_uv: u32) -> NV12Buffer {
|
||||
vf_imp::NV12Buffer::new(width, height, stride_y, stride_uv)
|
||||
}
|
||||
|
||||
pub fn new(width: u32, height: u32) -> NV12Buffer {
|
||||
Self::with_strides(width, height, width, width + width % 2)
|
||||
}
|
||||
|
||||
pub fn chroma_width(&self) -> u32 {
|
||||
self.handle.chroma_width()
|
||||
}
|
||||
|
||||
pub fn chroma_height(&self) -> u32 {
|
||||
self.handle.chroma_height()
|
||||
}
|
||||
|
||||
pub fn strides(&self) -> (u32, u32) {
|
||||
(self.handle.stride_y(), self.handle.stride_uv())
|
||||
}
|
||||
|
||||
pub fn data(&self) -> (&[u8], &[u8]) {
|
||||
self.handle.data()
|
||||
}
|
||||
|
||||
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8]) {
|
||||
let (data_y, data_uv) = self.handle.data();
|
||||
unsafe {
|
||||
(
|
||||
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
|
||||
std::slice::from_raw_parts_mut(data_uv.as_ptr() as *mut u8, data_uv.len()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> NV12Buffer {
|
||||
self.handle.scale(scaled_width, scaled_height)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod native {
|
||||
use std::fmt::Debug;
|
||||
|
||||
use super::{vf_imp, I420Buffer, VideoBuffer, VideoBufferType, VideoFormatType};
|
||||
|
||||
new_buffer_type!(NativeBuffer, Native, as_native);
|
||||
|
||||
impl NativeBuffer {
|
||||
pub fn from_fluxer_d3d11_texture(
|
||||
handle: u64,
|
||||
width: u32,
|
||||
height: u32,
|
||||
dxgi_format: u32,
|
||||
) -> Option<Self> {
|
||||
vf_imp::NativeBuffer::from_fluxer_d3d11_texture(
|
||||
handle,
|
||||
width,
|
||||
height,
|
||||
dxgi_format,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn from_fluxer_dmabuf_texture(
|
||||
fds: [i32; 4],
|
||||
plane_count: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
drm_format: u32,
|
||||
modifier: u64,
|
||||
strides: [u32; 4],
|
||||
offsets: [u32; 4],
|
||||
device_uuid_hi: u64,
|
||||
device_uuid_lo: u64,
|
||||
) -> Option<Self> {
|
||||
vf_imp::NativeBuffer::from_fluxer_dmabuf_texture(
|
||||
fds,
|
||||
plane_count,
|
||||
width,
|
||||
height,
|
||||
drm_format,
|
||||
modifier,
|
||||
strides,
|
||||
offsets,
|
||||
device_uuid_hi,
|
||||
device_uuid_lo,
|
||||
)
|
||||
}
|
||||
|
||||
/// Creates a `NativeBuffer` from a `CVPixelBufferRef` pointer.
|
||||
///
|
||||
/// This function does not bump the reference count of the pixel buffer.
|
||||
///
|
||||
/// Safety: The given pointer must be a valid `CVPixelBufferRef`.
|
||||
#[cfg(any(target_os = "macos", target_os = "ios"))]
|
||||
pub unsafe fn from_cv_pixel_buffer(cv_pixel_buffer: *mut std::ffi::c_void) -> Self {
|
||||
vf_imp::NativeBuffer::from_cv_pixel_buffer(cv_pixel_buffer)
|
||||
}
|
||||
|
||||
/// Returns the `CVPixelBufferRef` that backs this buffer, or `null` if
|
||||
/// this buffer is not currently backed by a `CVPixelBufferRef`.
|
||||
///
|
||||
/// This function does not bump the reference count of the pixel buffer.
|
||||
#[cfg(any(target_os = "macos", target_os = "ios"))]
|
||||
pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void {
|
||||
self.handle.get_cv_pixel_buffer()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait VideoFrameBufferExt: VideoBuffer {
|
||||
fn to_i420(&self) -> I420Buffer;
|
||||
fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
);
|
||||
}
|
||||
|
||||
impl<T: VideoBuffer> VideoFrameBufferExt for T {
|
||||
fn to_i420(&self) -> I420Buffer {
|
||||
self.to_i420()
|
||||
}
|
||||
|
||||
fn to_argb(
|
||||
&self,
|
||||
format: VideoFormatType,
|
||||
dst: &mut [u8],
|
||||
dst_stride: u32,
|
||||
dst_width: i32,
|
||||
dst_height: i32,
|
||||
) {
|
||||
self.to_argb(format, dst, dst_stride, dst_width, dst_height)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub mod web {
|
||||
use super::VideoFrameBuffer;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct WebGlBuffer {}
|
||||
|
||||
impl VideoFrameBuffer for WebGlBuffer {}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{enum_dispatch, imp::video_source as vs_imp};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VideoResolution {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
impl Default for VideoResolution {
|
||||
// Default to 720p
|
||||
fn default() -> Self {
|
||||
VideoResolution { width: 1280, height: 720 }
|
||||
}
|
||||
}
|
||||
|
||||
#[non_exhaustive]
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum RtcVideoSource {
|
||||
// TODO(theomonnom): Web video sources (eq. to tracks on browsers?)
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
Native(native::NativeVideoSource),
|
||||
}
|
||||
|
||||
// TODO(theomonnom): Support enum dispatch with conditional compilation?
|
||||
impl RtcVideoSource {
|
||||
enum_dispatch!(
|
||||
[Native];
|
||||
pub fn video_resolution(self: &Self) -> VideoResolution;
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod native {
|
||||
use std::fmt::{Debug, Formatter};
|
||||
|
||||
use super::*;
|
||||
use crate::native::packet_trailer::PacketTrailerHandler;
|
||||
use crate::video_frame::{VideoBuffer, VideoFrame};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NativeVideoSource {
|
||||
pub(crate) handle: vs_imp::NativeVideoSource,
|
||||
}
|
||||
|
||||
impl Debug for NativeVideoSource {
|
||||
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
|
||||
f.debug_struct("NativeVideoSource").finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for NativeVideoSource {
|
||||
fn default() -> Self {
|
||||
Self::new(VideoResolution::default(), false)
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeVideoSource {
|
||||
pub fn new(resolution: VideoResolution, is_screencast: bool) -> Self {
|
||||
Self { handle: vs_imp::NativeVideoSource::new(resolution, is_screencast) }
|
||||
}
|
||||
|
||||
pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {
|
||||
self.handle.capture_frame(frame)
|
||||
}
|
||||
|
||||
/// Set the packet trailer handler used by this source.
|
||||
///
|
||||
/// When set, any frame captured with a `user_timestamp` value will
|
||||
/// automatically have its timestamp stored in the handler (keyed by
|
||||
/// the TimestampAligner-adjusted capture timestamp) so the
|
||||
/// `PacketTrailerTransformer` can embed it into the encoded frame.
|
||||
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
|
||||
self.handle.set_packet_trailer_handler(handler)
|
||||
}
|
||||
|
||||
pub fn video_resolution(&self) -> VideoResolution {
|
||||
self.handle.video_resolution()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub mod web {}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::imp::video_stream as stream_imp;
|
||||
|
||||
// There is no shared sink between native and web platforms.
|
||||
// Each platform requires different configuration (e.g: WebGlContext, ..)
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod native {
|
||||
use std::{
|
||||
fmt::{Debug, Formatter},
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
use super::stream_imp;
|
||||
use crate::{
|
||||
native::packet_trailer::PacketTrailerHandler, video_frame::BoxVideoFrame,
|
||||
video_track::RtcVideoTrack,
|
||||
};
|
||||
use livekit_runtime::Stream;
|
||||
|
||||
const DEFAULT_QUEUE_SIZE_FRAMES: usize = 1;
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct NativeVideoStreamOptions {
|
||||
/// Maximum number of queued WebRTC sink frames after the video callback.
|
||||
///
|
||||
/// `None` uses the default bounded queue size of 1 frame. `Some(0)`
|
||||
/// opts into unbounded buffering. Positive values bound the queue, and
|
||||
/// the stream drops the oldest queued frames on overflow so render
|
||||
/// latency stays bounded.
|
||||
///
|
||||
/// If your application consumes both audio and video, keep the queue
|
||||
/// sizing strategy coordinated across both streams. Using a much larger
|
||||
/// queue, or unbounded buffering, for only one of them can increase
|
||||
/// end-to-end latency for that stream and cause audio/video drift.
|
||||
pub queue_size_frames: Option<usize>,
|
||||
}
|
||||
|
||||
pub struct NativeVideoStream {
|
||||
pub(crate) handle: stream_imp::NativeVideoStream,
|
||||
}
|
||||
|
||||
impl Debug for NativeVideoStream {
|
||||
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
|
||||
f.debug_struct("NativeVideoStream").field("track", &self.track()).finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl NativeVideoStream {
|
||||
pub fn new(video_track: RtcVideoTrack) -> Self {
|
||||
Self {
|
||||
handle: stream_imp::NativeVideoStream::new(
|
||||
video_track,
|
||||
Some(DEFAULT_QUEUE_SIZE_FRAMES),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_options(video_track: RtcVideoTrack, options: NativeVideoStreamOptions) -> Self {
|
||||
Self {
|
||||
handle: stream_imp::NativeVideoStream::new(
|
||||
video_track,
|
||||
normalize_queue_size_frames(options.queue_size_frames),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the packet trailer handler for this stream.
|
||||
///
|
||||
/// When set, each frame produced by this stream will have its
|
||||
/// `user_timestamp` field populated by looking up the user
|
||||
/// timestamp for each frame's RTP timestamp.
|
||||
///
|
||||
/// Note: If the handler was already set on the `RtcVideoTrack`
|
||||
/// before creating this stream, it is automatically wired up.
|
||||
/// This method is only needed to override or set the handler
|
||||
/// after construction.
|
||||
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
|
||||
self.handle.set_packet_trailer_handler(handler);
|
||||
}
|
||||
|
||||
pub fn track(&self) -> RtcVideoTrack {
|
||||
self.handle.track()
|
||||
}
|
||||
|
||||
pub fn close(&mut self) {
|
||||
self.handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for NativeVideoStream {
|
||||
type Item = BoxVideoFrame;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
|
||||
Pin::new(&mut self.get_mut().handle).poll_next(cx)
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_queue_size_frames(queue_size_frames: Option<usize>) -> Option<usize> {
|
||||
match queue_size_frames {
|
||||
None => Some(DEFAULT_QUEUE_SIZE_FRAMES),
|
||||
Some(0) => None,
|
||||
Some(value) => Some(value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub mod web {}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use crate::{
|
||||
imp::video_track as imp_vt,
|
||||
media_stream_track::{media_stream_track, RtcTrackState},
|
||||
};
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
use crate::native::packet_trailer::PacketTrailerHandler;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtcVideoTrack {
|
||||
pub(crate) handle: imp_vt::RtcVideoTrack,
|
||||
}
|
||||
|
||||
impl RtcVideoTrack {
|
||||
media_stream_track!();
|
||||
|
||||
/// Set the packet trailer handler for this track.
|
||||
///
|
||||
/// When set, any `NativeVideoStream` created from this track will
|
||||
/// automatically use this handler to populate `user_timestamp`
|
||||
/// on each decoded frame.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
|
||||
self.handle.set_packet_trailer_handler(handler);
|
||||
}
|
||||
|
||||
/// Get the packet trailer handler, if one has been set.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn packet_trailer_handler(&self) -> Option<PacketTrailerHandler> {
|
||||
self.handle.packet_trailer_handler()
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for RtcVideoTrack {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("RtcVideoTrack")
|
||||
.field("id", &self.id())
|
||||
.field("enabled", &self.enabled())
|
||||
.field("state", &self.state())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use core::str;
|
||||
use std::cell::RefCell;
|
||||
use std::rc::Rc;
|
||||
use wasm_bindgen::prelude::*;
|
||||
use web_sys::{MessageEvent, RtcDataChannelEvent, RtcDataChannelState};
|
||||
|
||||
use crate::data_channel::{
|
||||
DataChannelError, DataChannelTrait, DataState, OnBufferedAmountChange, OnMessage, OnStateChange,
|
||||
};
|
||||
|
||||
impl From<RtcDataChannelState> for DataState {
|
||||
fn from(value: RtcDataChannelState) -> Self {
|
||||
match value {
|
||||
RtcDataChannelState::Connecting => Self::Connecting,
|
||||
RtcDataChannelState::Open => Self::Open,
|
||||
RtcDataChannelState::Closing => Self::Closing,
|
||||
RtcDataChannelState::Closed => Self::Closed,
|
||||
_ => panic!("unknown data channel state"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct DataChannel {
|
||||
sys_handle: web_sys::RtcDataChannel,
|
||||
on_closing: Rc<RefCell<Option<JsValue>>>,
|
||||
}
|
||||
|
||||
impl DataChannelTrait for DataChannel {
|
||||
fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
|
||||
if binary {
|
||||
self.sys_handle
|
||||
.send_with_u8_array(data)
|
||||
.map_err(|_| DataChannelError::Send)
|
||||
} else {
|
||||
let utf8 = str::from_utf8(data)?;
|
||||
self.sys_handle
|
||||
.send_with_str(utf8)
|
||||
.map_err(|_| DataChannelError::Send)
|
||||
}
|
||||
}
|
||||
|
||||
fn label(&self) -> String {
|
||||
self.sys_handle.label()
|
||||
}
|
||||
|
||||
fn state(&self) -> DataState {
|
||||
self.sys_handle.ready_state().into()
|
||||
}
|
||||
|
||||
fn close(&self) {
|
||||
self.sys_handle.close();
|
||||
}
|
||||
|
||||
fn on_state_change(&self, callback: Option<OnStateChange>) {
|
||||
if let Some(mut callback) = callback {
|
||||
let dc = self.clone();
|
||||
let js_callback = Closure::new(move |_: RtcDataChannelEvent| {
|
||||
callback(dc.state());
|
||||
});
|
||||
let js_callback = js_callback.into_js_value();
|
||||
self.sys_handle
|
||||
.set_onopen(Some(js_callback.unchecked_ref()));
|
||||
self.sys_handle
|
||||
.set_onclose(Some(js_callback.unchecked_ref()));
|
||||
self.sys_handle
|
||||
.add_event_listener_with_callback("closing", js_callback.unchecked_ref())
|
||||
.unwrap();
|
||||
|
||||
self.on_closing.replace(Some(js_callback));
|
||||
} else {
|
||||
self.sys_handle.set_onopen(None);
|
||||
self.sys_handle.set_onclose(None);
|
||||
if let Some(on_closing) = self.on_closing.take() {
|
||||
self.sys_handle
|
||||
.remove_event_listener_with_callback("closing", on_closing.unchecked_ref())
|
||||
.unwrap();
|
||||
}
|
||||
self.on_closing.replace(None);
|
||||
}
|
||||
}
|
||||
|
||||
fn on_message(&self, callback: Option<OnMessage>) {
|
||||
let js_callback = callback.map(|mut callback| {
|
||||
Closure::new(move |event: MessageEvent| {
|
||||
if let Some(str) = event.as_string() {
|
||||
callback(str.as_bytes(), false);
|
||||
}
|
||||
})
|
||||
.into_js_value()
|
||||
});
|
||||
|
||||
self.sys_handle.set_onmessage(
|
||||
js_callback
|
||||
.as_ref()
|
||||
.map(|callback| callback.unchecked_ref()),
|
||||
);
|
||||
}
|
||||
|
||||
fn on_buffered_amount_change(&self, _callback: Option<OnBufferedAmountChange>) {
|
||||
todo!("onbufferedamountlow instead?")
|
||||
}
|
||||
}
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::VideoTrack;
|
||||
use crate::{
|
||||
media_stream::{
|
||||
BiplanarYuv8Buffer, BiplanarYuvBuffer, PlanarYuv16BBuffer, PlanarYuv8Buffer,
|
||||
PlanarYuvBuffer, VideoFrameBuffer,
|
||||
},
|
||||
video_frame::{BiplanarYuv8Buffer, I420Buffer, SinkError, VideoFrame, VideoFrameBuffer},
|
||||
I010Buffer, I420ABuffer, I422Buffer, I444Buffer, NV12Buffer,
|
||||
};
|
||||
use std::sync::mpsc;
|
||||
use web_sys::{WebGlRenderingContext, WebGlTexture};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct WebGlVideoSink {
|
||||
track: Arc<VideoTrack>,
|
||||
gl_ctx: WebGlRenderingContext,
|
||||
tex: WebGlTexture,
|
||||
}
|
||||
|
||||
/// Create a new WebGL texture and update it inside requestAnimationFrame
|
||||
impl WebGlVideoSink {
|
||||
pub fn new(
|
||||
track: Arc<VideoTrack>,
|
||||
gl_ctx: WebGlRenderingContext,
|
||||
) -> Result<(Self, mpsc::Receiver<VideoFrame<WebGlBuffer>>), SinkError> {
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
let tex = gl_ctx.create_texture()?;
|
||||
Ok((Self { track, gl_ctx, tex }, receiver))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WebGlBuffer {
|
||||
width: i32,
|
||||
height: i32,
|
||||
tex: WebGlTexture,
|
||||
}
|
||||
|
||||
impl VideoFrameBuffer for WebGlBuffer {
|
||||
fn width(&self) -> i32 {
|
||||
self.width
|
||||
}
|
||||
|
||||
fn height(&self) -> i32 {
|
||||
self.height
|
||||
}
|
||||
}
|
||||
|
||||
/// The following types could be implemented if we want
|
||||
/// to support VideoFrame with WebCodecs
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct I420Buffer {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct I420ABuffer {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct I422Buffer {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct I444Buffer {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct I010Buffer {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct NV12Buffer {}
|
||||
|
||||
impl VideoFrameBuffer for I420Buffer {
|
||||
fn width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl VideoFrameBuffer for I420ABuffer {
|
||||
fn width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl VideoFrameBuffer for I422Buffer {
|
||||
fn width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl VideoFrameBuffer for I444Buffer {
|
||||
fn width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl VideoFrameBuffer for I010Buffer {
|
||||
fn width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl VideoFrameBuffer for NV12Buffer {
|
||||
fn width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuvBuffer for I420Buffer {
|
||||
fn chroma_width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn chroma_height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_y(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_u(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_v(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuvBuffer for I420ABuffer {
|
||||
fn chroma_width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn chroma_height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_y(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_u(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_v(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuvBuffer for I422Buffer {
|
||||
fn chroma_width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn chroma_height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_y(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_u(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_v(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuvBuffer for I444Buffer {
|
||||
fn chroma_width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn chroma_height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_y(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_u(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_v(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuvBuffer for I010Buffer {
|
||||
fn chroma_width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn chroma_height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_y(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_u(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_v(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuvBuffer for NV12Buffer {
|
||||
fn chroma_width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn chroma_height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_y(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_u(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_v(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuv8Buffer for I420Buffer {
|
||||
fn data_y(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_u(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_v(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuv8Buffer for I420ABuffer {
|
||||
fn data_y(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_u(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_v(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuv8Buffer for I422Buffer {
|
||||
fn data_y(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_u(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_v(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuv8Buffer for I444Buffer {
|
||||
fn data_y(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_u(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_v(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl PlanarYuv16BBuffer for I010Buffer {
|
||||
fn data_y(&self) -> &[u16] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_u(&self) -> &[u16] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_v(&self) -> &[u16] {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl BiplanarYuvBuffer for NV12Buffer {
|
||||
fn chroma_width(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn chroma_height(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_y(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn stride_uv(&self) -> i32 {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl BiplanarYuv8Buffer for NV12Buffer {
|
||||
fn data_y(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn data_uv(&self) -> &[u8] {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright 2025 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
unimplemented!();
|
||||
@@ -0,0 +1,148 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2021"
|
||||
name = "tract-linalg"
|
||||
version = "0.19.16"
|
||||
authors = ["Mathieu Poumeyrol <kali@zoy.org>"]
|
||||
autobenches = false
|
||||
description = "Tiny, no-nonsense, self contained, TensorFlow and ONNX inference"
|
||||
readme = "README.md"
|
||||
keywords = [
|
||||
"TensorFlow",
|
||||
"NeuralNetworks",
|
||||
]
|
||||
categories = ["science"]
|
||||
license = "MIT/Apache-2.0"
|
||||
repository = "https://github.com/snipsco/tract"
|
||||
resolver = "1"
|
||||
|
||||
[[bench]]
|
||||
name = "arm64"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mat_vec"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_wavenet_hw"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "conv_for_wavenet_hw"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_inception"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "mm_for_asr_am"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "sigmoid"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "arm64simd"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "arm32neon"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "packing"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "virtual_im2col"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "x86_64"
|
||||
bench = false
|
||||
harness = false
|
||||
|
||||
[dependencies.derive-new]
|
||||
version = "0.5.9"
|
||||
|
||||
[dependencies.downcast-rs]
|
||||
version = "1.2.0"
|
||||
|
||||
[dependencies.dyn-clone]
|
||||
version = "1.0.4"
|
||||
|
||||
[dependencies.lazy_static]
|
||||
version = "1.4.0"
|
||||
|
||||
[dependencies.log]
|
||||
version = "0.4.14"
|
||||
|
||||
[dependencies.num-traits]
|
||||
version = "0.2.14"
|
||||
|
||||
[dependencies.paste]
|
||||
version = "1.0.5"
|
||||
|
||||
[dependencies.scan_fmt]
|
||||
version = "0.2.6"
|
||||
|
||||
[dependencies.tract-data]
|
||||
version = "=0.19.16"
|
||||
|
||||
[dev-dependencies.criterion]
|
||||
version = "0.4"
|
||||
|
||||
[dev-dependencies.nu-ansi-term]
|
||||
version = "0.46"
|
||||
|
||||
[dev-dependencies.proptest]
|
||||
version = "1.0.0"
|
||||
|
||||
[build-dependencies.cc]
|
||||
version = "1.0.69"
|
||||
|
||||
[build-dependencies.half]
|
||||
version = "2"
|
||||
features = [
|
||||
"std",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[build-dependencies.liquid]
|
||||
version = "0.26"
|
||||
|
||||
[build-dependencies.liquid-core]
|
||||
version = "0.26"
|
||||
|
||||
[build-dependencies.smallvec]
|
||||
version = "1.6.1"
|
||||
|
||||
[build-dependencies.unicode-normalization]
|
||||
version = "0.1.19"
|
||||
|
||||
[build-dependencies.walkdir]
|
||||
version = "2.3.2"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
no_fp16 = []
|
||||
|
||||
[badges.maintenance]
|
||||
status = "actively-developed"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user