Add native self-hosted instance connection to fluxer_desktop

Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 +
tools/ci) with a "Connect to a Different Server" menu item and popout
that lets the desktop app switch to any self-hosted Fluxer instance,
plus fixes for well-known discovery on single-domain self-hosted
deployments and a false-positive ERR_ABORTED on same-origin client
redirects during the switch. Defaults to chat.fluxr.chat and uses an
isolated userData directory from the official build.
This commit is contained in:
2026-07-01 18:22:43 -04:00
commit 682afacd30
1763 changed files with 613720 additions and 0 deletions
@@ -0,0 +1,35 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::borrow::Cow;
#[derive(Debug, Clone)]
pub struct AudioFrame<'a> {
pub data: Cow<'a, [i16]>,
pub sample_rate: u32,
pub num_channels: u32,
pub samples_per_channel: u32,
}
impl AudioFrame<'_> {
// Owned
pub fn new(sample_rate: u32, num_channels: u32, samples_per_channel: u32) -> Self {
Self {
data: vec![0; (num_channels * samples_per_channel) as usize].into(),
sample_rate,
num_channels,
samples_per_channel,
}
}
}
@@ -0,0 +1,222 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::imp::audio_source as imp_as;
/// Default sample rate used by WebRTC audio pipelines (48kHz).
pub const DEFAULT_SAMPLE_RATE: u32 = 48000;
/// Default number of audio channels (mono).
pub const DEFAULT_NUM_CHANNELS: u32 = 1;
#[derive(Default, Debug)]
pub struct AudioSourceOptions {
pub echo_cancellation: bool,
pub noise_suppression: bool,
pub auto_gain_control: bool,
}
/// Audio source type for creating audio tracks.
///
/// Choose the appropriate source based on your use case:
///
/// | Use Case | Source | Description |
/// |----------|--------|-------------|
/// | Manual audio (TTS, files) | `RtcAudioSource::Native(source)` | Push frames manually |
/// | Microphone capture | `RtcAudioSource::Device` | Automatic via Platform ADM |
/// | Both (mic + screen) | Use both types | Multiple tracks supported |
///
/// # Combining Sources
///
/// You can have multiple audio tracks with different source types:
/// - Track A: `RtcAudioSource::Device` for microphone (via `PlatformAudio`)
/// - Track B: `RtcAudioSource::Native` for screen capture or TTS
#[non_exhaustive]
#[derive(Debug, Clone)]
pub enum RtcAudioSource {
/// Native audio source for manual audio frame capture.
///
/// Use this with Synthetic ADM mode (the default). You push audio frames
/// manually via `NativeAudioSource::capture_frame()`.
///
/// # Example
///
/// ```rust,ignore
/// use livekit::webrtc::audio_source::native::NativeAudioSource;
/// use livekit::webrtc::audio_source::{AudioSourceOptions, RtcAudioSource};
///
/// let source = NativeAudioSource::new(
/// AudioSourceOptions::default(),
/// 48000, 2, 100,
/// );
/// source.capture_frame(&frame).await?;
///
/// let track = LocalAudioTrack::create_audio_track(
/// "audio",
/// RtcAudioSource::Native(source),
/// );
/// ```
#[cfg(not(target_arch = "wasm32"))]
Native(native::NativeAudioSource),
/// Device audio source - uses Platform ADM for automatic microphone capture.
///
/// WebRTC automatically captures audio from the selected recording device
/// (microphone). You do NOT push frames manually.
///
/// # Usage
///
/// Use `PlatformAudio` from the `livekit` crate, which manages the Platform ADM
/// lifecycle and provides `RtcAudioSource::Device` via `rtc_source()`:
///
/// ```rust,ignore
/// use livekit::prelude::*;
///
/// // Create PlatformAudio (enables Platform ADM)
/// let audio = PlatformAudio::new()?;
///
/// // Optionally select a specific device
/// if let Some(device) = audio.recording_devices().next() {
/// audio.set_recording_device(&device.id)?;
/// }
///
/// // Create track using the device source
/// let track = LocalAudioTrack::create_audio_track("mic", audio.rtc_source());
/// ```
///
/// # Combining with NativeAudioSource
///
/// You CAN use `NativeAudioSource` alongside Platform ADM to have multiple
/// audio tracks with different sources (e.g., microphone + screen capture).
///
/// # Platform Support
///
/// - **iOS**: CoreAudio with VPIO (Voice Processing IO)
/// - **macOS**: CoreAudio
/// - **Windows**: WASAPI
/// - **Linux**: PulseAudio / ALSA
/// - **Android**: AAudio / OpenSL ES
#[cfg(not(target_arch = "wasm32"))]
Device,
}
impl RtcAudioSource {
/// Set audio processing options.
/// Note: For `Device` source, options are controlled by the Platform ADM.
pub fn set_audio_options(&self, options: AudioSourceOptions) {
match self {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(source) => source.set_audio_options(options),
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Device => {
// Device source options are managed by the Platform ADM
// This is a no-op
}
}
}
/// Get audio processing options.
/// Note: For `Device` source, returns default options (actual options are managed by ADM).
pub fn audio_options(&self) -> AudioSourceOptions {
match self {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(source) => source.audio_options(),
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Device => AudioSourceOptions::default(),
}
}
/// Get the sample rate.
/// Note: For `Device` source, returns [`DEFAULT_SAMPLE_RATE`] (48kHz).
pub fn sample_rate(&self) -> u32 {
match self {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(source) => source.sample_rate(),
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Device => DEFAULT_SAMPLE_RATE,
}
}
/// Get the number of channels.
/// Note: For `Device` source, returns [`DEFAULT_NUM_CHANNELS`] (mono).
pub fn num_channels(&self) -> u32 {
match self {
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Native(source) => source.num_channels(),
#[cfg(not(target_arch = "wasm32"))]
RtcAudioSource::Device => DEFAULT_NUM_CHANNELS,
}
}
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::fmt::{Debug, Formatter};
use super::*;
use crate::{audio_frame::AudioFrame, RtcError};
#[derive(Clone)]
pub struct NativeAudioSource {
pub(crate) handle: imp_as::NativeAudioSource,
}
impl Debug for NativeAudioSource {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeAudioSource").finish()
}
}
impl NativeAudioSource {
pub fn new(
options: AudioSourceOptions,
sample_rate: u32,
num_channels: u32,
queue_size_ms: u32,
) -> NativeAudioSource {
Self {
handle: imp_as::NativeAudioSource::new(
options,
sample_rate,
num_channels,
queue_size_ms,
),
}
}
pub fn clear_buffer(&self) {
self.handle.clear_buffer()
}
pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> {
self.handle.capture_frame(frame).await
}
pub fn set_audio_options(&self, options: AudioSourceOptions) {
self.handle.set_audio_options(options)
}
pub fn audio_options(&self) -> AudioSourceOptions {
self.handle.audio_options()
}
pub fn sample_rate(&self) -> u32 {
self.handle.sample_rate()
}
pub fn num_channels(&self) -> u32 {
self.handle.num_channels()
}
}
}
@@ -0,0 +1,113 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::imp::audio_stream as stream_imp;
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::{
fmt::{Debug, Formatter},
pin::Pin,
task::{Context, Poll},
};
use livekit_runtime::Stream;
use super::stream_imp;
use crate::{audio_frame::AudioFrame, audio_track::RtcAudioTrack};
const DEFAULT_QUEUE_SIZE_FRAMES: usize = 10;
#[derive(Clone, Debug, Default)]
pub struct NativeAudioStreamOptions {
/// Maximum number of queued WebRTC sink frames after the audio callback.
///
/// Each queued frame corresponds to roughly 10 ms of decoded PCM audio
/// on the WebRTC sink path.
///
/// `None` uses the default bounded queue size of 10 frames. `Some(0)`
/// opts into unbounded buffering. Positive values bound the queue, and
/// the stream drops the oldest queued frames on overflow so latency
/// stays bounded.
///
/// If your application consumes both audio and video, keep the queue
/// sizing strategy coordinated across both streams. Using a much larger
/// queue, or unbounded buffering, for only one of them can increase
/// end-to-end latency for that stream and cause audio/video drift.
pub queue_size_frames: Option<usize>,
}
pub struct NativeAudioStream {
pub(crate) handle: stream_imp::NativeAudioStream,
}
impl Debug for NativeAudioStream {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeAudioStream").field("track", &self.track()).finish()
}
}
impl NativeAudioStream {
pub fn new(audio_track: RtcAudioTrack, sample_rate: i32, num_channels: i32) -> Self {
Self {
handle: stream_imp::NativeAudioStream::new(
audio_track,
sample_rate,
num_channels,
Some(DEFAULT_QUEUE_SIZE_FRAMES),
),
}
}
pub fn with_options(
audio_track: RtcAudioTrack,
sample_rate: i32,
num_channels: i32,
options: NativeAudioStreamOptions,
) -> Self {
Self {
handle: stream_imp::NativeAudioStream::new(
audio_track,
sample_rate,
num_channels,
normalize_queue_size_frames(options.queue_size_frames),
),
}
}
pub fn track(&self) -> RtcAudioTrack {
self.handle.track()
}
pub fn close(&mut self) {
self.handle.close()
}
}
impl Stream for NativeAudioStream {
type Item = AudioFrame<'static>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.get_mut().handle).poll_next(cx)
}
}
fn normalize_queue_size_frames(queue_size_frames: Option<usize>) -> Option<usize> {
match queue_size_frames {
None => Some(DEFAULT_QUEUE_SIZE_FRAMES),
Some(0) => None,
Some(value) => Some(value),
}
}
}
@@ -0,0 +1,39 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::audio_track as imp_at,
media_stream_track::{media_stream_track, RtcTrackState},
};
#[derive(Clone)]
pub struct RtcAudioTrack {
pub(crate) handle: imp_at::RtcAudioTrack,
}
impl RtcAudioTrack {
media_stream_track!();
}
impl Debug for RtcAudioTrack {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtcAudioTrack")
.field("id", &self.id())
.field("enabled", &self.enabled())
.field("state", &self.state())
.finish()
}
}
@@ -0,0 +1,125 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{fmt::Debug, str::Utf8Error};
use serde::Deserialize;
use thiserror::Error;
use crate::{imp::data_channel as dc_imp, rtp_parameters::Priority};
#[derive(Clone, Debug)]
pub struct DataChannelInit {
pub ordered: bool,
pub max_retransmit_time: Option<i32>,
pub max_retransmits: Option<i32>,
pub protocol: String,
pub negotiated: bool,
pub id: i32,
pub priority: Option<Priority>,
}
impl Default for DataChannelInit {
fn default() -> Self {
Self {
ordered: true,
max_retransmit_time: None,
max_retransmits: None,
protocol: String::new(),
negotiated: false,
id: -1,
priority: None,
}
}
}
#[derive(Debug, Error)]
pub enum DataChannelError {
#[error("failed to send data, dc not open? send buffer is full ?")]
Send,
#[error("only utf8 strings can be sent")]
Utf8(#[from] Utf8Error),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DataChannelState {
Connecting,
Open,
Closing,
Closed,
}
#[derive(Debug)]
pub struct DataBuffer<'a> {
pub data: &'a [u8],
pub binary: bool,
}
pub type OnStateChange = Box<dyn FnMut(DataChannelState) + Send + Sync>;
pub type OnMessage = Box<dyn FnMut(DataBuffer) + Send + Sync>;
pub type OnBufferedAmountChange = Box<dyn FnMut(u64) + Send + Sync>;
#[derive(Clone)]
pub struct DataChannel {
pub(crate) handle: dc_imp::DataChannel,
}
impl DataChannel {
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
self.handle.send(data, binary)
}
pub fn id(&self) -> i32 {
self.handle.id()
}
pub fn label(&self) -> String {
self.handle.label()
}
pub fn state(&self) -> DataChannelState {
self.handle.state()
}
pub fn close(&self) {
self.handle.close()
}
pub fn buffered_amount(&self) -> u64 {
self.handle.buffered_amount()
}
pub fn on_state_change(&self, callback: Option<OnStateChange>) {
self.handle.on_state_change(callback)
}
pub fn on_message(&self, callback: Option<OnMessage>) {
self.handle.on_message(callback)
}
pub fn on_buffered_amount_change(&self, callback: Option<OnBufferedAmountChange>) {
self.handle.on_buffered_amount_change(callback)
}
}
impl Debug for DataChannel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DataChannel")
.field("id", &self.id())
.field("label", &self.label())
.field("state", &self.state())
.finish()
}
}
@@ -0,0 +1,230 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::imp::desktop_capturer as imp_dc;
/// Configuration options for creating a desktop capturer.
///
/// It contains a subset of libwebrtc's DesktopCaptureOptions.
///
/// By default, it captures the entire screen and does not include the cursor.
///
/// # Example
/// ```no_run
/// use libwebrtc::desktop_capturer::{DesktopCapturerOptions, DesktopCaptureSourceType};
///
/// let mut options = DesktopCapturerOptions::new(DesktopCaptureSourceType::Screen);
/// options.set_include_cursor(true);
/// ```
pub struct DesktopCapturerOptions {
sys_handle: imp_dc::DesktopCapturerOptions,
}
/// Specifies the type of source that a desktop capturer should capture.
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum DesktopCaptureSourceType {
Screen,
Window,
#[cfg(any(target_os = "macos", target_os = "linux"))]
Generic,
}
impl DesktopCapturerOptions {
/// Creates a new `DesktopCapturerOptions` with default values.
///
/// # Arguments
///
/// * `source_type` - The type of source to capture (screen or window).
///
/// # Defaults
///
/// - Cursor is not included in captured frames (use [`set_include_cursor`](Self::set_include_cursor) to change)
/// - On macOS, the ScreenCaptureKit system picker is enabled (use [`set_sck_system_picker`](Self::set_sck_system_picker) to change)
pub fn new(source_type: DesktopCaptureSourceType) -> Self {
let source_type = match source_type {
DesktopCaptureSourceType::Screen => imp_dc::SourceType::Screen,
DesktopCaptureSourceType::Window => imp_dc::SourceType::Window,
#[cfg(any(target_os = "macos", target_os = "linux"))]
DesktopCaptureSourceType::Generic => imp_dc::SourceType::Generic,
};
Self { sys_handle: imp_dc::DesktopCapturerOptions::new(source_type) }
}
/// Sets whether to include the cursor in captured frames.
pub fn set_include_cursor(&mut self, include: bool) {
self.sys_handle = self.sys_handle.with_cursor(include);
}
/// Sets whether to allow the ScreenCaptureKit system picker on macOS.
///
/// This is enabled by default.
///
/// When disabled, for capturing displays the client should get the source id
/// via a different way as [`DesktopCapturer::get_source_list`] returns an empty vector.
#[cfg(target_os = "macos")]
pub fn set_sck_system_picker(&mut self, allow_sck_system_picker: bool) {
self.sys_handle = self.sys_handle.with_sck_system_picker(allow_sck_system_picker);
}
}
/// A desktop capturer for capturing screens or windows.
pub struct DesktopCapturer {
handle: imp_dc::DesktopCapturer,
}
impl DesktopCapturer {
/// Creates a new `DesktopCapturer` with the specified callback and options.
///
/// # Arguments
///
/// * `options` - Configuration options for the capturer
///
/// # Returns
///
/// Returns `Some(DesktopCapturer)` if the capturer was created successfully,
/// or `None` if creation failed (e.g., due to platform limitations or permissions).
pub fn new(options: DesktopCapturerOptions) -> Option<Self> {
let desktop_capturer = imp_dc::DesktopCapturer::new(options.sys_handle);
if desktop_capturer.is_none() {
return None;
}
Some(Self { handle: desktop_capturer.unwrap() })
}
/// Starts capturing from the specified source.
///
/// # Arguments
///
/// * `source` - The capture source to use. It should be None when the capturer
/// is configured to use the system picker (on platforms that support it).
/// * `callback` - A function that will be called for each captured frame. The callback
/// receives a [`CaptureResult`] indicating success or error, and a [`DesktopFrame`]
/// containing the captured image data.
///
/// # Note
///
/// After calling this method, you must call [`capture_frame`](Self::capture_frame)
/// to actually capture frames. This method only initializes the capture session.
pub fn start_capture<T>(&mut self, source: Option<CaptureSource>, mut callback: T)
where
T: FnMut(Result<DesktopFrame, CaptureError>) + Send + 'static,
{
if let Some(source) = source {
self.handle.select_source(source.sys_handle.id());
}
let inner_callback = move |result: Result<imp_dc::DesktopFrame, imp_dc::CaptureError>| {
callback(capture_result_from_sys(result));
};
self.handle.start(inner_callback);
}
/// Captures a single frame.
///
/// You must call [`start_capture`](Self::start_capture) before calling this method.
pub fn capture_frame(&mut self) {
self.handle.capture_frame();
}
/// Retrieves a list of available capture sources.
///
/// Returns a list of screens or windows that can be captured, depending
/// on whether the capturer was configured for window or screen capture.
///
/// # Returns
///
/// A vector of [`CaptureSource`] objects representing available capture sources.
pub fn get_source_list(&self) -> Vec<CaptureSource> {
let source_list = self.handle.get_source_list();
source_list.into_iter().map(|source| CaptureSource { sys_handle: source }).collect()
}
}
pub struct DesktopFrame {
sys_handle: imp_dc::DesktopFrame,
}
impl DesktopFrame {
fn new(sys_handle: imp_dc::DesktopFrame) -> Self {
Self { sys_handle }
}
pub fn width(&self) -> i32 {
self.sys_handle.width() as i32
}
pub fn height(&self) -> i32 {
self.sys_handle.height() as i32
}
pub fn stride(&self) -> u32 {
self.sys_handle.stride() as u32
}
pub fn left(&self) -> i32 {
self.sys_handle.left()
}
pub fn top(&self) -> i32 {
self.sys_handle.top()
}
pub fn data(&self) -> &[u8] {
self.sys_handle.data()
}
}
#[derive(Clone)]
pub struct CaptureSource {
sys_handle: imp_dc::CaptureSource,
}
impl CaptureSource {
pub fn id(&self) -> u64 {
self.sys_handle.id()
}
pub fn title(&self) -> String {
self.sys_handle.title()
}
pub fn display_id(&self) -> i64 {
self.sys_handle.display_id()
}
}
impl std::fmt::Display for CaptureSource {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CaptureSource")
.field("id", &self.id())
.field("title", &self.title())
.field("display_id", &self.display_id())
.finish()
}
}
#[derive(Debug, PartialEq)]
pub enum CaptureError {
Temporary,
Permanent,
}
fn capture_result_from_sys(
result: Result<imp_dc::DesktopFrame, imp_dc::CaptureError>,
) -> Result<DesktopFrame, CaptureError> {
match result {
Ok(frame) => Ok(DesktopFrame::new(frame)),
Err(error) => Err(match error {
imp_dc::CaptureError::Temporary => CaptureError::Temporary,
imp_dc::CaptureError::Permanent => CaptureError::Permanent,
}),
}
}
@@ -0,0 +1,41 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// TODO(theomonnom): Async methods
#[macro_export]
macro_rules! enum_dispatch {
// This arm is used to avoid nested loops with the arguments
// The arguments are transformed to $combined_args tt
(@match [$($variant:ident),+]: $fnc:ident, $self:ident, $combined_args:tt) => {
match $self {
$(
Self::$variant(inner) => inner.$fnc$combined_args,
)+
}
};
// Create the function and extract self fron the $args tt (little hack)
(@fnc [$($variant:ident),+]: $vis:vis fn $fnc:ident($self:ident: $sty:ty $(, $arg:ident: $t:ty)*) -> $ret:ty) => {
#[inline]
$vis fn $fnc($self: $sty, $($arg: $t),*) -> $ret {
$crate::enum_dispatch!(@match [$($variant),+]: $fnc, $self, ($($arg,)*))
}
};
($variants:tt; $($vis:vis fn $fnc:ident$args:tt -> $ret:ty;)+) => {
$(
$crate::enum_dispatch!(@fnc $variants: $vis fn $fnc$args -> $ret);
)+
};
}
@@ -0,0 +1,55 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{imp::ice_candidate as imp_ic, session_description::SdpParseError};
pub struct IceCandidate {
pub(crate) handle: imp_ic::IceCandidate,
}
impl IceCandidate {
pub fn parse(
sdp_mid: &str,
sdp_mline_index: i32,
sdp: &str,
) -> Result<IceCandidate, SdpParseError> {
imp_ic::IceCandidate::parse(sdp_mid, sdp_mline_index, sdp)
}
pub fn sdp_mid(&self) -> String {
self.handle.sdp_mid()
}
pub fn sdp_mline_index(&self) -> i32 {
self.handle.sdp_mline_index()
}
pub fn candidate(&self) -> String {
self.handle.candidate()
}
}
impl ToString for IceCandidate {
fn to_string(&self) -> String {
self.handle.to_string()
}
}
impl Debug for IceCandidate {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("IceCandidate").field("candidate", &self.to_string()).finish()
}
}
@@ -0,0 +1,82 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use thiserror::Error;
#[cfg_attr(target_arch = "wasm32", path = "web/mod.rs")]
#[cfg_attr(not(target_arch = "wasm32"), path = "native/mod.rs")]
mod imp;
mod enum_dispatch;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum MediaType {
Audio,
Video,
Data,
Unsupported,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtcErrorType {
Internal,
InvalidSdp,
InvalidState,
}
#[derive(Error, Debug)]
#[error("an RtcError occurred: {error_type:?} - {message}")]
pub struct RtcError {
pub error_type: RtcErrorType,
pub message: String,
}
pub mod audio_frame;
pub mod audio_source;
pub mod audio_stream;
pub mod audio_track;
pub mod data_channel;
#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
pub mod desktop_capturer;
pub mod ice_candidate;
pub mod media_stream;
pub mod media_stream_track;
pub mod peer_connection;
pub mod peer_connection_factory;
pub mod prelude;
pub mod recorded_audio;
pub mod rtp_parameters;
pub mod rtp_receiver;
pub mod rtp_sender;
pub mod rtp_transceiver;
pub mod session_description;
pub mod stats;
pub mod video_frame;
pub mod video_source;
pub mod video_stream;
pub mod video_track;
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
pub use webrtc_sys::webrtc::ffi::create_random_uuid;
pub use crate::imp::{
apm, audio_mixer, audio_resampler, frame_cryptor, packet_trailer, yuv_helper,
};
}
#[cfg(target_os = "android")]
pub mod android {
pub use crate::imp::android::*;
}
@@ -0,0 +1,46 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{audio_track::RtcAudioTrack, imp::media_stream as imp_ms, video_track::RtcVideoTrack};
#[derive(Clone)]
pub struct MediaStream {
pub(crate) handle: imp_ms::MediaStream,
}
impl MediaStream {
pub fn id(&self) -> String {
self.handle.id()
}
pub fn audio_tracks(&self) -> Vec<RtcAudioTrack> {
self.handle.audio_tracks()
}
pub fn video_tracks(&self) -> Vec<RtcVideoTrack> {
self.handle.video_tracks()
}
}
impl Debug for MediaStream {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("MediaStream")
.field("id", &self.id())
.field("audio_tracks", &self.audio_tracks())
.field("video_tracks", &self.video_tracks())
.finish()
}
}
@@ -0,0 +1,86 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{audio_track::RtcAudioTrack, enum_dispatch, video_track::RtcVideoTrack};
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtcTrackState {
Live,
Ended,
}
#[derive(Debug, Clone)]
pub enum MediaStreamTrack {
Video(RtcVideoTrack),
Audio(RtcAudioTrack),
}
#[cfg(not(target_arch = "wasm32"))]
impl MediaStreamTrack {
enum_dispatch!(
[Video, Audio];
pub(crate) fn sys_handle(self: &Self) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack>;
);
}
impl MediaStreamTrack {
enum_dispatch!(
[Video, Audio];
pub fn id(self: &Self) -> String;
pub fn enabled(self: &Self) -> bool;
pub fn set_enabled(self: &Self, enabled: bool) -> bool;
pub fn state(self: &Self) -> RtcTrackState;
);
}
macro_rules! media_stream_track {
() => {
pub fn id(&self) -> String {
self.handle.id()
}
pub fn enabled(&self) -> bool {
self.handle.enabled()
}
pub fn set_enabled(&self, enabled: bool) -> bool {
self.handle.set_enabled(enabled)
}
pub fn state(&self) -> RtcTrackState {
self.handle.state().into()
}
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn sys_handle(
&self,
) -> cxx::SharedPtr<webrtc_sys::media_stream::ffi::MediaStreamTrack> {
self.handle.sys_handle()
}
};
}
pub(crate) use media_stream_track;
impl From<RtcAudioTrack> for MediaStreamTrack {
fn from(track: RtcAudioTrack) -> Self {
Self::Audio(track)
}
}
impl From<RtcVideoTrack> for MediaStreamTrack {
fn from(track: RtcVideoTrack) -> Self {
Self::Video(track)
}
}
@@ -0,0 +1,65 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use jni::objects::JObject;
use webrtc_sys::android::ffi as sys_android;
/// Initialize Android WebRTC with the JVM.
///
/// This is automatically called by [`initialize_android_context`], so you only
/// need to call this directly if you don't have access to an Android Context
/// (e.g., in `JNI_OnLoad`).
///
/// This function is idempotent - safe to call multiple times.
pub fn initialize_android(vm: &jni::JavaVM) {
unsafe {
sys_android::init_android(vm.get_java_vm_pointer() as *mut _);
}
}
/// Initialize Android WebRTC with the application context.
///
/// This is the main initialization function for Android. It performs both:
/// 1. JVM initialization (same as [`initialize_android`])
/// 2. Context initialization (required for PlatformAudio)
///
/// This function is idempotent - safe to call multiple times.
///
/// # Arguments
/// * `vm` - The JavaVM instance
/// * `context` - The Android application context
///
/// # Returns
/// `true` if context initialization succeeded, `false` otherwise.
/// Note: JVM initialization always happens regardless of return value.
///
/// # Example
/// ```ignore
/// use jni::JavaVM;
/// use jni::objects::JObject;
/// use livekit::webrtc::android::initialize_android_context;
///
/// fn init(vm: JavaVM, context: JObject) {
/// // Just one call needed - handles both JVM and context init
/// initialize_android_context(&vm, &context);
/// }
/// ```
pub fn initialize_android_context(vm: &jni::JavaVM, context: &JObject) -> bool {
unsafe {
sys_android::init_android_context(
vm.get_java_vm_pointer() as *mut _,
context.as_raw() as usize,
)
}
}
@@ -0,0 +1,119 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use webrtc_sys::apm::ffi as sys_apm;
use crate::{RtcError, RtcErrorType};
pub struct AudioProcessingModule {
sys_handle: UniquePtr<sys_apm::AudioProcessingModule>,
}
impl AudioProcessingModule {
pub fn new(
echo_canceller_enabled: bool,
gain_controller_enabled: bool,
high_pass_filter_enabled: bool,
noise_suppression_enabled: bool,
) -> Self {
Self {
sys_handle: sys_apm::create_apm(
echo_canceller_enabled,
gain_controller_enabled,
high_pass_filter_enabled,
noise_suppression_enabled,
),
}
}
pub fn process_stream(
&mut self,
data: &mut [i16],
sample_rate: i32,
num_channels: i32,
) -> Result<(), RtcError> {
let samples_per_10ms = (sample_rate as usize / 100) * num_channels as usize;
assert!(
data.len() % samples_per_10ms == 0 && data.len() >= samples_per_10ms,
"slice must have a multiple of 10ms worth of samples"
);
for chunk in data.chunks_mut(samples_per_10ms) {
if unsafe {
self.sys_handle.pin_mut().process_stream(
chunk.as_mut_ptr(),
chunk.len(),
chunk.as_mut_ptr(),
chunk.len(),
sample_rate,
num_channels,
)
} != 0
{
return Err(RtcError {
error_type: RtcErrorType::Internal,
message: "Failed to process stream".to_string(),
});
}
}
Ok(())
}
pub fn process_reverse_stream(
&mut self,
data: &mut [i16],
sample_rate: i32,
num_channels: i32,
) -> Result<(), RtcError> {
let samples_per_10ms = (sample_rate as usize / 100) * num_channels as usize;
assert!(
data.len() % samples_per_10ms == 0 && data.len() >= samples_per_10ms,
"slice must have a multiple of 10ms worth of samples"
);
for chunk in data.chunks_mut(samples_per_10ms) {
if unsafe {
self.sys_handle.pin_mut().process_reverse_stream(
chunk.as_mut_ptr(),
chunk.len(),
chunk.as_mut_ptr(),
chunk.len(),
sample_rate,
num_channels,
)
} != 0
{
return Err(RtcError {
error_type: RtcErrorType::Internal,
message: "Failed to process reverse stream".to_string(),
});
}
}
Ok(())
}
pub fn set_stream_delay_ms(&mut self, delay_ms: i32) -> Result<(), RtcError> {
if self.sys_handle.pin_mut().set_stream_delay_ms(delay_ms) == 0 {
Ok(())
} else {
Err(RtcError {
error_type: RtcErrorType::Internal,
message: "Failed to set stream delay".to_string(),
})
}
}
}
@@ -0,0 +1,108 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::audio_frame::AudioFrame;
use cxx::UniquePtr;
use std::sync::Arc;
use webrtc_sys::audio_mixer as sys;
use webrtc_sys::audio_mixer::ffi;
pub struct AudioMixer {
sys_handle: UniquePtr<ffi::AudioMixer>,
}
pub use ffi::AudioFrameInfo;
pub trait AudioMixerSource {
fn ssrc(&self) -> i32;
fn preferred_sample_rate(&self) -> u32;
fn get_audio_frame_with_info(&self, target_sample_rate: u32) -> Option<AudioFrame<'_>>;
}
struct AudioMixerSourceImpl<T> {
inner: T,
}
impl<T: AudioMixerSource> sys::AudioMixerSource for AudioMixerSourceImpl<T> {
fn ssrc(&self) -> i32 {
self.inner.ssrc()
}
fn preferred_sample_rate(&self) -> i32 {
self.inner.preferred_sample_rate() as i32
}
fn get_audio_frame_with_info(
&self,
target_sample_rate: i32,
native_frame: sys::NativeAudioFrame,
) -> AudioFrameInfo {
if let Some(frame) = self.inner.get_audio_frame_with_info(target_sample_rate as u32) {
let samples_count = (frame.sample_rate as usize / 100) as usize;
assert_eq!(
frame.sample_rate, target_sample_rate as u32,
"sample rate must match target_sample_rate"
);
assert_eq!(
frame.samples_per_channel as usize, samples_count,
"frame must contain 10ms of samples"
);
assert_eq!(
frame.data.len(),
samples_count * frame.num_channels as usize,
"slice must contain 10ms of samples"
);
unsafe {
native_frame.update_frame(
0,
frame.data.as_ptr(),
frame.samples_per_channel as usize,
frame.sample_rate as i32,
frame.num_channels as usize,
);
}
return ffi::AudioFrameInfo::Normal;
} else {
return ffi::AudioFrameInfo::Muted;
}
}
}
impl AudioMixer {
pub fn new() -> Self {
let sys_handle = ffi::create_audio_mixer();
Self { sys_handle }
}
pub fn add_source(&mut self, source: impl AudioMixerSource + 'static) {
let source_impl = AudioMixerSourceImpl { inner: source };
let wrapper = Box::new(sys::AudioMixerSourceWrapper::new(Arc::new(source_impl)));
unsafe {
self.sys_handle.pin_mut().add_source(wrapper);
}
}
pub fn remove_source(&mut self, ssrc: i32) {
unsafe {
self.sys_handle.pin_mut().remove_source(ssrc);
}
}
pub fn mix(&mut self, num_channels: usize) -> &[i16] {
unsafe {
let len = self.sys_handle.pin_mut().mix(num_channels);
std::slice::from_raw_parts(self.sys_handle.data(), len)
}
}
}
@@ -0,0 +1,53 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use webrtc_sys::audio_resampler as sys_ar;
pub struct AudioResampler {
sys_handle: UniquePtr<sys_ar::ffi::AudioResampler>,
}
impl Default for AudioResampler {
fn default() -> Self {
Self { sys_handle: sys_ar::ffi::create_audio_resampler() }
}
}
impl AudioResampler {
pub fn remix_and_resample<'a>(
&'a mut self,
src: &[i16],
samples_per_channel: u32,
num_channels: u32,
sample_rate: u32,
dst_num_channels: u32,
dst_sample_rate: u32,
) -> &'a [i16] {
assert!(src.len() >= (samples_per_channel * num_channels) as usize, "src buffer too small");
unsafe {
let len = self.sys_handle.pin_mut().remix_and_resample(
src.as_ptr(),
samples_per_channel as usize,
num_channels as usize,
sample_rate as i32,
dst_num_channels as usize,
dst_sample_rate as i32,
);
std::slice::from_raw_parts(self.sys_handle.data(), len / 2)
}
}
}
@@ -0,0 +1,205 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use tokio::sync::oneshot;
use webrtc_sys::audio_track as sys_at;
use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions, RtcError, RtcErrorType};
#[derive(Clone)]
pub struct NativeAudioSource {
sys_handle: SharedPtr<sys_at::ffi::AudioTrackSource>,
sample_rate: u32,
num_channels: u32,
queue_size_samples: u32,
}
impl NativeAudioSource {
/// Creates a new [`NativeAudioSource`].
///
/// # Arguments
/// * `options` Configuration options for the source (e.g. echo cancellation, noise suppression).
/// * `sample_rate` Sampling rate in Hz (for example, `48000`).
/// * `num_channels` Number of audio channels (`1` for mono, `2` for stereo, etc.).
/// * `queue_size_ms` Size of the internal buffering queue, in milliseconds.
///
/// # Behavior
/// - If `queue_size_ms` is **zero**, buffering is **disabled** and audio frames are
/// delivered directly to webrtc sinks. In this mode, the caller **must provide 10 ms frames**
/// (i.e., `sample_rate / 100` samples per channel) when calling [`capture_frame`].
/// - If `queue_size_ms` is **non-zero**, buffering is enabled. The value must be a
/// **multiple of 10**, representing the total buffering duration in milliseconds.
/// Frames will be queued and flushed to sinks asynchronously once the buffer
/// reaches the configured threshold.
///
/// # Panics
/// assert if `queue_size_ms` is not a multiple of 10.
pub fn new(
options: AudioSourceOptions,
sample_rate: u32,
num_channels: u32,
queue_size_ms: u32,
) -> NativeAudioSource {
assert!(queue_size_ms % 10 == 0, "queue_size_ms must be a multiple of 10");
let sys_handle = sys_at::ffi::new_audio_track_source(
options.into(),
sample_rate.try_into().unwrap(),
num_channels.try_into().unwrap(),
queue_size_ms.try_into().unwrap(),
);
let queue_size_samples = (queue_size_ms * sample_rate * num_channels) / 1000;
Self { sys_handle, sample_rate, num_channels, queue_size_samples }
}
pub fn sys_handle(&self) -> SharedPtr<sys_at::ffi::AudioTrackSource> {
self.sys_handle.clone()
}
pub fn set_audio_options(&self, options: AudioSourceOptions) {
self.sys_handle.set_audio_options(&sys_at::ffi::AudioSourceOptions::from(options))
}
pub fn audio_options(&self) -> AudioSourceOptions {
self.sys_handle.audio_options().into()
}
pub fn sample_rate(&self) -> u32 {
self.sample_rate
}
pub fn num_channels(&self) -> u32 {
self.num_channels
}
pub fn clear_buffer(&self) {
self.sys_handle.clear_buffer();
}
pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> {
if self.sample_rate != frame.sample_rate || self.num_channels != frame.num_channels {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "sample_rate and num_channels don't match".to_owned(),
});
}
// Fast path: no buffering
if self.queue_size_samples == 0 {
// frame size must be 10ms for fast path
let expected_frames_per_ch = (self.sample_rate / 100) as usize;
if frame.data.len() % (self.num_channels as usize) != 0 {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "frame.data length not divisible by channel count".to_owned(),
});
}
let nb_frames = frame.data.len() / (self.num_channels as usize);
if nb_frames != expected_frames_per_ch {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: format!(
"direct capture requires 10ms frames: got {} frames, expected {}",
nb_frames, expected_frames_per_ch
),
});
}
// Define a no-op callback for fast path (queue_size_ms=0)
// This is safer than passing null, which can cause UB in release mode optimizations
extern "C" fn noop_complete_callback(_ctx: *const sys_at::SourceContext) {
// No-op: fast path completes synchronously, no callback needed
}
unsafe {
let data: &[i16] = frame.data.as_ref();
// Use a valid no-op callback instead of null for safety
// In release mode, transmuting null pointers can cause UB
let noop_callback = sys_at::CompleteCallback(noop_complete_callback);
let ok = self.sys_handle.capture_frame(
data,
self.sample_rate,
self.num_channels,
nb_frames,
std::ptr::null(), // Context is still null - callback won't use it
noop_callback,
);
if !ok {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "failed to capture frame without buffering".to_owned(),
});
}
}
return Ok(());
}
// Buffered path.
extern "C" fn lk_audio_source_complete(userdata: *const sys_at::SourceContext) {
let tx = unsafe { Box::from_raw(userdata as *mut oneshot::Sender<()>) };
let _ = tx.send(());
}
// iterate over chunks of self._queue_size_samples
for chunk in frame.data.chunks(self.queue_size_samples as usize) {
let nb_frames = chunk.len() / self.num_channels as usize;
let (tx, rx) = oneshot::channel::<()>();
let ctx = Box::new(tx);
let ctx_ptr = Box::into_raw(ctx) as *const sys_at::SourceContext;
unsafe {
// In the fast path, C++ never store / invoke on_complete / ctx.
if !self.sys_handle.capture_frame(
chunk,
self.sample_rate,
self.num_channels,
nb_frames,
ctx_ptr,
sys_at::CompleteCallback(lk_audio_source_complete),
) {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "failed to capture frame".to_owned(),
});
}
}
let _ = rx.await;
}
Ok(())
}
}
impl From<sys_at::ffi::AudioSourceOptions> for AudioSourceOptions {
fn from(options: sys_at::ffi::AudioSourceOptions) -> Self {
Self {
echo_cancellation: options.echo_cancellation,
noise_suppression: options.noise_suppression,
auto_gain_control: options.auto_gain_control,
}
}
}
impl From<AudioSourceOptions> for sys_at::ffi::AudioSourceOptions {
fn from(options: AudioSourceOptions) -> Self {
Self {
echo_cancellation: options.echo_cancellation,
noise_suppression: options.noise_suppression,
auto_gain_control: options.auto_gain_control,
}
}
}
@@ -0,0 +1,314 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
collections::VecDeque,
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
},
task::{Context, Poll, Waker},
};
use cxx::SharedPtr;
use livekit_runtime::Stream;
use parking_lot::Mutex;
use rtrb::{Consumer, Producer, PushError, RingBuffer};
use webrtc_sys::audio_track as sys_at;
use crate::{audio_frame::AudioFrame, audio_track::RtcAudioTrack};
pub struct NativeAudioStream {
native_sink: SharedPtr<sys_at::ffi::NativeAudioSink>,
audio_track: RtcAudioTrack,
frame_queue: Arc<AudioFrameQueue>,
}
impl NativeAudioStream {
pub fn new(
audio_track: RtcAudioTrack,
sample_rate: i32,
num_channels: i32,
queue_size_frames: Option<usize>,
) -> Self {
let frame_queue = Arc::new(AudioFrameQueue::new(queue_size_frames));
let observer = Arc::new(AudioTrackObserver { frame_queue: frame_queue.clone() });
let native_sink = sys_at::ffi::new_native_audio_sink(
Box::new(sys_at::AudioSinkWrapper::new(observer.clone())),
sample_rate,
num_channels,
);
let audio = unsafe { sys_at::ffi::media_to_audio(audio_track.sys_handle()) };
audio.add_sink(&native_sink);
Self { native_sink, audio_track, frame_queue }
}
pub fn track(&self) -> RtcAudioTrack {
self.audio_track.clone()
}
pub fn close(&mut self) {
let audio = unsafe { sys_at::ffi::media_to_audio(self.audio_track.sys_handle()) };
audio.remove_sink(&self.native_sink);
self.frame_queue.close();
}
}
impl Drop for NativeAudioStream {
fn drop(&mut self) {
self.close();
}
}
impl Stream for NativeAudioStream {
type Item = AudioFrame<'static>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
self.frame_queue.poll_recv(cx)
}
}
pub struct AudioTrackObserver {
frame_queue: Arc<AudioFrameQueue>,
}
impl sys_at::AudioSink for AudioTrackObserver {
fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) {
self.frame_queue.push(AudioFrame {
data: data.to_owned().into(),
sample_rate: sample_rate as u32,
num_channels: nb_channels as u32,
samples_per_channel: nb_frames as u32,
});
}
}
struct AudioFrameQueue {
kind: AudioFrameQueueKind,
closed: AtomicBool,
dropped_frames: AtomicU64,
waker: Mutex<Option<Waker>>,
}
enum AudioFrameQueueKind {
Bounded(BoundedAudioFrameQueue),
Unbounded(UnboundedAudioFrameQueue),
}
struct BoundedAudioFrameQueue {
producer: Mutex<Producer<AudioFrame<'static>>>,
consumer: Mutex<Consumer<AudioFrame<'static>>>,
}
struct UnboundedAudioFrameQueue {
frames: Mutex<VecDeque<AudioFrame<'static>>>,
}
impl AudioFrameQueue {
fn new(capacity: Option<usize>) -> Self {
let kind = match capacity.filter(|capacity| *capacity > 0) {
Some(capacity) => {
let (producer, consumer) = RingBuffer::new(capacity);
AudioFrameQueueKind::Bounded(BoundedAudioFrameQueue {
producer: Mutex::new(producer),
consumer: Mutex::new(consumer),
})
}
None => AudioFrameQueueKind::Unbounded(UnboundedAudioFrameQueue {
frames: Mutex::new(VecDeque::new()),
}),
};
Self {
kind,
closed: AtomicBool::new(false),
dropped_frames: AtomicU64::new(0),
waker: Mutex::new(None),
}
}
fn push(&self, frame: AudioFrame<'static>) {
if self.closed.load(Ordering::Acquire) {
return;
}
match &self.kind {
AudioFrameQueueKind::Bounded(queue) => self.push_bounded(queue, frame),
AudioFrameQueueKind::Unbounded(queue) => {
queue.frames.lock().push_back(frame);
}
}
self.wake_receiver();
}
fn push_bounded(&self, queue: &BoundedAudioFrameQueue, mut frame: AudioFrame<'static>) {
loop {
let push_result = queue.producer.lock().push(frame);
match push_result {
Ok(()) => return,
Err(PushError::Full(returned_frame)) => {
frame = returned_frame;
let dropped = queue.consumer.lock().pop().is_ok();
if dropped {
self.record_drop();
} else {
return;
}
}
}
}
}
fn close(&self) {
self.closed.store(true, Ordering::Release);
self.wake_receiver();
match &self.kind {
AudioFrameQueueKind::Bounded(queue) => {
let mut consumer = queue.consumer.lock();
while consumer.pop().is_ok() {}
}
AudioFrameQueueKind::Unbounded(queue) => {
queue.frames.lock().clear();
}
}
}
fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Option<AudioFrame<'static>>> {
if let Some(frame) = self.try_pop() {
return Poll::Ready(Some(frame));
}
if self.closed.load(Ordering::Acquire) {
return Poll::Ready(None);
}
*self.waker.lock() = Some(cx.waker().clone());
if let Some(frame) = self.try_pop() {
self.waker.lock().take();
Poll::Ready(Some(frame))
} else if self.closed.load(Ordering::Acquire) {
Poll::Ready(None)
} else {
Poll::Pending
}
}
fn try_pop(&self) -> Option<AudioFrame<'static>> {
match &self.kind {
AudioFrameQueueKind::Bounded(queue) => queue.consumer.lock().pop().ok(),
AudioFrameQueueKind::Unbounded(queue) => queue.frames.lock().pop_front(),
}
}
fn wake_receiver(&self) {
let waker = self.waker.lock().take();
if let Some(waker) = waker {
waker.wake();
}
}
fn record_drop(&self) {
let dropped_frames = self.dropped_frames.fetch_add(1, Ordering::Relaxed) + 1;
if dropped_frames == 1 || dropped_frames % 100 == 0 {
log::warn!(
"native audio stream queue overflow; dropped {} queued frames",
dropped_frames
);
}
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::Ordering;
use super::AudioFrameQueue;
use crate::audio_frame::AudioFrame;
fn test_frame(marker: i16) -> AudioFrame<'static> {
AudioFrame {
data: vec![marker].into(),
sample_rate: 48_000,
num_channels: 1,
samples_per_channel: 1,
}
}
fn pop_marker(queue: &AudioFrameQueue) -> Option<i16> {
queue.try_pop().map(|frame| frame.data[0])
}
#[test]
fn bounded_queue_preserves_fifo_order_under_capacity() {
let queue = AudioFrameQueue::new(Some(3));
queue.push(test_frame(1));
queue.push(test_frame(2));
queue.push(test_frame(3));
assert_eq!(pop_marker(&queue), Some(1));
assert_eq!(pop_marker(&queue), Some(2));
assert_eq!(pop_marker(&queue), Some(3));
assert_eq!(pop_marker(&queue), None);
}
#[test]
fn bounded_queue_drops_oldest_when_full() {
let queue = AudioFrameQueue::new(Some(2));
queue.push(test_frame(1));
queue.push(test_frame(2));
queue.push(test_frame(3));
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 1);
assert_eq!(pop_marker(&queue), Some(2));
assert_eq!(pop_marker(&queue), Some(3));
assert_eq!(pop_marker(&queue), None);
}
#[test]
fn unbounded_queue_retains_all_frames() {
let queue = AudioFrameQueue::new(None);
for marker in 1..=4 {
queue.push(test_frame(marker));
}
for marker in 1..=4 {
assert_eq!(pop_marker(&queue), Some(marker));
}
assert_eq!(pop_marker(&queue), None);
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 0);
}
#[test]
fn close_clears_buffer_and_rejects_future_pushes() {
let queue = AudioFrameQueue::new(Some(2));
queue.push(test_frame(1));
queue.close();
queue.push(test_frame(2));
assert_eq!(pop_marker(&queue), None);
}
}
@@ -0,0 +1,33 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use sys_at::ffi::audio_to_media;
use webrtc_sys::audio_track as sys_at;
use super::media_stream_track::impl_media_stream_track;
use crate::media_stream_track::RtcTrackState;
#[derive(Clone)]
pub struct RtcAudioTrack {
pub(crate) sys_handle: SharedPtr<sys_at::ffi::AudioTrack>,
}
impl RtcAudioTrack {
impl_media_stream_track!(audio_to_media);
pub fn sys_handle(&self) -> SharedPtr<sys_at::ffi::MediaStreamTrack> {
audio_to_media(self.sys_handle.clone())
}
}
@@ -0,0 +1,142 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{str, sync::Arc};
use cxx::SharedPtr;
use parking_lot::Mutex;
use webrtc_sys::data_channel as sys_dc;
use crate::data_channel::{
DataBuffer, DataChannelError, DataChannelInit, DataChannelState, OnBufferedAmountChange,
OnMessage, OnStateChange,
};
impl From<sys_dc::ffi::DataState> for DataChannelState {
fn from(value: sys_dc::ffi::DataState) -> Self {
match value {
sys_dc::ffi::DataState::Connecting => Self::Connecting,
sys_dc::ffi::DataState::Open => Self::Open,
sys_dc::ffi::DataState::Closing => Self::Closing,
sys_dc::ffi::DataState::Closed => Self::Closed,
_ => panic!("unknown data channel state"),
}
}
}
impl From<DataChannelInit> for sys_dc::ffi::DataChannelInit {
fn from(value: DataChannelInit) -> Self {
Self {
ordered: value.ordered,
has_max_retransmit_time: value.max_retransmit_time.is_some(),
max_retransmit_time: value.max_retransmit_time.unwrap_or_default(),
has_max_retransmits: value.max_retransmits.is_some(),
max_retransmits: value.max_retransmits.unwrap_or_default(),
protocol: value.protocol,
id: value.id,
has_priority: false,
priority: sys_dc::ffi::Priority::Medium,
negotiated: value.negotiated,
}
}
}
#[derive(Clone)]
pub struct DataChannel {
observer: Arc<DataChannelObserver>,
pub(crate) sys_handle: SharedPtr<sys_dc::ffi::DataChannel>,
}
impl DataChannel {
pub fn configure(sys_handle: SharedPtr<sys_dc::ffi::DataChannel>) -> Self {
let observer = Arc::new(DataChannelObserver::default());
let dc = Self { sys_handle: sys_handle.clone(), observer: observer.clone() };
dc.sys_handle
.register_observer(Box::new(sys_dc::DataChannelObserverWrapper::new(observer)));
dc
}
pub fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
if !binary {
str::from_utf8(data)?;
}
let buffer = sys_dc::ffi::DataBuffer { ptr: data.as_ptr(), len: data.len(), binary };
self.sys_handle.send(&buffer).then_some(()).ok_or(DataChannelError::Send)
}
pub fn id(&self) -> i32 {
self.sys_handle.id()
}
pub fn label(&self) -> String {
self.sys_handle.label()
}
pub fn state(&self) -> DataChannelState {
self.sys_handle.state().into()
}
pub fn close(&self) {
self.sys_handle.close();
}
pub fn buffered_amount(&self) -> u64 {
self.sys_handle.buffered_amount()
}
pub fn on_state_change(&self, handler: Option<OnStateChange>) {
*self.observer.state_change_handler.lock() = handler;
}
pub fn on_message(&self, handler: Option<OnMessage>) {
*self.observer.message_handler.lock() = handler;
}
pub fn on_buffered_amount_change(&self, handler: Option<OnBufferedAmountChange>) {
*self.observer.buffered_amount_change_handler.lock() = handler;
}
}
#[derive(Default)]
struct DataChannelObserver {
state_change_handler: Mutex<Option<OnStateChange>>,
message_handler: Mutex<Option<OnMessage>>,
buffered_amount_change_handler: Mutex<Option<OnBufferedAmountChange>>,
}
impl sys_dc::DataChannelObserver for DataChannelObserver {
fn on_state_change(&self, state: sys_dc::ffi::DataState) {
let mut handler = self.state_change_handler.lock();
if let Some(f) = handler.as_mut() {
f(state.into());
}
}
fn on_message(&self, data: &[u8], binary: bool) {
let mut handler = self.message_handler.lock();
if let Some(f) = handler.as_mut() {
f(DataBuffer { data, binary });
}
}
fn on_buffered_amount_change(&self, sent_data_size: u64) {
let mut handler = self.buffered_amount_change_handler.lock();
if let Some(f) = handler.as_mut() {
f(sent_data_size);
}
}
}
@@ -0,0 +1,241 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use webrtc_sys::desktop_capturer::{self as sys_dc, ffi::new_desktop_capturer};
#[derive(Debug, Copy, Clone, PartialEq)]
pub(crate) enum SourceType {
Screen,
Window,
Generic,
}
#[derive(Copy, Clone, Debug)]
pub(crate) struct DesktopCapturerOptions {
source_type: SourceType,
include_cursor: bool,
#[cfg(target_os = "macos")]
allow_sck_system_picker: bool,
}
impl Default for DesktopCapturerOptions {
fn default() -> Self {
Self {
source_type: SourceType::Screen,
include_cursor: false,
#[cfg(target_os = "macos")]
allow_sck_system_picker: true,
}
}
}
impl DesktopCapturerOptions {
pub(crate) fn new(source_type: SourceType) -> Self {
Self { source_type, ..Default::default() }
}
pub(crate) fn with_cursor(mut self, include: bool) -> Self {
self.include_cursor = include;
self
}
#[cfg(target_os = "macos")]
pub(crate) fn with_sck_system_picker(mut self, allow_sck_system_picker: bool) -> Self {
self.allow_sck_system_picker = allow_sck_system_picker;
self
}
pub(crate) fn to_sys_handle(&self) -> sys_dc::ffi::DesktopCapturerOptions {
let source_type = match self.source_type {
SourceType::Screen => sys_dc::ffi::SourceType::Screen,
SourceType::Window => sys_dc::ffi::SourceType::Window,
SourceType::Generic => sys_dc::ffi::SourceType::Generic,
};
let mut sys_handle = sys_dc::ffi::DesktopCapturerOptions {
source_type,
include_cursor: self.include_cursor,
allow_sck_system_picker: false,
};
#[cfg(target_os = "macos")]
{
sys_handle.allow_sck_system_picker = self.allow_sck_system_picker;
}
sys_handle
}
}
pub(crate) struct DesktopCapturer {
sys_handle: UniquePtr<sys_dc::ffi::DesktopCapturer>,
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
glib_loop: Option<glib::MainLoop>,
}
impl DesktopCapturer {
pub(crate) fn new(options: DesktopCapturerOptions) -> Option<Self> {
let sys_handle = new_desktop_capturer(options.to_sys_handle());
if sys_handle.is_null() {
None
} else {
Some(Self {
sys_handle,
#[cfg(all(
any(target_os = "linux", target_os = "freebsd"),
feature = "glib-main-loop"
))]
glib_loop: None,
})
}
}
pub(crate) fn capture_frame(&self) {
self.sys_handle.capture_frame();
}
pub(crate) fn start<T>(&mut self, callback: T)
where
T: FnMut(Result<DesktopFrame, CaptureError>) + Send + 'static,
{
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
if std::env::var("WAYLAND_DISPLAY").is_ok() {
let main_loop = glib::MainLoop::new(None, false);
self.glib_loop = Some(main_loop.clone());
let _handle = std::thread::spawn(move || {
main_loop.run();
});
}
let pin_handle = self.sys_handle.pin_mut();
let callback = DesktopCallback::new(callback);
let callback_wrapper = sys_dc::DesktopCapturerCallbackWrapper::new(Box::new(callback));
pin_handle.start(Box::new(callback_wrapper));
}
pub(crate) fn select_source(&self, id: u64) -> bool {
self.sys_handle.select_source(id)
}
pub(crate) fn get_source_list(&self) -> Vec<CaptureSource> {
let mut sources = Vec::new();
let source_list = self.sys_handle.get_source_list();
for source in source_list.iter() {
sources.push(CaptureSource { sys_handle: source.clone() });
}
sources
}
}
#[cfg(all(any(target_os = "linux", target_os = "freebsd"), feature = "glib-main-loop"))]
impl Drop for DesktopCapturer {
fn drop(&mut self) {
if let Some(glib_loop) = &self.glib_loop {
glib_loop.quit();
}
}
}
pub(crate) struct DesktopFrame {
sys_handle: UniquePtr<sys_dc::ffi::DesktopFrame>,
}
impl DesktopFrame {
fn new(sys_handle: UniquePtr<sys_dc::ffi::DesktopFrame>) -> Self {
Self { sys_handle }
}
pub(crate) fn width(&self) -> i32 {
self.sys_handle.width()
}
pub(crate) fn height(&self) -> i32 {
self.sys_handle.height()
}
pub(crate) fn stride(&self) -> u32 {
self.sys_handle.stride() as u32
}
pub(crate) fn left(&self) -> i32 {
self.sys_handle.left()
}
pub(crate) fn top(&self) -> i32 {
self.sys_handle.top()
}
pub(crate) fn data(&self) -> &[u8] {
let data = self.sys_handle.data();
unsafe { std::slice::from_raw_parts(data, self.stride() as usize * self.height() as usize) }
}
}
struct DesktopCallback<T: FnMut(Result<DesktopFrame, CaptureError>) + Send> {
callback: T,
}
impl<T> DesktopCallback<T>
where
T: FnMut(Result<DesktopFrame, CaptureError>) + Send,
{
fn new(callback: T) -> Self {
Self { callback }
}
fn capture_result_from_sys(
result: Result<UniquePtr<sys_dc::ffi::DesktopFrame>, sys_dc::CaptureError>,
) -> Result<DesktopFrame, CaptureError> {
match result {
Ok(frame) => Ok(DesktopFrame::new(frame)),
Err(error) => Err(match error {
sys_dc::CaptureError::Temporary => CaptureError::Temporary,
sys_dc::CaptureError::Permanent => CaptureError::Permanent,
}),
}
}
}
impl<T> sys_dc::DesktopCapturerCallback for DesktopCallback<T>
where
T: FnMut(Result<DesktopFrame, CaptureError>) + Send,
{
fn on_capture_result(
&mut self,
result: Result<UniquePtr<sys_dc::ffi::DesktopFrame>, sys_dc::CaptureError>,
) {
(self.callback)(DesktopCallback::<T>::capture_result_from_sys(result));
}
}
#[derive(Clone)]
pub(crate) struct CaptureSource {
sys_handle: sys_dc::ffi::Source,
}
impl CaptureSource {
pub(crate) fn id(&self) -> u64 {
self.sys_handle.id
}
pub(crate) fn title(&self) -> String {
self.sys_handle.title.clone()
}
pub(crate) fn display_id(&self) -> i64 {
self.sys_handle.display_id
}
}
pub(crate) enum CaptureError {
Temporary,
Permanent,
}
@@ -0,0 +1,316 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use cxx::SharedPtr;
use parking_lot::Mutex;
use webrtc_sys::frame_cryptor::{self as sys_fc};
use crate::{
native::packet_trailer::PacketTrailerHandler, peer_connection_factory::PeerConnectionFactory,
rtp_receiver::RtpReceiver, rtp_sender::RtpSender,
};
pub type OnStateChange = Box<dyn FnMut(String, EncryptionState) + Send + Sync>;
#[derive(Copy, Clone, Debug)]
#[non_exhaustive]
pub enum KeyDerivationAlgorithm {
PBKDF2,
HKDF,
}
impl Into<sys_fc::ffi::KeyDerivationAlgorithm> for KeyDerivationAlgorithm {
fn into(self) -> sys_fc::ffi::KeyDerivationAlgorithm {
match self {
KeyDerivationAlgorithm::PBKDF2 => sys_fc::ffi::KeyDerivationAlgorithm::PBKDF2,
KeyDerivationAlgorithm::HKDF => sys_fc::ffi::KeyDerivationAlgorithm::HKDF,
}
}
}
#[derive(Debug, Clone)]
pub struct KeyProviderOptions {
pub shared_key: bool,
pub ratchet_window_size: i32,
pub ratchet_salt: Vec<u8>,
pub failure_tolerance: i32,
pub key_ring_size: i32,
pub key_derivation_algorithm: KeyDerivationAlgorithm,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncryptionAlgorithm {
AesGcm,
AesCbc,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncryptionState {
New,
Ok,
EncryptionFailed,
DecryptionFailed,
MissingKey,
KeyRatcheted,
InternalError,
}
#[derive(Debug, Clone)]
pub struct EncryptedPacket {
pub data: Vec<u8>,
pub iv: Vec<u8>,
pub key_index: u32,
}
#[derive(Clone)]
pub struct KeyProvider {
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::KeyProvider>,
}
impl KeyProvider {
pub fn new(options: KeyProviderOptions) -> Self {
Self { sys_handle: sys_fc::ffi::new_key_provider(options.into()) }
}
pub fn set_shared_key(&self, key_index: i32, key: Vec<u8>) -> bool {
self.sys_handle.set_shared_key(key_index, key)
}
pub fn ratchet_shared_key(&self, key_index: i32) -> Option<Vec<u8>> {
self.sys_handle.ratchet_shared_key(key_index).ok()
}
pub fn get_shared_key(&self, key_index: i32) -> Option<Vec<u8>> {
self.sys_handle.get_shared_key(key_index).ok()
}
pub fn set_key(&self, participant_id: String, key_index: i32, key: Vec<u8>) -> bool {
self.sys_handle.set_key(participant_id, key_index, key)
}
pub fn ratchet_key(&self, participant_id: String, key_index: i32) -> Option<Vec<u8>> {
self.sys_handle.ratchet_key(participant_id, key_index).ok()
}
pub fn get_key(&self, participant_id: String, key_index: i32) -> Option<Vec<u8>> {
self.sys_handle.get_key(participant_id, key_index).ok()
}
pub fn set_sif_trailer(&self, trailer: Vec<u8>) {
self.sys_handle.set_sif_trailer(trailer);
}
}
#[derive(Clone)]
pub struct FrameCryptor {
observer: Arc<RtcFrameCryptorObserver>,
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::FrameCryptor>,
}
impl FrameCryptor {
pub fn new_for_rtp_sender(
peer_factory: &PeerConnectionFactory,
participant_id: String,
algorithm: EncryptionAlgorithm,
key_provider: KeyProvider,
sender: RtpSender,
) -> Self {
let observer = Arc::new(RtcFrameCryptorObserver::default());
let sys_handle = sys_fc::ffi::new_frame_cryptor_for_rtp_sender(
peer_factory.handle.sys_handle.clone(),
participant_id,
algorithm.into(),
key_provider.sys_handle,
sender.handle.sys_handle,
);
let fc = Self { observer: observer.clone(), sys_handle: sys_handle.clone() };
fc.sys_handle
.register_observer(Box::new(sys_fc::RtcFrameCryptorObserverWrapper::new(observer)));
fc
}
pub fn new_for_rtp_receiver(
peer_factory: &PeerConnectionFactory,
participant_id: String,
algorithm: EncryptionAlgorithm,
key_provider: KeyProvider,
receiver: RtpReceiver,
) -> Self {
let observer = Arc::new(RtcFrameCryptorObserver::default());
let sys_handle = sys_fc::ffi::new_frame_cryptor_for_rtp_receiver(
peer_factory.handle.sys_handle.clone(),
participant_id,
algorithm.into(),
key_provider.sys_handle,
receiver.handle.sys_handle,
);
let fc = Self { observer: observer.clone(), sys_handle: sys_handle.clone() };
fc.sys_handle
.register_observer(Box::new(sys_fc::RtcFrameCryptorObserverWrapper::new(observer)));
fc
}
pub fn set_enabled(self: &FrameCryptor, enabled: bool) {
self.sys_handle.set_enabled(enabled);
}
pub fn enabled(self: &FrameCryptor) -> bool {
self.sys_handle.enabled()
}
pub fn set_key_index(self: &FrameCryptor, index: i32) {
self.sys_handle.set_key_index(index);
}
pub fn key_index(self: &FrameCryptor) -> i32 {
self.sys_handle.key_index()
}
pub fn participant_id(self: &FrameCryptor) -> String {
self.sys_handle.participant_id()
}
pub fn on_state_change(&self, handler: Option<OnStateChange>) {
*self.observer.state_change_handler.lock() = handler;
}
pub fn set_packet_trailer_handler(&self, handler: &PacketTrailerHandler) {
self.sys_handle.set_packet_trailer_handler(handler.sys_handle());
}
}
#[derive(Clone)]
pub struct DataPacketCryptor {
pub(crate) sys_handle: SharedPtr<sys_fc::ffi::DataPacketCryptor>,
}
impl DataPacketCryptor {
pub fn new(algorithm: EncryptionAlgorithm, key_provider: KeyProvider) -> Self {
Self {
sys_handle: sys_fc::ffi::new_data_packet_cryptor(
algorithm.into(),
key_provider.sys_handle,
),
}
}
pub fn encrypt(
&self,
participant_id: &str,
key_index: u32,
data: &[u8],
) -> Result<EncryptedPacket, Box<dyn std::error::Error>> {
let data_vec: Vec<u8> = data.to_vec();
match self.sys_handle.encrypt_data_packet(participant_id.to_string(), key_index, data_vec) {
Ok(packet) => Ok(packet.into()),
Err(e) => Err(format!("Encryption failed: {}", e).into()),
}
}
pub fn decrypt(
&self,
participant_id: &str,
encrypted_packet: &EncryptedPacket,
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
match self
.sys_handle
.decrypt_data_packet(participant_id.to_string(), &encrypted_packet.clone().into())
{
Ok(data) => Ok(data.into_iter().collect()),
Err(e) => Err(format!("Decryption failed: {}", e).into()),
}
}
}
#[derive(Default)]
struct RtcFrameCryptorObserver {
state_change_handler: Mutex<Option<OnStateChange>>,
}
impl sys_fc::RtcFrameCryptorObserver for RtcFrameCryptorObserver {
fn on_frame_cryption_state_change(
&self,
participant_id: String,
state: sys_fc::ffi::FrameCryptionState,
) {
let mut handler = self.state_change_handler.lock();
if let Some(f) = handler.as_mut() {
f(participant_id, state.into());
}
}
}
impl From<sys_fc::ffi::Algorithm> for EncryptionAlgorithm {
fn from(value: sys_fc::ffi::Algorithm) -> Self {
match value {
sys_fc::ffi::Algorithm::AesGcm => Self::AesGcm,
sys_fc::ffi::Algorithm::AesCbc => Self::AesCbc,
_ => panic!("unknown frame cyrptor Algorithm"),
}
}
}
impl From<EncryptionAlgorithm> for sys_fc::ffi::Algorithm {
fn from(value: EncryptionAlgorithm) -> Self {
match value {
EncryptionAlgorithm::AesGcm => Self::AesGcm,
EncryptionAlgorithm::AesCbc => Self::AesCbc,
}
}
}
impl From<sys_fc::ffi::FrameCryptionState> for EncryptionState {
fn from(value: sys_fc::ffi::FrameCryptionState) -> Self {
match value {
sys_fc::ffi::FrameCryptionState::New => Self::New,
sys_fc::ffi::FrameCryptionState::Ok => Self::Ok,
sys_fc::ffi::FrameCryptionState::EncryptionFailed => Self::EncryptionFailed,
sys_fc::ffi::FrameCryptionState::DecryptionFailed => Self::DecryptionFailed,
sys_fc::ffi::FrameCryptionState::MissingKey => Self::MissingKey,
sys_fc::ffi::FrameCryptionState::KeyRatcheted => Self::KeyRatcheted,
sys_fc::ffi::FrameCryptionState::InternalError => Self::InternalError,
_ => panic!("unknown frame cyrptor FrameCryptionState"),
}
}
}
impl From<KeyProviderOptions> for sys_fc::ffi::KeyProviderOptions {
fn from(value: KeyProviderOptions) -> Self {
Self {
shared_key: value.shared_key,
ratchet_window_size: value.ratchet_window_size,
ratchet_salt: value.ratchet_salt,
failure_tolerance: value.failure_tolerance,
key_ring_size: value.key_ring_size,
key_derivation_algorithm: value.key_derivation_algorithm.into(),
}
}
}
impl From<sys_fc::ffi::EncryptedPacket> for EncryptedPacket {
fn from(value: sys_fc::ffi::EncryptedPacket) -> Self {
Self {
data: value.data.into_iter().collect(),
iv: value.iv.into_iter().collect(),
key_index: value.key_index,
}
}
}
impl From<EncryptedPacket> for sys_fc::ffi::EncryptedPacket {
fn from(value: EncryptedPacket) -> Self {
Self { data: value.data, iv: value.iv, key_index: value.key_index }
}
}
@@ -0,0 +1,60 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use webrtc_sys::jsep as sys_jsep;
use crate::{ice_candidate as ic, session_description::SdpParseError};
#[derive(Clone)]
pub struct IceCandidate {
pub(crate) sys_handle: SharedPtr<sys_jsep::ffi::IceCandidate>,
}
impl IceCandidate {
pub fn parse(
sdp_mid: &str,
sdp_mline_index: i32,
sdp: &str,
) -> Result<ic::IceCandidate, SdpParseError> {
let res = sys_jsep::ffi::create_ice_candidate(
sdp_mid.to_string(),
sdp_mline_index,
sdp.to_string(),
);
match res {
Ok(sys_handle) => Ok(ic::IceCandidate { handle: IceCandidate { sys_handle } }),
Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
}
}
pub fn sdp_mid(&self) -> String {
self.sys_handle.sdp_mid()
}
pub fn sdp_mline_index(&self) -> i32 {
self.sys_handle.sdp_mline_index()
}
pub fn candidate(&self) -> String {
self.sys_handle.candidate()
}
}
impl ToString for IceCandidate {
fn to_string(&self) -> String {
self.sys_handle.stringify()
}
}
@@ -0,0 +1,49 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use webrtc_sys::media_stream as sys_ms;
use crate::{
audio_track,
imp::{audio_track::RtcAudioTrack, video_track::RtcVideoTrack},
video_track,
};
#[derive(Clone)]
pub struct MediaStream {
pub(crate) sys_handle: SharedPtr<sys_ms::ffi::MediaStream>,
}
impl MediaStream {
pub fn id(&self) -> String {
self.sys_handle.id()
}
pub fn audio_tracks(&self) -> Vec<audio_track::RtcAudioTrack> {
self.sys_handle
.get_audio_tracks()
.into_iter()
.map(|t| audio_track::RtcAudioTrack { handle: RtcAudioTrack { sys_handle: t.ptr } })
.collect()
}
pub fn video_tracks(&self) -> Vec<video_track::RtcVideoTrack> {
self.sys_handle
.get_video_tracks()
.into_iter()
.map(|t| video_track::RtcVideoTrack { handle: RtcVideoTrack::new(t.ptr) })
.collect()
}
}
@@ -0,0 +1,78 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use webrtc_sys::{
audio_track::ffi::media_to_audio, media_stream_track as sys_mst,
video_track::ffi::media_to_video, MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO,
};
use crate::{
audio_track,
imp::{audio_track::RtcAudioTrack, video_track::RtcVideoTrack},
media_stream_track::{MediaStreamTrack, RtcTrackState},
video_track,
};
impl From<sys_mst::ffi::TrackState> for RtcTrackState {
fn from(state: sys_mst::ffi::TrackState) -> Self {
match state {
sys_mst::ffi::TrackState::Live => RtcTrackState::Live,
sys_mst::ffi::TrackState::Ended => RtcTrackState::Ended,
_ => panic!("unknown TrackState"),
}
}
}
pub fn new_media_stream_track(
sys_handle: SharedPtr<sys_mst::ffi::MediaStreamTrack>,
) -> MediaStreamTrack {
if sys_handle.kind() == MEDIA_TYPE_AUDIO {
MediaStreamTrack::Audio(audio_track::RtcAudioTrack {
handle: RtcAudioTrack { sys_handle: unsafe { media_to_audio(sys_handle) } },
})
} else if sys_handle.kind() == MEDIA_TYPE_VIDEO {
MediaStreamTrack::Video(video_track::RtcVideoTrack {
handle: RtcVideoTrack::new(unsafe { media_to_video(sys_handle) }),
})
} else {
panic!("unknown track kind")
}
}
macro_rules! impl_media_stream_track {
($cast:expr) => {
pub fn id(&self) -> String {
let ptr = $cast(self.sys_handle.clone());
ptr.id()
}
pub fn enabled(&self) -> bool {
let ptr = $cast(self.sys_handle.clone());
ptr.enabled()
}
pub fn set_enabled(&self, enabled: bool) -> bool {
let ptr = $cast(self.sys_handle.clone());
ptr.set_enabled(enabled)
}
pub fn state(&self) -> RtcTrackState {
let ptr = $cast(self.sys_handle.clone());
ptr.state().into()
}
};
}
pub(super) use impl_media_stream_track;
@@ -0,0 +1,72 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(target_os = "android")]
pub mod android;
pub mod apm;
pub mod audio_mixer;
pub mod audio_resampler;
pub mod audio_source;
pub mod audio_stream;
pub mod audio_track;
pub mod data_channel;
#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
pub mod desktop_capturer;
pub mod frame_cryptor;
pub mod ice_candidate;
pub mod media_stream;
pub mod media_stream_track;
pub mod packet_trailer;
pub mod peer_connection;
pub mod peer_connection_factory;
pub mod rtp_parameters;
pub mod rtp_receiver;
pub mod rtp_sender;
pub mod rtp_transceiver;
pub mod session_description;
pub mod video_frame;
pub mod video_source;
pub mod video_stream;
pub mod video_track;
pub mod yuv_helper;
use webrtc_sys::{rtc_error as sys_err, webrtc as sys_rtc};
use crate::{MediaType, RtcError, RtcErrorType};
impl From<sys_err::ffi::RtcErrorType> for RtcErrorType {
fn from(value: sys_err::ffi::RtcErrorType) -> Self {
match value {
sys_err::ffi::RtcErrorType::InvalidState => Self::InvalidState,
_ => Self::Internal,
}
}
}
impl From<sys_err::ffi::RtcError> for RtcError {
fn from(value: sys_err::ffi::RtcError) -> Self {
Self { error_type: value.error_type.into(), message: value.message }
}
}
impl From<MediaType> for sys_rtc::ffi::MediaType {
fn from(value: MediaType) -> Self {
match value {
MediaType::Audio => Self::Audio,
MediaType::Video => Self::Video,
MediaType::Data => Self::Data,
MediaType::Unsupported => Self::Unsupported,
}
}
}
@@ -0,0 +1,273 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Packet trailer support for end-to-end frame metadata propagation.
//!
//! This module provides functionality to embed user-supplied metadata
//! in encoded video frames as trailers. The timestamps/frameIDs are preserved
//! through the WebRTC pipeline and can be extracted on the receiver side.
//!
//! On the send side, user timestamps/frameIDs are stored in the handler's internal
//! map keyed by RTP timestamp. When the encoder produces a frame,
//! the transformer looks up the metadata via the frame's CaptureTime().
//!
//! On the receive side, extracted frame metadata is stored in an
//! internal map keyed by RTP timestamp. Decoded frames look up their
//! metadata via lookup_frame_metadata(rtp_timestamp).
use std::sync::Arc;
use cxx::SharedPtr;
use webrtc_sys::packet_trailer::ffi as sys_pt;
use crate::{
peer_connection_factory::PeerConnectionFactory, rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
};
/// Stage reached by a native local video frame in the publish pipeline.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PublishTimingStage {
/// The adapted raw frame was handed to WebRTC's encoder path.
EncoderUpload,
/// WebRTC produced an encoded frame for packetization.
EncoderOutput,
/// The encoded frame was handed back to WebRTC's packetizer.
WebrtcPacketize,
}
/// Stage reached by a native remote video frame in the subscribe pipeline.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SubscribeTimingStage {
/// WebRTC produced an encoded frame after RTP depacketization.
WebrtcReceive,
/// The encoded frame was handed to WebRTC's decoder.
DecoderUpload,
/// WebRTC produced a decoded frame for the native video sink.
DecoderOutput,
}
/// Timestamped native local video publish pipeline event.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PublishTimingEvent {
/// Publish pipeline stage reached by the frame.
pub stage: PublishTimingStage,
/// Wall-clock time when this stage was observed, in microseconds since the Unix epoch.
pub timestamp_us: u64,
/// User capture timestamp associated with this frame, in microseconds since the Unix epoch.
pub capture_timestamp_us: u64,
/// Optional application frame ID associated with this frame.
pub frame_id: Option<u32>,
}
/// Timestamped native remote video subscribe pipeline event.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SubscribeTimingEvent {
/// Subscribe pipeline stage reached by the frame.
pub stage: SubscribeTimingStage,
/// Wall-clock time when this stage was observed, in microseconds since the Unix epoch.
pub timestamp_us: u64,
/// User capture timestamp associated with this frame, in microseconds since the Unix epoch.
pub capture_timestamp_us: u64,
/// Optional application frame ID associated with this frame.
pub frame_id: Option<u32>,
}
/// Callback invoked for native local video publish timing events.
pub type PublishTimingObserver = Arc<dyn Fn(PublishTimingEvent) + Send + Sync + 'static>;
/// Callback invoked for native remote video subscribe timing events.
pub type SubscribeTimingObserver = Arc<dyn Fn(SubscribeTimingEvent) + Send + Sync + 'static>;
impl From<sys_pt::VideoPublishTimingStage> for PublishTimingStage {
fn from(stage: sys_pt::VideoPublishTimingStage) -> Self {
match stage {
sys_pt::VideoPublishTimingStage::EncoderUpload => Self::EncoderUpload,
sys_pt::VideoPublishTimingStage::EncoderOutput => Self::EncoderOutput,
sys_pt::VideoPublishTimingStage::WebrtcPacketize => Self::WebrtcPacketize,
_ => Self::WebrtcPacketize,
}
}
}
impl From<sys_pt::VideoPublishTimingEvent> for PublishTimingEvent {
fn from(event: sys_pt::VideoPublishTimingEvent) -> Self {
Self {
stage: event.stage.into(),
timestamp_us: event.timestamp_us,
capture_timestamp_us: event.capture_timestamp_us,
frame_id: (event.frame_id != 0).then_some(event.frame_id),
}
}
}
impl From<sys_pt::VideoSubscribeTimingStage> for SubscribeTimingStage {
fn from(stage: sys_pt::VideoSubscribeTimingStage) -> Self {
match stage {
sys_pt::VideoSubscribeTimingStage::WebrtcReceive => Self::WebrtcReceive,
sys_pt::VideoSubscribeTimingStage::DecoderUpload => Self::DecoderUpload,
sys_pt::VideoSubscribeTimingStage::DecoderOutput => Self::DecoderOutput,
_ => Self::DecoderOutput,
}
}
}
impl From<sys_pt::VideoSubscribeTimingEvent> for SubscribeTimingEvent {
fn from(event: sys_pt::VideoSubscribeTimingEvent) -> Self {
Self {
stage: event.stage.into(),
timestamp_us: event.timestamp_us,
capture_timestamp_us: event.capture_timestamp_us,
frame_id: (event.frame_id != 0).then_some(event.frame_id),
}
}
}
/// Handler for packet trailer embedding/extraction on RTP streams.
///
/// For sender side: Stores frame metadata keyed by capture timestamp
/// and embeds them as binary payload trailers on encoded frames before they
/// are sent. Use `store_frame_metadata()` to associate metadata with
/// a captured frame.
///
/// For receiver side: Extracts frame metadata from received frames
/// and makes them available for retrieval via `lookup_frame_metadata()`.
#[derive(Clone)]
pub struct PacketTrailerHandler {
sys_handle: SharedPtr<sys_pt::PacketTrailerHandler>,
}
impl PacketTrailerHandler {
/// Enable or disable timestamp embedding/extraction.
pub fn set_enabled(&self, enabled: bool) {
self.sys_handle.set_enabled(enabled);
}
/// Check if timestamp embedding/extraction is enabled.
pub fn enabled(&self) -> bool {
self.sys_handle.enabled()
}
/// Lookup the frame metadata for a given RTP timestamp (receiver side).
/// Returns `Some((user_timestamp, frame_id))` if found, `None` otherwise.
/// The entry is removed from the map after a successful lookup.
pub fn lookup_frame_metadata(&self, rtp_timestamp: u32) -> Option<(u64, u32)> {
let ts = self.sys_handle.lookup_timestamp(rtp_timestamp);
if ts != u64::MAX {
let frame_id = self.sys_handle.last_lookup_frame_id();
Some((ts, frame_id))
} else {
None
}
}
/// Store frame metadata for a given capture timestamp (sender side).
///
/// The `capture_timestamp_us` must be the TimestampAligner-adjusted
/// timestamp (as produced by `VideoTrackSource::on_captured_frame`),
/// NOT the original `timestamp_us` from the VideoFrame. The transformer
/// looks up the metadata by the frame's `CaptureTime()` which is
/// derived from the aligned value.
///
/// In normal usage this is called automatically by the C++ layer --
/// callers should set `user_timestamp` and `frame_id` on the
/// `VideoFrame` and let `capture_frame` / `on_captured_frame` handle
/// the rest.
pub fn store_frame_metadata(
&self,
capture_timestamp_us: i64,
user_timestamp: u64,
frame_id: u32,
) {
self.sys_handle.store_frame_metadata(capture_timestamp_us, user_timestamp, frame_id);
}
pub(crate) fn sys_handle(&self) -> SharedPtr<sys_pt::PacketTrailerHandler> {
self.sys_handle.clone()
}
/// Set the callback receiving sender-side publish timing events.
pub fn set_publish_timing_observer(&self, observer: Option<PublishTimingObserver>) {
if let Some(observer) = observer {
self.sys_handle.set_publish_timing_observer(Box::new(
webrtc_sys::packet_trailer::VideoPublishTimingObserverWrapper::new(Box::new(
move |event| observer(event.into()),
)),
));
} else {
self.sys_handle.clear_publish_timing_observer();
}
}
/// Set the callback receiving receiver-side subscribe timing events.
pub fn set_subscribe_timing_observer(&self, observer: Option<SubscribeTimingObserver>) {
if let Some(observer) = observer {
self.sys_handle.set_subscribe_timing_observer(Box::new(
webrtc_sys::packet_trailer::VideoSubscribeTimingObserverWrapper::new(Box::new(
move |event| observer(event.into()),
)),
));
} else {
self.sys_handle.clear_subscribe_timing_observer();
}
}
pub(crate) fn emit_subscribe_timing(
&self,
stage: SubscribeTimingStage,
capture_timestamp_us: u64,
frame_id: u32,
) {
let stage = match stage {
SubscribeTimingStage::WebrtcReceive => sys_pt::VideoSubscribeTimingStage::WebrtcReceive,
SubscribeTimingStage::DecoderUpload => sys_pt::VideoSubscribeTimingStage::DecoderUpload,
SubscribeTimingStage::DecoderOutput => sys_pt::VideoSubscribeTimingStage::DecoderOutput,
};
self.sys_handle.emit_subscribe_timing(stage, capture_timestamp_us, frame_id);
}
}
/// Create a sender-side packet trailer handler.
///
/// This handler will embed frame metadata into encoded frames before
/// they are packetized and sent. Use `store_frame_metadata()` to
/// associate metadata with a captured frame's capture timestamp.
pub fn create_sender_handler(
peer_factory: &PeerConnectionFactory,
sender: &RtpSender,
) -> PacketTrailerHandler {
PacketTrailerHandler {
sys_handle: sys_pt::new_packet_trailer_sender(
peer_factory.handle.sys_handle.clone(),
sender.handle.sys_handle.clone(),
),
}
}
/// Create a receiver-side packet trailer handler.
///
/// This handler will extract frame metadata from received frames
/// and store them in a map keyed by RTP timestamp. Use
/// `lookup_frame_metadata(rtp_timestamp)` to retrieve the metadata
/// for a specific decoded frame.
pub fn create_receiver_handler(
peer_factory: &PeerConnectionFactory,
receiver: &RtpReceiver,
) -> PacketTrailerHandler {
PacketTrailerHandler {
sys_handle: sys_pt::new_packet_trailer_receiver(
peer_factory.handle.sys_handle.clone(),
receiver.handle.sys_handle.clone(),
),
}
}
@@ -0,0 +1,619 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use cxx::SharedPtr;
use parking_lot::Mutex;
use tokio::sync::{mpsc, oneshot};
use webrtc_sys::{
data_channel as sys_dc, jsep as sys_jsep, peer_connection as sys_pc,
peer_connection_factory as sys_pcf, rtc_error as sys_err,
};
use crate::{
data_channel::{DataChannel, DataChannelInit},
ice_candidate::IceCandidate,
imp::{
data_channel as imp_dc, ice_candidate as imp_ic, media_stream as imp_ms,
media_stream_track as imp_mst, rtp_receiver as imp_rr, rtp_sender as imp_rs,
rtp_transceiver as imp_rt, session_description as imp_sdp,
},
media_stream::MediaStream,
media_stream_track::MediaStreamTrack,
peer_connection::{
AnswerOptions, IceCandidateError, IceConnectionState, IceGatheringState, OfferOptions,
OnConnectionChange, OnDataChannel, OnIceCandidate, OnIceCandidateError,
OnIceConnectionChange, OnIceGatheringChange, OnNegotiationNeeded, OnSignalingChange,
OnTrack, PeerConnectionState, SignalingState, TrackEvent,
},
peer_connection_factory::{
ContinualGatheringPolicy, IceServer, IceTransportsType, RtcConfiguration,
},
rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
rtp_transceiver::{RtpTransceiver, RtpTransceiverInit},
session_description::SessionDescription,
stats::RtcStats,
MediaType, RtcError, RtcErrorType,
};
impl From<OfferOptions> for sys_pc::ffi::RtcOfferAnswerOptions {
fn from(options: OfferOptions) -> Self {
Self {
ice_restart: options.ice_restart,
offer_to_receive_audio: options.offer_to_receive_audio as i32,
offer_to_receive_video: options.offer_to_receive_video as i32,
..Default::default()
}
}
}
impl From<AnswerOptions> for sys_pc::ffi::RtcOfferAnswerOptions {
fn from(_options: AnswerOptions) -> Self {
Self::default()
}
}
impl From<sys_pc::ffi::PeerConnectionState> for PeerConnectionState {
fn from(state: sys_pc::ffi::PeerConnectionState) -> Self {
match state {
sys_pc::ffi::PeerConnectionState::New => PeerConnectionState::New,
sys_pc::ffi::PeerConnectionState::Connecting => PeerConnectionState::Connecting,
sys_pc::ffi::PeerConnectionState::Connected => PeerConnectionState::Connected,
sys_pc::ffi::PeerConnectionState::Disconnected => PeerConnectionState::Disconnected,
sys_pc::ffi::PeerConnectionState::Failed => PeerConnectionState::Failed,
sys_pc::ffi::PeerConnectionState::Closed => PeerConnectionState::Closed,
_ => panic!("unknown PeerConnectionState"),
}
}
}
impl From<sys_pc::ffi::IceConnectionState> for IceConnectionState {
fn from(state: sys_pc::ffi::IceConnectionState) -> Self {
match state {
sys_pc::ffi::IceConnectionState::IceConnectionNew => IceConnectionState::New,
sys_pc::ffi::IceConnectionState::IceConnectionChecking => IceConnectionState::Checking,
sys_pc::ffi::IceConnectionState::IceConnectionConnected => {
IceConnectionState::Connected
}
sys_pc::ffi::IceConnectionState::IceConnectionCompleted => {
IceConnectionState::Completed
}
sys_pc::ffi::IceConnectionState::IceConnectionFailed => IceConnectionState::Failed,
sys_pc::ffi::IceConnectionState::IceConnectionDisconnected => {
IceConnectionState::Disconnected
}
sys_pc::ffi::IceConnectionState::IceConnectionClosed => IceConnectionState::Closed,
sys_pc::ffi::IceConnectionState::IceConnectionMax => IceConnectionState::Max,
_ => panic!("unknown IceConnectionState"),
}
}
}
impl From<sys_pc::ffi::IceGatheringState> for IceGatheringState {
fn from(state: sys_pc::ffi::IceGatheringState) -> Self {
match state {
sys_pc::ffi::IceGatheringState::IceGatheringNew => IceGatheringState::New,
sys_pc::ffi::IceGatheringState::IceGatheringGathering => IceGatheringState::Gathering,
sys_pc::ffi::IceGatheringState::IceGatheringComplete => IceGatheringState::Complete,
_ => panic!("unknown IceGatheringState"),
}
}
}
impl From<sys_pc::ffi::SignalingState> for SignalingState {
fn from(state: sys_pc::ffi::SignalingState) -> Self {
match state {
sys_pc::ffi::SignalingState::Stable => SignalingState::Stable,
sys_pc::ffi::SignalingState::HaveLocalOffer => SignalingState::HaveLocalOffer,
sys_pc::ffi::SignalingState::HaveRemoteOffer => SignalingState::HaveRemoteOffer,
sys_pc::ffi::SignalingState::HaveLocalPrAnswer => SignalingState::HaveLocalPrAnswer,
sys_pc::ffi::SignalingState::HaveRemotePrAnswer => SignalingState::HaveRemotePrAnswer,
sys_pc::ffi::SignalingState::Closed => SignalingState::Closed,
_ => panic!("unknown SignalingState"),
}
}
}
impl From<IceServer> for sys_pc::ffi::IceServer {
fn from(value: IceServer) -> Self {
sys_pc::ffi::IceServer {
urls: value.urls,
username: value.username,
password: value.password,
}
}
}
impl From<ContinualGatheringPolicy> for sys_pc::ffi::ContinualGatheringPolicy {
fn from(value: ContinualGatheringPolicy) -> Self {
match value {
ContinualGatheringPolicy::GatherOnce => {
sys_pc::ffi::ContinualGatheringPolicy::GatherOnce
}
ContinualGatheringPolicy::GatherContinually => {
sys_pc::ffi::ContinualGatheringPolicy::GatherContinually
}
}
}
}
impl From<IceTransportsType> for sys_pc::ffi::IceTransportsType {
fn from(value: IceTransportsType) -> Self {
match value {
IceTransportsType::Relay => sys_pc::ffi::IceTransportsType::Relay,
IceTransportsType::NoHost => sys_pc::ffi::IceTransportsType::NoHost,
IceTransportsType::All => sys_pc::ffi::IceTransportsType::All,
}
}
}
impl From<RtcConfiguration> for sys_pc::ffi::RtcConfiguration {
fn from(value: RtcConfiguration) -> Self {
Self {
ice_servers: value.ice_servers.into_iter().map(Into::into).collect(),
continual_gathering_policy: value.continual_gathering_policy.into(),
ice_transport_type: value.ice_transport_type.into(),
}
}
}
#[derive(Clone)]
pub struct PeerConnection {
observer: Arc<PeerObserver>,
pub(crate) sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
}
impl PeerConnection {
pub fn configure(
sys_handle: SharedPtr<sys_pc::ffi::PeerConnection>,
observer: Arc<PeerObserver>,
) -> Self {
Self { sys_handle, observer }
}
pub fn set_configuration(&self, config: RtcConfiguration) -> Result<(), RtcError> {
let res = self.sys_handle.set_configuration(config.into());
match res {
Ok(_) => Ok(()),
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
}
}
pub async fn create_offer(
&self,
options: OfferOptions,
) -> Result<SessionDescription, RtcError> {
let (tx, mut rx) = mpsc::channel::<Result<SessionDescription, RtcError>>(1);
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
type CtxType = mpsc::Sender<Result<SessionDescription, RtcError>>;
self.sys_handle.create_offer(
options.into(),
ctx,
|ctx, sdp| {
let tx = *ctx.0.downcast::<CtxType>().unwrap();
let _ = tx.blocking_send(Ok(SessionDescription {
handle: imp_sdp::SessionDescription { sys_handle: sdp },
}));
},
|ctx, error| {
let tx = *ctx.0.downcast::<CtxType>().unwrap();
let _ = tx.blocking_send(Err(error.into()));
},
);
rx.recv().await.unwrap()
}
pub async fn create_answer(
&self,
options: AnswerOptions,
) -> Result<SessionDescription, RtcError> {
let (tx, mut rx) = mpsc::channel::<Result<SessionDescription, RtcError>>(1);
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
type CtxType = mpsc::Sender<Result<SessionDescription, RtcError>>;
self.sys_handle.create_answer(
options.into(),
ctx,
|ctx, sdp| {
let tx = *ctx.0.downcast::<CtxType>().unwrap();
let _ = tx.blocking_send(Ok(SessionDescription {
handle: imp_sdp::SessionDescription { sys_handle: sdp },
}));
},
|ctx, error| {
let tx = *ctx.0.downcast::<CtxType>().unwrap();
let _ = tx.blocking_send(Err(error.into()));
},
);
rx.recv().await.unwrap()
}
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
self.sys_handle.set_local_description(desc.handle.sys_handle, ctx, |ctx, err| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
if err.ok() {
let _ = tx.send(Ok(()));
} else {
let _ = tx.send(Err(err.into()));
}
});
rx.await.unwrap()
}
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
self.sys_handle.set_remote_description(desc.handle.sys_handle, ctx, |ctx, err| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
if err.ok() {
let _ = tx.send(Ok(()));
} else {
let _ = tx.send(Err(err.into()));
}
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "set_remote_description cancelled".to_owned(),
})?
}
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
let (tx, rx) = oneshot::channel::<Result<(), RtcError>>();
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
self.sys_handle.add_ice_candidate(candidate.handle.sys_handle, ctx, |ctx, err| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<(), RtcError>>>().unwrap();
if err.ok() {
let _ = tx.send(Ok(()));
} else {
let _ = tx.send(Err(err.into()));
}
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "add_ice_candidate cancelled".to_owned(),
})?
}
pub fn create_data_channel(
&self,
label: &str,
init: DataChannelInit,
) -> Result<DataChannel, RtcError> {
let res = self.sys_handle.create_data_channel(label.to_string(), init.into());
match res {
Ok(sys_handle) => {
Ok(DataChannel { handle: imp_dc::DataChannel::configure(sys_handle) })
}
Err(e) => Err(unsafe { sys_err::ffi::RtcError::from(e.what()).into() }),
}
}
pub fn add_track<T: AsRef<str>>(
&self,
track: MediaStreamTrack,
stream_ids: &[T],
) -> Result<RtpSender, RtcError> {
let stream_ids = stream_ids.iter().map(|s| s.as_ref().to_owned()).collect();
let res = self.sys_handle.add_track(track.sys_handle(), &stream_ids);
match res {
Ok(sys_handle) => Ok(RtpSender { handle: imp_rs::RtpSender { sys_handle } }),
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
}
}
pub fn add_transceiver(
&self,
track: MediaStreamTrack,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
let res = self.sys_handle.add_transceiver(track.sys_handle(), init.into());
match res {
Ok(sys_handle) => Ok(RtpTransceiver { handle: imp_rt::RtpTransceiver { sys_handle } }),
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
}
}
pub fn add_transceiver_for_media(
&self,
media_type: MediaType,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
let res = self.sys_handle.add_transceiver_for_media(media_type.into(), init.into());
match res {
Ok(cxx_handle) => {
Ok(RtpTransceiver { handle: imp_rt::RtpTransceiver { sys_handle: cxx_handle } })
}
Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) },
}
}
pub fn restart_ice(&self) {
self.sys_handle.restart_ice();
}
pub fn close(&self) {
self.sys_handle.close();
}
pub fn connection_state(&self) -> PeerConnectionState {
self.sys_handle.connection_state().into()
}
pub fn ice_connection_state(&self) -> IceConnectionState {
self.sys_handle.ice_connection_state().into()
}
pub fn ice_gathering_state(&self) -> IceGatheringState {
self.sys_handle.ice_gathering_state().into()
}
pub fn signaling_state(&self) -> SignalingState {
self.sys_handle.signaling_state().into()
}
pub fn current_local_description(&self) -> Option<SessionDescription> {
let sdp = self.sys_handle.current_local_description();
if sdp.is_null() {
return None;
}
Some(SessionDescription { handle: imp_sdp::SessionDescription { sys_handle: sdp } })
}
pub fn current_remote_description(&self) -> Option<SessionDescription> {
let sdp = self.sys_handle.current_remote_description();
if sdp.is_null() {
return None;
}
Some(SessionDescription { handle: imp_sdp::SessionDescription { sys_handle: sdp } })
}
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
self.sys_handle
.remove_track(sender.handle.sys_handle)
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
let ctx = Box::new(sys_pc::PeerContext(Box::new(tx)));
self.sys_handle.get_stats(ctx, |ctx, stats| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
if stats.is_empty() {
let _ = tx.send(Ok(vec![]));
return;
}
// Unwrap because it should not happens
let vec = serde_json::from_str(&stats).unwrap();
let _ = tx.send(Ok(vec));
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "get_stats cancelled".to_owned(),
})?
}
pub fn senders(&self) -> Vec<RtpSender> {
self.sys_handle
.get_senders()
.into_iter()
.map(|sender| RtpSender { handle: imp_rs::RtpSender { sys_handle: sender.ptr } })
.collect()
}
pub fn receivers(&self) -> Vec<RtpReceiver> {
self.sys_handle
.get_receivers()
.into_iter()
.map(|receiver| RtpReceiver {
handle: imp_rr::RtpReceiver { sys_handle: receiver.ptr },
})
.collect()
}
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
self.sys_handle
.get_transceivers()
.into_iter()
.map(|transceiver| RtpTransceiver {
handle: imp_rt::RtpTransceiver { sys_handle: transceiver.ptr },
})
.collect()
}
pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
*self.observer.connection_change_handler.lock() = f;
}
pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
*self.observer.data_channel_handler.lock() = f;
}
pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
*self.observer.ice_candidate_handler.lock() = f;
}
pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
*self.observer.ice_candidate_error_handler.lock() = f;
}
pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
*self.observer.ice_connection_change_handler.lock() = f;
}
pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
*self.observer.ice_gathering_change_handler.lock() = f;
}
pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
*self.observer.negotiation_needed_handler.lock() = f;
}
pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
*self.observer.signaling_change_handler.lock() = f;
}
pub fn on_track(&self, f: Option<OnTrack>) {
*self.observer.track_handler.lock() = f;
}
}
#[derive(Default)]
pub struct PeerObserver {
pub connection_change_handler: Mutex<Option<OnConnectionChange>>,
pub data_channel_handler: Mutex<Option<OnDataChannel>>,
pub ice_candidate_handler: Mutex<Option<OnIceCandidate>>,
pub ice_candidate_error_handler: Mutex<Option<OnIceCandidateError>>,
pub ice_connection_change_handler: Mutex<Option<OnIceConnectionChange>>,
pub ice_gathering_change_handler: Mutex<Option<OnIceGatheringChange>>,
pub negotiation_needed_handler: Mutex<Option<OnNegotiationNeeded>>,
pub signaling_change_handler: Mutex<Option<OnSignalingChange>>,
pub track_handler: Mutex<Option<OnTrack>>,
}
impl sys_pcf::PeerConnectionObserver for PeerObserver {
fn on_signaling_change(&self, new_state: sys_pc::ffi::SignalingState) {
if let Some(f) = self.signaling_change_handler.lock().as_mut() {
f(new_state.into());
}
}
fn on_add_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
fn on_remove_stream(&self, _stream: SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>) {}
fn on_data_channel(&self, data_channel: SharedPtr<sys_dc::ffi::DataChannel>) {
if let Some(f) = self.data_channel_handler.lock().as_mut() {
f(DataChannel { handle: imp_dc::DataChannel::configure(data_channel) });
}
}
fn on_renegotiation_needed(&self) {}
fn on_negotiation_needed_event(&self, event: u32) {
if let Some(f) = self.negotiation_needed_handler.lock().as_mut() {
f(event);
}
}
fn on_ice_connection_change(&self, _new_state: sys_pc::ffi::IceConnectionState) {}
fn on_standardized_ice_connection_change(&self, new_state: sys_pc::ffi::IceConnectionState) {
if let Some(f) = self.ice_connection_change_handler.lock().as_mut() {
f(new_state.into());
}
}
fn on_connection_change(&self, new_state: sys_pc::ffi::PeerConnectionState) {
if let Some(f) = self.connection_change_handler.lock().as_mut() {
f(new_state.into());
}
}
fn on_ice_gathering_change(&self, new_state: sys_pc::ffi::IceGatheringState) {
if let Some(f) = self.ice_gathering_change_handler.lock().as_mut() {
f(new_state.into());
}
}
fn on_ice_candidate(&self, candidate: SharedPtr<sys_jsep::ffi::IceCandidate>) {
if let Some(f) = self.ice_candidate_handler.lock().as_mut() {
f(IceCandidate { handle: imp_ic::IceCandidate { sys_handle: candidate } });
}
}
fn on_ice_candidate_error(
&self,
address: String,
port: i32,
url: String,
error_code: i32,
error_text: String,
) {
if let Some(f) = self.ice_candidate_error_handler.lock().as_mut() {
f(IceCandidateError { address, port, url, error_code, error_text });
}
}
fn on_ice_candidates_removed(
&self,
_removed: Vec<SharedPtr<webrtc_sys::candidate::ffi::Candidate>>,
) {
}
fn on_ice_connection_receiving_change(&self, _receiving: bool) {}
fn on_ice_selected_candidate_pair_changed(
&self,
_event: sys_pcf::ffi::CandidatePairChangeEvent,
) {
}
fn on_add_track(
&self,
_receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>,
_streams: Vec<SharedPtr<webrtc_sys::media_stream::ffi::MediaStream>>,
) {
}
fn on_track(&self, transceiver: SharedPtr<webrtc_sys::rtp_transceiver::ffi::RtpTransceiver>) {
if let Some(f) = self.track_handler.lock().as_mut() {
let receiver = transceiver.receiver();
let streams = receiver.streams();
let track = receiver.track();
f(TrackEvent {
receiver: RtpReceiver { handle: imp_rr::RtpReceiver { sys_handle: receiver } },
streams: streams
.into_iter()
.map(|s| MediaStream { handle: imp_ms::MediaStream { sys_handle: s.ptr } })
.collect(),
track: imp_mst::new_media_stream_track(track),
transceiver: RtpTransceiver {
handle: imp_rt::RtpTransceiver { sys_handle: transceiver },
},
});
}
}
fn on_remove_track(&self, _receiver: SharedPtr<webrtc_sys::rtp_receiver::ffi::RtpReceiver>) {}
fn on_interesting_usage(&self, _usage_pattern: i32) {}
}
@@ -0,0 +1,361 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use cxx::{SharedPtr, UniquePtr};
use lazy_static::lazy_static;
use parking_lot::Mutex;
use webrtc_sys::{peer_connection_factory as sys_pcf, rtc_error as sys_err, webrtc as sys_rtc};
use crate::{
audio_source::native::NativeAudioSource,
audio_track::RtcAudioTrack,
imp::{audio_track as imp_at, peer_connection as imp_pc, video_track as imp_vt},
peer_connection::PeerConnection,
peer_connection_factory::RtcConfiguration,
rtp_parameters::RtpCapabilities,
video_source::native::NativeVideoSource,
video_track::RtcVideoTrack,
MediaType, RtcError,
};
lazy_static! {
static ref LOG_SINK: Mutex<Option<UniquePtr<sys_rtc::ffi::LogSink>>> = Default::default();
}
#[derive(Clone)]
pub struct PeerConnectionFactory {
pub(crate) sys_handle: SharedPtr<sys_pcf::ffi::PeerConnectionFactory>,
}
impl Default for PeerConnectionFactory {
fn default() -> Self {
let mut log_sink = LOG_SINK.lock();
if log_sink.is_none() {
*log_sink = Some(sys_rtc::ffi::new_log_sink(|msg, _| {
let msg = msg.strip_suffix("\r\n").or(msg.strip_suffix('\n')).unwrap_or(&msg);
log::debug!(target: "libwebrtc", "{}", msg);
}));
}
let sys_handle = sys_pcf::ffi::create_peer_connection_factory();
Self { sys_handle }
}
}
impl PeerConnectionFactory {
pub fn create_peer_connection(
&self,
config: RtcConfiguration,
) -> Result<PeerConnection, RtcError> {
let observer = Arc::new(imp_pc::PeerObserver::default());
let res = self.sys_handle.create_peer_connection(
config.into(),
Box::new(sys_pcf::PeerConnectionObserverWrapper::new(observer.clone())),
);
match res {
Ok(sys_handle) => Ok(PeerConnection {
handle: imp_pc::PeerConnection::configure(sys_handle, observer),
}),
Err(e) => Err(unsafe { sys_err::ffi::RtcError::from(e.what()).into() }),
}
}
pub fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
RtcVideoTrack {
handle: imp_vt::RtcVideoTrack::new(
self.sys_handle.create_video_track(label.to_string(), source.handle.sys_handle()),
),
}
}
pub fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
RtcAudioTrack {
handle: imp_at::RtcAudioTrack {
sys_handle: self
.sys_handle
.create_audio_track(label.to_string(), source.handle.sys_handle()),
},
}
}
/// Create an audio track that uses the Platform ADM for capture.
///
/// This requires that `enable_platform_adm()` was called first.
/// The track will capture audio from the selected recording device.
pub fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack {
RtcAudioTrack {
handle: imp_at::RtcAudioTrack {
sys_handle: self.sys_handle.create_device_audio_track(label.to_string()),
},
}
}
pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.sys_handle.rtp_sender_capabilities(media_type.into()).into()
}
pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.sys_handle.rtp_receiver_capabilities(media_type.into()).into()
}
// ===== Device Management Methods =====
/// Get the number of playout (output) devices
pub fn playout_devices(&self) -> i16 {
self.sys_handle.audio_device().playout_devices()
}
/// Get the number of recording (input) devices
pub fn recording_devices(&self) -> i16 {
self.sys_handle.audio_device().recording_devices()
}
/// Get the name of a playout device by index
pub fn playout_device_name(&self, index: u16) -> String {
self.sys_handle.audio_device().playout_device_name(index)
}
/// Get the name of a recording device by index
pub fn recording_device_name(&self, index: u16) -> String {
self.sys_handle.audio_device().recording_device_name(index)
}
/// Get the GUID of a playout device by index
/// The GUID is a platform-specific unique identifier that is stable across device hot-plug events.
pub fn playout_device_guid(&self, index: u16) -> String {
self.sys_handle.audio_device().playout_device_guid(index)
}
/// Get the GUID of a recording device by index
/// The GUID is a platform-specific unique identifier that is stable across device hot-plug events.
pub fn recording_device_guid(&self, index: u16) -> String {
self.sys_handle.audio_device().recording_device_guid(index)
}
/// Set the playout device by index
pub fn set_playout_device(&self, index: u16) -> bool {
self.sys_handle.audio_device().set_playout_device(index)
}
/// Set the recording device by index
pub fn set_recording_device(&self, index: u16) -> bool {
self.sys_handle.audio_device().set_recording_device(index)
}
/// Set the playout device by GUID
/// This is preferred over index as GUIDs are stable across device hot-plug events.
pub fn set_playout_device_by_guid(&self, guid: &str) -> bool {
self.sys_handle.audio_device().set_playout_device_by_guid(guid.to_string())
}
/// Set the recording device by GUID
/// This is preferred over index as GUIDs are stable across device hot-plug events.
pub fn set_recording_device_by_guid(&self, guid: &str) -> bool {
self.sys_handle.audio_device().set_recording_device_by_guid(guid.to_string())
}
/// Stop recording (clears initialized state, allowing device switch)
pub fn stop_recording(&self) -> bool {
self.sys_handle.audio_device().stop_recording()
}
/// Initialize recording
pub fn init_recording(&self) -> bool {
self.sys_handle.audio_device().init_recording()
}
/// Start recording
pub fn start_recording(&self) -> bool {
self.sys_handle.audio_device().start_recording()
}
/// Check if recording is initialized
pub fn recording_is_initialized(&self) -> bool {
self.sys_handle.audio_device().recording_is_initialized()
}
/// Stop playout (clears initialized state, allowing device switch)
pub fn stop_playout(&self) -> bool {
self.sys_handle.audio_device().stop_playout()
}
/// Initialize playout
pub fn init_playout(&self) -> bool {
self.sys_handle.audio_device().init_playout()
}
/// Start playout
pub fn start_playout(&self) -> bool {
self.sys_handle.audio_device().start_playout()
}
/// Check if playout is initialized
pub fn playout_is_initialized(&self) -> bool {
self.sys_handle.audio_device().playout_is_initialized()
}
// ===== Built-in Audio Processing Methods =====
// These control hardware AEC/AGC/NS on platforms that support it (iOS, some Android)
/// Check if built-in (hardware) AEC is available on this device.
///
/// Returns true on iOS (VPIO) and some Android devices.
/// Returns false on desktop platforms (macOS, Windows, Linux).
pub fn builtin_aec_is_available(&self) -> bool {
self.sys_handle.audio_device().builtin_aec_is_available()
}
/// Check if built-in (hardware) AGC is available on this device.
///
/// Returns true on iOS (VPIO) and some Android devices.
/// Returns false on desktop platforms (macOS, Windows, Linux).
pub fn builtin_agc_is_available(&self) -> bool {
self.sys_handle.audio_device().builtin_agc_is_available()
}
/// Check if built-in (hardware) NS is available on this device.
///
/// Returns true on iOS (VPIO) and some Android devices.
/// Returns false on desktop platforms (macOS, Windows, Linux).
pub fn builtin_ns_is_available(&self) -> bool {
self.sys_handle.audio_device().builtin_ns_is_available()
}
/// Enable or disable built-in (hardware) AEC.
///
/// When disabled on platforms that support it, WebRTC's software AEC
/// will be used instead.
pub fn enable_builtin_aec(&self, enable: bool) -> bool {
self.sys_handle.audio_device().enable_builtin_aec(enable)
}
/// Enable or disable built-in (hardware) AGC.
///
/// When disabled on platforms that support it, WebRTC's software AGC
/// will be used instead.
pub fn enable_builtin_agc(&self, enable: bool) -> bool {
self.sys_handle.audio_device().enable_builtin_agc(enable)
}
/// Enable or disable built-in (hardware) NS.
///
/// When disabled on platforms that support it, WebRTC's software NS
/// will be used instead.
pub fn enable_builtin_ns(&self, enable: bool) -> bool {
self.sys_handle.audio_device().enable_builtin_ns(enable)
}
/// Control whether ADM recording (microphone) is enabled.
///
/// When disabled, WebRTC's calls to InitRecording/StartRecording will be no-ops.
/// Use this when only using NativeAudioSource (no microphone capture needed).
/// This prevents the microphone from interfering with the audio pipeline.
pub fn set_adm_recording_enabled(&self, enabled: bool) {
self.sys_handle.audio_device().set_adm_recording_enabled(enabled)
}
/// Check if ADM recording (microphone) is enabled.
pub fn adm_recording_enabled(&self) -> bool {
self.sys_handle.audio_device().adm_recording_enabled()
}
/// Control whether ADM playout (speakers) is enabled.
///
/// When disabled (default), playout uses synthetic mode - remote audio is
/// delivered via FFI callbacks to the application (e.g., Unity AudioSource).
/// When enabled, remote audio plays through the platform speakers with AEC.
pub fn set_adm_playout_enabled(&self, enabled: bool) {
self.sys_handle.audio_device().set_adm_playout_enabled(enabled)
}
/// Check if ADM playout (speakers) is enabled.
pub fn adm_playout_enabled(&self) -> bool {
self.sys_handle.audio_device().adm_playout_enabled()
}
// ===== Platform ADM Lifecycle Management =====
/// Acquires a reference to the Platform ADM.
///
/// On first call, creates and initializes the Platform ADM. On subsequent
/// calls, just increments the reference count.
///
/// Returns true if Platform ADM is ready for use, false if initialization failed.
pub fn acquire_platform_adm(&self) -> bool {
self.sys_handle.audio_device().acquire_platform_adm()
}
/// Releases a reference to the Platform ADM.
///
/// When the reference count reaches zero, the Platform ADM is terminated
/// and the proxy returns to synthetic mode.
pub fn release_platform_adm(&self) {
self.sys_handle.audio_device().release_platform_adm()
}
/// Returns the current reference count for the Platform ADM.
pub fn platform_adm_ref_count(&self) -> i32 {
self.sys_handle.audio_device().platform_adm_ref_count()
}
/// Returns true if Platform ADM is currently active (ref_count > 0).
pub fn is_platform_adm_active(&self) -> bool {
self.sys_handle.audio_device().is_platform_adm_active()
}
/// Ensures the Platform ADM exists, retrying creation if an earlier
/// attempt failed (e.g. the OS audio stack was still starting up).
///
/// Returns true if the Platform ADM is available after the call.
pub fn ensure_platform_adm(&self) -> bool {
self.sys_handle.audio_device().ensure_platform_adm()
}
/// Returns true if the Platform ADM has been created and initialized.
/// Distinguishes "audio stack unavailable" from "zero audio devices".
pub fn platform_adm_available(&self) -> bool {
self.sys_handle.audio_device().platform_adm_available()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
static TEST_MUTEX: Mutex<()> = Mutex::new(());
#[tokio::test]
async fn test_peer_connection_factory_and_audio_device_controller_bridge() {
let _guard = TEST_MUTEX.lock().expect("test mutex poisoned");
let _ = env_logger::builder().is_test(true).try_init();
let factory = PeerConnectionFactory::default();
let source = NativeVideoSource::default();
let _track = factory.create_video_track("test", source);
let recording_count = factory.recording_devices();
let playout_count = factory.playout_devices();
assert!(recording_count >= 0);
assert!(playout_count >= 0);
let initial_recording = factory.adm_recording_enabled();
factory.set_adm_recording_enabled(!initial_recording);
assert_eq!(factory.adm_recording_enabled(), !initial_recording);
factory.set_adm_recording_enabled(initial_recording);
}
}
@@ -0,0 +1,263 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use webrtc_sys::{rtp_parameters as sys_rp, webrtc as sys_webrtc};
use crate::rtp_parameters::*;
impl From<sys_webrtc::ffi::Priority> for Priority {
fn from(value: sys_webrtc::ffi::Priority) -> Self {
match value {
sys_webrtc::ffi::Priority::VeryLow => Self::VeryLow,
sys_webrtc::ffi::Priority::Low => Self::Low,
sys_webrtc::ffi::Priority::Medium => Self::Medium,
sys_webrtc::ffi::Priority::High => Self::High,
_ => panic!("unknown Priority"),
}
}
}
impl From<sys_rp::ffi::RtpExtension> for RtpHeaderExtensionParameters {
fn from(value: sys_rp::ffi::RtpExtension) -> Self {
Self { uri: value.uri, id: value.id, encrypted: value.encrypt }
}
}
impl From<sys_rp::ffi::RtpParameters> for RtpParameters {
fn from(value: sys_rp::ffi::RtpParameters) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
rtcp: value.rtcp.into(),
}
}
}
impl From<sys_rp::ffi::RtpCodecParameters> for RtpCodecParameters {
fn from(value: sys_rp::ffi::RtpCodecParameters) -> Self {
Self {
mime_type: value.mime_type,
payload_type: value.payload_type as u8,
clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
channels: value.has_num_channels.then_some(value.num_channels as u16),
}
}
}
impl From<sys_rp::ffi::RtcpParameters> for RtcpParameters {
fn from(value: sys_rp::ffi::RtcpParameters) -> Self {
Self { cname: value.cname, reduced_size: value.reduced_size }
}
}
impl From<sys_rp::ffi::RtpEncodingParameters> for RtpEncodingParameters {
fn from(value: sys_rp::ffi::RtpEncodingParameters) -> Self {
Self {
active: value.active,
max_bitrate: value.has_max_bitrate_bps.then_some(value.max_bitrate_bps as u64),
max_framerate: value.has_max_framerate.then_some(value.max_framerate),
priority: value.network_priority.into(),
rid: value.rid,
scale_resolution_down_by: value
.has_scale_resolution_down_by
.then_some(value.scale_resolution_down_by),
scalability_mode: value.has_scalability_mode.then_some(value.scalability_mode),
}
}
}
impl From<sys_rp::ffi::RtpCodecCapability> for RtpCodecCapability {
fn from(value: sys_rp::ffi::RtpCodecCapability) -> Self {
Self {
channels: value.has_num_channels.then_some(value.num_channels as u16),
mime_type: value.mime_type,
clock_rate: value.has_clock_rate.then_some(value.clock_rate as u64),
sdp_fmtp_line: {
let parameters: Vec<String> = value
.parameters
.into_iter()
.map(|key_value| {
if !key_value.key.is_empty() {
format!("{}={}", key_value.key, key_value.value)
} else {
key_value.value
}
})
.collect();
if !parameters.is_empty() {
Some(parameters.join(";"))
} else {
None
}
},
}
}
}
impl From<sys_rp::ffi::RtpHeaderExtensionCapability> for RtpHeaderExtensionCapability {
fn from(value: sys_rp::ffi::RtpHeaderExtensionCapability) -> Self {
Self { direction: value.direction.into(), uri: value.uri }
}
}
impl From<sys_rp::ffi::RtpCapabilities> for RtpCapabilities {
fn from(value: sys_rp::ffi::RtpCapabilities) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
}
}
}
impl From<Priority> for sys_webrtc::ffi::Priority {
fn from(value: Priority) -> Self {
match value {
Priority::VeryLow => Self::VeryLow,
Priority::Low => Self::Low,
Priority::Medium => Self::Medium,
Priority::High => Self::High,
}
}
}
impl From<RtpHeaderExtensionParameters> for sys_rp::ffi::RtpExtension {
fn from(value: RtpHeaderExtensionParameters) -> Self {
Self { uri: value.uri, id: value.id, encrypt: value.encrypted }
}
}
impl From<RtpParameters> for sys_rp::ffi::RtpParameters {
fn from(value: RtpParameters) -> Self {
Self {
codecs: value.codecs.into_iter().map(Into::into).collect(),
header_extensions: value.header_extensions.into_iter().map(Into::into).collect(),
encodings: Vec::new(),
rtcp: value.rtcp.into(),
transaction_id: "".to_string(),
mid: "".to_string(),
has_degradation_preference: false,
degradation_preference: sys_rp::ffi::DegradationPreference::Balanced,
}
}
}
impl From<RtpCodecParameters> for sys_rp::ffi::RtpCodecParameters {
fn from(value: RtpCodecParameters) -> Self {
Self {
payload_type: value.payload_type as i32,
mime_type: value.mime_type,
has_clock_rate: value.clock_rate.is_some(),
clock_rate: value.clock_rate.unwrap_or_default() as i32,
has_num_channels: value.channels.is_some(),
num_channels: value.channels.unwrap_or_default() as i32,
name: "".to_string(),
kind: sys_rp::ffi::MediaType::Audio,
has_max_ptime: false,
max_ptime: 0,
has_ptime: false,
ptime: 0,
rtcp_feedback: Vec::new(),
parameters: Vec::new(),
}
}
}
impl From<RtcpParameters> for sys_rp::ffi::RtcpParameters {
fn from(value: RtcpParameters) -> Self {
Self {
cname: value.cname,
reduced_size: value.reduced_size,
has_ssrc: false,
ssrc: 0,
mux: false,
}
}
}
impl From<RtpEncodingParameters> for sys_rp::ffi::RtpEncodingParameters {
fn from(value: RtpEncodingParameters) -> Self {
Self {
active: value.active,
has_max_bitrate_bps: value.max_bitrate.is_some(),
max_bitrate_bps: value.max_bitrate.unwrap_or_default() as i32,
has_max_framerate: value.max_framerate.is_some(),
max_framerate: value.max_framerate.unwrap_or_default(),
network_priority: value.priority.into(),
rid: value.rid,
has_scale_resolution_down_by: value.scale_resolution_down_by.is_some(),
scale_resolution_down_by: value.scale_resolution_down_by.unwrap_or_default(),
adaptive_ptime: false,
bitrate_priority: sys_rp::DEFAULT_BITRATE_PRIORITY,
has_min_bitrate_bps: false,
min_bitrate_bps: 0,
has_num_temporal_layers: false,
num_temporal_layers: 0,
has_scalability_mode: value.scalability_mode.is_some(),
scalability_mode: value.scalability_mode.unwrap_or_default(),
has_ssrc: false,
ssrc: 0,
}
}
}
impl From<RtpCodecCapability> for sys_rp::ffi::RtpCodecCapability {
fn from(value: RtpCodecCapability) -> Self {
let mime_type: Vec<&str> = value.mime_type.split('/').collect();
let kind = match mime_type[0] {
"audio" => sys_webrtc::ffi::MediaType::Audio,
"video" => sys_webrtc::ffi::MediaType::Video,
_ => panic!("invalid media type"),
};
let name = mime_type[1].to_string();
Self {
name,
kind,
has_clock_rate: value.clock_rate.is_some(),
clock_rate: value.clock_rate.unwrap_or_default() as i32,
has_num_channels: value.channels.is_some(),
num_channels: value.channels.unwrap_or_default() as i32,
parameters: {
value
.sdp_fmtp_line
.map(|sdp_fmtp_line| {
sdp_fmtp_line
.split(';')
.map(|v| {
let key_value: Vec<&str> = v.split('=').collect();
if key_value.len() == 2 {
sys_rp::ffi::StringKeyValue {
key: key_value[0].to_string(),
value: key_value[1].to_string(),
}
} else {
sys_rp::ffi::StringKeyValue {
key: "".to_string(),
value: key_value[0].to_string(),
}
}
})
.collect()
})
.unwrap_or_default()
},
// Ignore
mime_type: String::default(), // !!
has_preferred_payload_type: false,
preferred_payload_type: 0,
rtcp_feedback: Vec::default(),
}
}
}
@@ -0,0 +1,65 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use tokio::sync::oneshot;
use webrtc_sys::rtp_receiver as sys_rr;
use crate::{
imp::media_stream_track::new_media_stream_track, media_stream_track::MediaStreamTrack,
rtp_parameters::RtpParameters, stats::RtcStats, RtcError, RtcErrorType,
};
#[derive(Clone)]
pub struct RtpReceiver {
pub(crate) sys_handle: SharedPtr<sys_rr::ffi::RtpReceiver>,
}
impl RtpReceiver {
pub fn track(&self) -> Option<MediaStreamTrack> {
let track_handle = self.sys_handle.track();
if track_handle.is_null() {
return None;
}
Some(new_media_stream_track(track_handle))
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
let ctx = Box::new(sys_rr::ReceiverContext(Box::new(tx)));
self.sys_handle.get_stats(ctx, |ctx, stats| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
if stats.is_empty() {
let _ = tx.send(Ok(vec![]));
return;
}
// Unwrap because it should not happens
let vec = serde_json::from_str(&stats).unwrap();
let _ = tx.send(Ok(vec));
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "get_stats cancelled".to_owned(),
})?
}
pub fn parameters(&self) -> RtpParameters {
self.sys_handle.get_parameters().into()
}
}
@@ -0,0 +1,83 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use tokio::sync::oneshot;
use webrtc_sys::{rtc_error as sys_err, rtp_sender as sys_rs};
use super::media_stream_track::new_media_stream_track;
use crate::{
media_stream_track::MediaStreamTrack, rtp_parameters::RtpParameters, stats::RtcStats, RtcError,
RtcErrorType,
};
#[derive(Clone)]
pub struct RtpSender {
pub(crate) sys_handle: SharedPtr<sys_rs::ffi::RtpSender>,
}
impl RtpSender {
pub fn track(&self) -> Option<MediaStreamTrack> {
let track_handle = self.sys_handle.track();
if track_handle.is_null() {
return None;
}
Some(new_media_stream_track(track_handle))
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
let (tx, rx) = oneshot::channel::<Result<Vec<RtcStats>, RtcError>>();
let ctx = Box::new(sys_rs::SenderContext(Box::new(tx)));
self.sys_handle.get_stats(ctx, |ctx, stats| {
let tx = ctx.0.downcast::<oneshot::Sender<Result<Vec<RtcStats>, RtcError>>>().unwrap();
if stats.is_empty() {
let _ = tx.send(Ok(vec![]));
return;
}
// Unwrap because it should not happens
let vec = serde_json::from_str(&stats).unwrap();
let _ = tx.send(Ok(vec));
});
rx.await.map_err(|_| RtcError {
error_type: RtcErrorType::Internal,
message: "get_stats cancelled".to_owned(),
})?
}
pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
if !self.sys_handle.set_track(track.map_or(SharedPtr::null(), |t| t.sys_handle())) {
return Err(RtcError {
error_type: RtcErrorType::InvalidState,
message: "Failed to set track".to_string(),
});
}
Ok(())
}
pub fn parameters(&self) -> RtpParameters {
self.sys_handle.get_parameters().into()
}
pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
self.sys_handle
.set_parameters(parameters.into())
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
}
}
@@ -0,0 +1,98 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::SharedPtr;
use webrtc_sys::{rtc_error as sys_err, rtp_transceiver as sys_rt, webrtc as sys_webrtc};
use crate::{
imp::{rtp_receiver::RtpReceiver, rtp_sender::RtpSender},
rtp_parameters::RtpCodecCapability,
rtp_receiver, rtp_sender,
rtp_transceiver::{RtpTransceiverDirection, RtpTransceiverInit},
RtcError,
};
impl From<sys_webrtc::ffi::RtpTransceiverDirection> for RtpTransceiverDirection {
fn from(value: sys_webrtc::ffi::RtpTransceiverDirection) -> Self {
match value {
sys_webrtc::ffi::RtpTransceiverDirection::SendRecv => Self::SendRecv,
sys_webrtc::ffi::RtpTransceiverDirection::SendOnly => Self::SendOnly,
sys_webrtc::ffi::RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
sys_webrtc::ffi::RtpTransceiverDirection::Inactive => Self::Inactive,
sys_webrtc::ffi::RtpTransceiverDirection::Stopped => Self::Stopped,
_ => panic!("unknown RtpTransceiverDirection"),
}
}
}
impl From<RtpTransceiverDirection> for sys_webrtc::ffi::RtpTransceiverDirection {
fn from(value: RtpTransceiverDirection) -> Self {
match value {
RtpTransceiverDirection::SendRecv => Self::SendRecv,
RtpTransceiverDirection::SendOnly => Self::SendOnly,
RtpTransceiverDirection::RecvOnly => Self::RecvOnly,
RtpTransceiverDirection::Inactive => Self::Inactive,
RtpTransceiverDirection::Stopped => Self::Stopped,
}
}
}
impl From<RtpTransceiverInit> for sys_rt::ffi::RtpTransceiverInit {
fn from(value: RtpTransceiverInit) -> Self {
Self {
direction: value.direction.into(),
stream_ids: value.stream_ids,
send_encodings: value.send_encodings.into_iter().map(Into::into).collect(),
}
}
}
#[derive(Clone)]
pub struct RtpTransceiver {
pub(crate) sys_handle: SharedPtr<sys_rt::ffi::RtpTransceiver>,
}
impl RtpTransceiver {
pub fn mid(&self) -> Option<String> {
self.sys_handle.mid().ok()
}
pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
self.sys_handle.current_direction().ok().map(Into::into)
}
pub fn direction(&self) -> RtpTransceiverDirection {
self.sys_handle.direction().into()
}
pub fn sender(&self) -> rtp_sender::RtpSender {
rtp_sender::RtpSender { handle: RtpSender { sys_handle: self.sys_handle.sender() } }
}
pub fn receiver(&self) -> rtp_receiver::RtpReceiver {
rtp_receiver::RtpReceiver { handle: RtpReceiver { sys_handle: self.sys_handle.receiver() } }
}
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
self.sys_handle
.set_codec_preferences(codecs.into_iter().map(Into::into).collect())
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
}
pub fn stop(&self) -> Result<(), RtcError> {
self.sys_handle
.stop_standard()
.map_err(|e| unsafe { sys_err::ffi::RtcError::from(e.what()).into() })
}
}
@@ -0,0 +1,82 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use webrtc_sys::jsep as sys_jsep;
use crate::session_description::{self, SdpParseError, SdpType};
impl From<sys_jsep::ffi::SdpType> for SdpType {
fn from(sdp_type: sys_jsep::ffi::SdpType) -> Self {
match sdp_type {
sys_jsep::ffi::SdpType::Offer => SdpType::Offer,
sys_jsep::ffi::SdpType::PrAnswer => SdpType::PrAnswer,
sys_jsep::ffi::SdpType::Answer => SdpType::Answer,
sys_jsep::ffi::SdpType::Rollback => SdpType::Rollback,
_ => panic!("unknown SdpType"),
}
}
}
impl From<SdpType> for sys_jsep::ffi::SdpType {
fn from(sdp_type: SdpType) -> Self {
match sdp_type {
SdpType::Offer => sys_jsep::ffi::SdpType::Offer,
SdpType::PrAnswer => sys_jsep::ffi::SdpType::PrAnswer,
SdpType::Answer => sys_jsep::ffi::SdpType::Answer,
SdpType::Rollback => sys_jsep::ffi::SdpType::Rollback,
}
}
}
impl From<sys_jsep::ffi::SdpParseError> for SdpParseError {
fn from(e: sys_jsep::ffi::SdpParseError) -> Self {
Self { line: e.line, description: e.description }
}
}
pub struct SessionDescription {
pub(crate) sys_handle: UniquePtr<sys_jsep::ffi::SessionDescription>,
}
impl SessionDescription {
pub fn parse(
sdp: &str,
sdp_type: SdpType,
) -> Result<session_description::SessionDescription, SdpParseError> {
let res = sys_jsep::ffi::create_session_description(sdp_type.into(), sdp.to_owned());
match res {
Ok(sys_handle) => Ok(session_description::SessionDescription {
handle: SessionDescription { sys_handle },
}),
Err(e) => Err(unsafe { sys_jsep::ffi::SdpParseError::from(e.what()).into() }),
}
}
pub fn sdp_type(&self) -> SdpType {
self.sys_handle.sdp_type().into()
}
}
impl ToString for SessionDescription {
fn to_string(&self) -> String {
self.sys_handle.stringify()
}
}
impl Clone for SessionDescription {
fn clone(&self) -> Self {
SessionDescription { sys_handle: self.sys_handle.clone() }
}
}
@@ -0,0 +1,956 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::slice;
use cxx::UniquePtr;
use webrtc_sys::{video_frame as vf_sys, video_frame_buffer as vfb_sys};
use super::yuv_helper;
use crate::video_frame::{self as vf, VideoFormatType, VideoRotation};
/// We don't use vf::VideoFrameBuffer trait for the types inside this module to avoid confusion
/// because directly using platform specific types is not valid (e.g user callback)
/// All the types inside this module are only used internally. For public types, see the top level
/// video_frame.rs
pub fn new_video_frame_buffer(
mut sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
) -> Box<dyn vf::VideoBuffer + Send + Sync> {
unsafe {
match sys_handle.buffer_type() {
vfb_sys::ffi::VideoFrameBufferType::Native => {
Box::new(vf::native::NativeBuffer { handle: NativeBuffer { sys_handle } })
}
vfb_sys::ffi::VideoFrameBufferType::I420 => Box::new(vf::I420Buffer {
handle: I420Buffer { sys_handle: sys_handle.pin_mut().get_i420() },
}),
vfb_sys::ffi::VideoFrameBufferType::I420A => Box::new(vf::I420ABuffer {
handle: I420ABuffer { sys_handle: sys_handle.pin_mut().get_i420a() },
}),
vfb_sys::ffi::VideoFrameBufferType::I422 => Box::new(vf::I422Buffer {
handle: I422Buffer { sys_handle: sys_handle.pin_mut().get_i422() },
}),
vfb_sys::ffi::VideoFrameBufferType::I444 => Box::new(vf::I444Buffer {
handle: I444Buffer { sys_handle: sys_handle.pin_mut().get_i444() },
}),
vfb_sys::ffi::VideoFrameBufferType::I010 => Box::new(vf::I010Buffer {
handle: I010Buffer { sys_handle: sys_handle.pin_mut().get_i010() },
}),
vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer {
handle: NV12Buffer { sys_handle: sys_handle.pin_mut().get_nv12() },
}),
_ => unreachable!(),
}
}
}
impl From<vf_sys::ffi::VideoRotation> for VideoRotation {
fn from(rotation: vf_sys::ffi::VideoRotation) -> Self {
match rotation {
vf_sys::ffi::VideoRotation::VideoRotation0 => Self::VideoRotation0,
vf_sys::ffi::VideoRotation::VideoRotation90 => Self::VideoRotation90,
vf_sys::ffi::VideoRotation::VideoRotation180 => Self::VideoRotation180,
vf_sys::ffi::VideoRotation::VideoRotation270 => Self::VideoRotation270,
_ => panic!("invalid VideoRotation"),
}
}
}
impl From<VideoRotation> for vf_sys::ffi::VideoRotation {
fn from(rotation: VideoRotation) -> Self {
match rotation {
VideoRotation::VideoRotation0 => Self::VideoRotation0,
VideoRotation::VideoRotation90 => Self::VideoRotation90,
VideoRotation::VideoRotation180 => Self::VideoRotation180,
VideoRotation::VideoRotation270 => Self::VideoRotation270,
}
}
}
macro_rules! recursive_cast {
($ptr:expr $(, $fnc:ident)*) => {
{
let ptr = $ptr;
$(
let ptr = vfb_sys::ffi::$fnc(ptr);
)*
ptr
}
};
}
pub struct NativeBuffer {
sys_handle: UniquePtr<vfb_sys::ffi::VideoFrameBuffer>,
}
pub struct I420Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I420Buffer>,
}
pub struct I420ABuffer {
sys_handle: UniquePtr<vfb_sys::ffi::I420ABuffer>,
}
pub struct I422Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I422Buffer>,
}
pub struct I444Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I444Buffer>,
}
pub struct I010Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::I010Buffer>,
}
pub struct NV12Buffer {
sys_handle: UniquePtr<vfb_sys::ffi::NV12Buffer>,
}
macro_rules! impl_to_argb {
(I420Buffer [$($variant:ident: $fnc:ident),+], $format:ident, $self:ident, $dst:ident, $dst_stride:ident, $dst_width:ident, $dst_height:ident) => {
match $format {
$(
VideoFormatType::$variant => {
let (data_y, data_u, data_v) = $self.data();
yuv_helper::$fnc(
data_y,
$self.stride_y(),
data_u,
$self.stride_u(),
data_v,
$self.stride_v(),
$dst,
$dst_stride,
$dst_width,
$dst_height,
)
}
)+
}
};
(I420ABuffer) => {
todo!();
}
}
#[allow(unused_unsafe)]
impl NativeBuffer {
pub fn from_fluxer_d3d11_texture(
handle: u64,
width: u32,
height: u32,
dxgi_format: u32,
) -> Option<vf::native::NativeBuffer> {
let sys_handle = vfb_sys::ffi::new_fluxer_d3d11_texture_buffer(
handle,
width,
height,
dxgi_format,
);
if sys_handle.is_null() {
return None;
}
Some(vf::native::NativeBuffer {
handle: NativeBuffer { sys_handle },
})
}
#[allow(clippy::too_many_arguments)]
pub fn from_fluxer_dmabuf_texture(
fds: [i32; 4],
plane_count: u32,
width: u32,
height: u32,
drm_format: u32,
modifier: u64,
strides: [u32; 4],
offsets: [u32; 4],
device_uuid_hi: u64,
device_uuid_lo: u64,
) -> Option<vf::native::NativeBuffer> {
let sys_handle = vfb_sys::ffi::new_fluxer_dmabuf_texture_buffer(
fds[0],
fds[1],
fds[2],
fds[3],
plane_count,
width,
height,
drm_format,
modifier,
strides[0],
strides[1],
strides[2],
strides[3],
offsets[0],
offsets[1],
offsets[2],
offsets[3],
device_uuid_hi,
device_uuid_lo,
);
if sys_handle.is_null() {
return None;
}
Some(vf::native::NativeBuffer {
handle: NativeBuffer { sys_handle },
})
}
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub unsafe fn from_cv_pixel_buffer(
cv_pixel_buffer: *mut std::ffi::c_void,
) -> vf::native::NativeBuffer {
vf::native::NativeBuffer {
handle: NativeBuffer {
sys_handle: vfb_sys::ffi::new_native_buffer_from_platform_image_buffer(
cv_pixel_buffer as *mut _,
),
},
}
}
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void {
unsafe { vfb_sys::ffi::native_buffer_to_platform_image_buffer(&self.sys_handle) as *mut _ }
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
&self.sys_handle
}
pub fn width(&self) -> u32 {
self.sys_handle.width()
}
pub fn height(&self) -> u32 {
self.sys_handle.height()
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer { sys_handle: unsafe { self.sys_handle.to_i420() } }
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
}
impl I420Buffer {
pub fn new(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> vf::I420Buffer {
vf::I420Buffer {
handle: I420Buffer {
sys_handle: vfb_sys::ffi::new_i420_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_u.try_into().unwrap(),
stride_v.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
// We make a copy of the buffer because internally, when calling ToI420()
// if the buffer is of type I420, libwebrtc will reuse the same underlying pointer
// for the new created type
let copy = vfb_sys::ffi::copy_i420_buffer(&self.sys_handle);
let ptr = recursive_cast!(&*copy, i420_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
impl_to_argb!(
I420Buffer
[
ARGB: i420_to_argb,
BGRA: i420_to_bgra,
ABGR: i420_to_abgr,
RGBA: i420_to_rgba
],
format, self, dst, dst_stride, dst_width, dst_height
)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420_to_yuv8);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420Buffer {
vf::I420Buffer {
handle: I420Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl I420ABuffer {
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn stride_a(&self) -> u32 {
self.sys_handle.stride_a()
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, i420a_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i420a_to_yuv8);
let chroma_height = (self.height() + 1) / 2;
let data_a = self.sys_handle.data_a();
let has_data_a = !data_a.is_null();
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * chroma_height) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * chroma_height) as usize),
has_data_a.then_some(slice::from_raw_parts(
data_a,
(self.stride_a() * self.height()) as usize,
)),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I420ABuffer {
vf::I420ABuffer {
handle: I420ABuffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl I422Buffer {
pub fn new(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> vf::I422Buffer {
vf::I422Buffer {
handle: I422Buffer {
sys_handle: vfb_sys::ffi::new_i422_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_u.try_into().unwrap(),
stride_v.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i422_to_yuv8);
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I422Buffer {
vf::I422Buffer {
handle: I422Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl I444Buffer {
pub fn new(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> vf::I444Buffer {
vf::I444Buffer {
handle: I444Buffer {
sys_handle: vfb_sys::ffi::new_i444_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_u.try_into().unwrap(),
stride_v.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8, yuv8_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i444_to_yuv8);
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_u(), (self.stride_u() * self.height()) as usize),
slice::from_raw_parts((*ptr).data_v(), (self.stride_v() * self.height()) as usize),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I444Buffer {
vf::I444Buffer {
handle: I444Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl I010Buffer {
pub fn new(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> vf::I010Buffer {
vf::I010Buffer {
handle: I010Buffer {
sys_handle: vfb_sys::ffi::new_i010_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_u.try_into().unwrap(),
stride_v.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe { &*recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb) }
}
pub fn width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_y()
}
}
pub fn stride_u(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_u()
}
}
pub fn stride_v(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv);
(*ptr).stride_v()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, i010_to_yuv16b, yuv16b_to_yuv, yuv_to_vfb);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, i010_to_yuv16b);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts(
(*ptr).data_y(),
(self.stride_y() * self.height()) as usize / 2,
),
slice::from_raw_parts(
(*ptr).data_u(),
(self.stride_u() * chroma_height) as usize / 2,
),
slice::from_raw_parts(
(*ptr).data_v(),
(self.stride_v() * chroma_height) as usize / 2,
),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::I010Buffer {
vf::I010Buffer {
handle: I010Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
impl NV12Buffer {
pub fn new(width: u32, height: u32, stride_y: u32, stride_uv: u32) -> vf::NV12Buffer {
vf::NV12Buffer {
handle: NV12Buffer {
sys_handle: vfb_sys::ffi::new_nv12_buffer(
width.try_into().unwrap(),
height.try_into().unwrap(),
stride_y.try_into().unwrap(),
stride_uv.try_into().unwrap(),
),
},
}
}
pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
unsafe {
&*recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb)
}
}
pub fn width(&self) -> u32 {
unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb);
(*ptr).width()
}
}
pub fn height(&self) -> u32 {
unsafe {
let ptr =
recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv, biyuv_to_vfb);
(*ptr).height()
}
}
pub fn chroma_width(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).chroma_width()
}
}
pub fn chroma_height(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).chroma_height()
}
}
pub fn stride_y(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).stride_y()
}
}
pub fn stride_uv(&self) -> u32 {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8, biyuv8_to_biyuv);
(*ptr).stride_uv()
}
}
pub fn to_i420(&self) -> I420Buffer {
I420Buffer {
sys_handle: unsafe {
let ptr = recursive_cast!(
&*self.sys_handle,
nv12_to_biyuv8,
biyuv8_to_biyuv,
biyuv_to_vfb
);
(*ptr).to_i420()
},
}
}
pub fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_i420().to_argb(format, dst, dst_stride, dst_width, dst_height)
}
pub fn data(&self) -> (&[u8], &[u8]) {
unsafe {
let ptr = recursive_cast!(&*self.sys_handle, nv12_to_biyuv8);
let chroma_height = (self.height() + 1) / 2;
(
slice::from_raw_parts((*ptr).data_y(), (self.stride_y() * self.height()) as usize),
slice::from_raw_parts(
(*ptr).data_uv(),
(self.stride_uv() * chroma_height) as usize,
),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> vf::NV12Buffer {
vf::NV12Buffer {
handle: NV12Buffer {
sys_handle: self.sys_handle.pin_mut().scale(scaled_width, scaled_height),
},
}
}
}
@@ -0,0 +1,147 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
sync::Arc,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use cxx::SharedPtr;
use livekit_runtime::interval;
use parking_lot::Mutex;
use webrtc_sys::{video_frame as vf_sys, video_frame::ffi::VideoRotation, video_track as vt_sys};
use crate::{
native::packet_trailer::PacketTrailerHandler,
video_frame::{I420Buffer, VideoBuffer, VideoFrame},
video_source::VideoResolution,
};
impl From<vt_sys::ffi::VideoResolution> for VideoResolution {
fn from(res: vt_sys::ffi::VideoResolution) -> Self {
Self { width: res.width, height: res.height }
}
}
impl From<VideoResolution> for vt_sys::ffi::VideoResolution {
fn from(res: VideoResolution) -> Self {
Self { width: res.width, height: res.height }
}
}
#[derive(Clone)]
pub struct NativeVideoSource {
sys_handle: SharedPtr<vt_sys::ffi::VideoTrackSource>,
inner: Arc<Mutex<VideoSourceInner>>,
}
struct VideoSourceInner {
captured_frames: usize,
}
impl NativeVideoSource {
pub fn new(resolution: VideoResolution, is_screencast: bool) -> NativeVideoSource {
let source = Self {
sys_handle: vt_sys::ffi::new_video_track_source(
&vt_sys::ffi::VideoResolution::from(resolution.clone()),
is_screencast,
),
inner: Arc::new(Mutex::new(VideoSourceInner { captured_frames: 0 })),
};
livekit_runtime::spawn({
let source = source.clone();
let i420 = I420Buffer::new(resolution.width, resolution.height);
async move {
let mut interval = interval(Duration::from_millis(100)); // 10 fps
loop {
interval.tick().await;
let inner = source.inner.lock();
if inner.captured_frames > 0 {
break;
}
let mut builder = vf_sys::ffi::new_video_frame_builder();
builder.pin_mut().set_rotation(VideoRotation::VideoRotation0);
builder.pin_mut().set_video_frame_buffer(i420.as_ref().sys_handle());
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
builder.pin_mut().set_timestamp_us(now.as_micros() as i64);
source.sys_handle.on_captured_frame(
&builder.pin_mut().build(),
&vt_sys::ffi::FrameMetadata {
has_packet_trailer: false,
user_timestamp: 0,
frame_id: 0,
},
);
}
}
});
source
}
pub fn sys_handle(&self) -> SharedPtr<vt_sys::ffi::VideoTrackSource> {
self.sys_handle.clone()
}
pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {
let mut builder = vf_sys::ffi::new_video_frame_builder();
builder.pin_mut().set_rotation(frame.rotation.into());
builder.pin_mut().set_video_frame_buffer(frame.buffer.as_ref().sys_handle());
let capture_ts = if frame.timestamp_us == 0 {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
now.as_micros() as i64
} else {
frame.timestamp_us
};
builder.pin_mut().set_timestamp_us(capture_ts);
let (has_trailer, user_ts, fid) = match frame.frame_metadata {
Some(meta) => (true, meta.user_timestamp.unwrap_or(0), meta.frame_id.unwrap_or(0)),
None => (false, 0, 0),
};
self.inner.lock().captured_frames += 1;
self.sys_handle.on_captured_frame(
&builder.pin_mut().build(),
&vt_sys::ffi::FrameMetadata {
has_packet_trailer: has_trailer,
user_timestamp: user_ts,
frame_id: fid,
},
);
}
/// Set the packet trailer handler used by this source.
///
/// When set, any frame captured with a `user_timestamp` value will
/// automatically have its timestamp stored in the handler so the
/// `PacketTrailerTransformer` can embed it into the encoded frame.
/// The handler is set on the C++ VideoTrackSource so it has access to
/// the TimestampAligner-adjusted capture timestamp for correct keying.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.sys_handle.set_packet_trailer_handler(handler.sys_handle());
}
pub fn video_resolution(&self) -> VideoResolution {
self.sys_handle.video_resolution().into()
}
}
@@ -0,0 +1,350 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
collections::VecDeque,
pin::Pin,
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
},
task::{Context, Poll, Waker},
};
use cxx::{SharedPtr, UniquePtr};
use livekit_runtime::Stream;
use parking_lot::Mutex;
use rtrb::{Consumer, Producer, PushError, RingBuffer};
use webrtc_sys::video_track as sys_vt;
use super::{packet_trailer::SubscribeTimingStage, video_frame::new_video_frame_buffer};
use crate::{
native::packet_trailer::PacketTrailerHandler,
video_frame::{BoxVideoFrame, FrameMetadata, VideoFrame},
video_track::RtcVideoTrack,
};
pub struct NativeVideoStream {
native_sink: SharedPtr<sys_vt::ffi::NativeVideoSink>,
observer: Arc<VideoTrackObserver>,
video_track: RtcVideoTrack,
frame_queue: Arc<VideoFrameQueue>,
}
impl NativeVideoStream {
pub fn new(video_track: RtcVideoTrack, queue_size_frames: Option<usize>) -> Self {
let frame_queue = Arc::new(VideoFrameQueue::new(queue_size_frames));
// Auto-wire the packet trailer handler from the track if one is set.
let handler = video_track.handle.packet_trailer_handler();
let observer = Arc::new(VideoTrackObserver {
frame_queue: frame_queue.clone(),
packet_trailer_handler: parking_lot::Mutex::new(handler),
});
let native_sink = sys_vt::ffi::new_native_video_sink(Box::new(
sys_vt::VideoSinkWrapper::new(observer.clone()),
));
let video = unsafe { sys_vt::ffi::media_to_video(video_track.sys_handle()) };
video.add_sink(&native_sink);
Self { native_sink, observer, video_track, frame_queue }
}
/// Set the packet trailer handler for this stream.
///
/// When set, each frame produced by this stream will have its
/// `user_timestamp` field populated from the handler's receive
/// map (looked up by RTP timestamp).
///
/// Note: If the handler was already set on the `RtcVideoTrack` before
/// creating this stream, it is automatically wired up. This method is
/// only needed if you want to override or set the handler after
/// construction.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
*self.observer.packet_trailer_handler.lock() = Some(handler);
}
pub fn track(&self) -> RtcVideoTrack {
self.video_track.clone()
}
pub fn close(&mut self) {
let video = unsafe { sys_vt::ffi::media_to_video(self.video_track.sys_handle()) };
video.remove_sink(&self.native_sink);
self.frame_queue.close();
}
}
impl Drop for NativeVideoStream {
fn drop(&mut self) {
self.close();
}
}
impl Stream for NativeVideoStream {
type Item = BoxVideoFrame;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
self.frame_queue.poll_recv(cx)
}
}
struct VideoTrackObserver {
frame_queue: Arc<VideoFrameQueue>,
packet_trailer_handler: parking_lot::Mutex<Option<PacketTrailerHandler>>,
}
impl sys_vt::VideoSink for VideoTrackObserver {
fn on_frame(&self, frame: UniquePtr<webrtc_sys::video_frame::ffi::VideoFrame>) {
let rtp_timestamp = frame.timestamp();
let packet_trailer_handler = self.packet_trailer_handler.lock().clone();
let frame_metadata = packet_trailer_handler
.as_ref()
.and_then(|handler| {
handler.lookup_frame_metadata(rtp_timestamp).map(|(ts, fid)| {
handler.emit_subscribe_timing(SubscribeTimingStage::DecoderOutput, ts, fid);
(ts, fid)
})
})
.map(|(ts, fid)| FrameMetadata {
user_timestamp: Some(ts),
frame_id: if fid != 0 { Some(fid) } else { None },
});
self.frame_queue.push(VideoFrame {
rotation: frame.rotation().into(),
timestamp_us: frame.timestamp_us(),
frame_metadata,
buffer: new_video_frame_buffer(unsafe { frame.video_frame_buffer() }),
});
}
fn on_discarded_frame(&self) {}
fn on_constraints_changed(&self, _constraints: sys_vt::ffi::VideoTrackSourceConstraints) {}
}
struct VideoFrameQueue {
kind: VideoFrameQueueKind,
closed: AtomicBool,
dropped_frames: AtomicU64,
waker: Mutex<Option<Waker>>,
}
enum VideoFrameQueueKind {
Bounded(BoundedVideoFrameQueue),
Unbounded(UnboundedVideoFrameQueue),
}
struct BoundedVideoFrameQueue {
producer: Mutex<Producer<BoxVideoFrame>>,
consumer: Mutex<Consumer<BoxVideoFrame>>,
}
struct UnboundedVideoFrameQueue {
frames: Mutex<VecDeque<BoxVideoFrame>>,
}
impl VideoFrameQueue {
fn new(capacity: Option<usize>) -> Self {
let kind = match capacity.filter(|capacity| *capacity > 0) {
Some(capacity) => {
let (producer, consumer) = RingBuffer::new(capacity);
VideoFrameQueueKind::Bounded(BoundedVideoFrameQueue {
producer: Mutex::new(producer),
consumer: Mutex::new(consumer),
})
}
None => VideoFrameQueueKind::Unbounded(UnboundedVideoFrameQueue {
frames: Mutex::new(VecDeque::new()),
}),
};
Self {
kind,
closed: AtomicBool::new(false),
dropped_frames: AtomicU64::new(0),
waker: Mutex::new(None),
}
}
fn push(&self, frame: BoxVideoFrame) {
if self.closed.load(Ordering::Acquire) {
return;
}
match &self.kind {
VideoFrameQueueKind::Bounded(queue) => self.push_bounded(queue, frame),
VideoFrameQueueKind::Unbounded(queue) => {
queue.frames.lock().push_back(frame);
}
}
self.wake_receiver();
}
fn push_bounded(&self, queue: &BoundedVideoFrameQueue, mut frame: BoxVideoFrame) {
loop {
let push_result = queue.producer.lock().push(frame);
match push_result {
Ok(()) => return,
Err(PushError::Full(returned_frame)) => {
frame = returned_frame;
let dropped = queue.consumer.lock().pop().is_ok();
if dropped {
self.record_drop();
} else {
return;
}
}
}
}
}
fn close(&self) {
self.closed.store(true, Ordering::Release);
self.wake_receiver();
match &self.kind {
VideoFrameQueueKind::Bounded(queue) => {
let mut consumer = queue.consumer.lock();
while consumer.pop().is_ok() {}
}
VideoFrameQueueKind::Unbounded(queue) => {
queue.frames.lock().clear();
}
}
}
fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Option<BoxVideoFrame>> {
if let Some(frame) = self.try_pop() {
return Poll::Ready(Some(frame));
}
if self.closed.load(Ordering::Acquire) {
return Poll::Ready(None);
}
*self.waker.lock() = Some(cx.waker().clone());
if let Some(frame) = self.try_pop() {
self.waker.lock().take();
Poll::Ready(Some(frame))
} else if self.closed.load(Ordering::Acquire) {
Poll::Ready(None)
} else {
Poll::Pending
}
}
fn try_pop(&self) -> Option<BoxVideoFrame> {
match &self.kind {
VideoFrameQueueKind::Bounded(queue) => queue.consumer.lock().pop().ok(),
VideoFrameQueueKind::Unbounded(queue) => queue.frames.lock().pop_front(),
}
}
fn wake_receiver(&self) {
let waker = self.waker.lock().take();
if let Some(waker) = waker {
waker.wake();
}
}
fn record_drop(&self) {
let dropped_frames = self.dropped_frames.fetch_add(1, Ordering::Relaxed) + 1;
if dropped_frames == 1 || dropped_frames % 100 == 0 {
log::warn!(
"native video stream queue overflow; dropped {} queued frames",
dropped_frames
);
}
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::Ordering;
use super::VideoFrameQueue;
use crate::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation};
fn test_frame(timestamp_us: i64) -> BoxVideoFrame {
VideoFrame {
rotation: VideoRotation::VideoRotation0,
timestamp_us,
frame_metadata: None,
buffer: Box::new(I420Buffer::new(2, 2)),
}
}
fn pop_timestamp(queue: &VideoFrameQueue) -> Option<i64> {
queue.try_pop().map(|frame| frame.timestamp_us)
}
#[test]
fn bounded_queue_preserves_fifo_order_under_capacity() {
let queue = VideoFrameQueue::new(Some(3));
queue.push(test_frame(1));
queue.push(test_frame(2));
queue.push(test_frame(3));
assert_eq!(pop_timestamp(&queue), Some(1));
assert_eq!(pop_timestamp(&queue), Some(2));
assert_eq!(pop_timestamp(&queue), Some(3));
assert_eq!(pop_timestamp(&queue), None);
}
#[test]
fn bounded_queue_drops_oldest_when_full() {
let queue = VideoFrameQueue::new(Some(2));
queue.push(test_frame(1));
queue.push(test_frame(2));
queue.push(test_frame(3));
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 1);
assert_eq!(pop_timestamp(&queue), Some(2));
assert_eq!(pop_timestamp(&queue), Some(3));
assert_eq!(pop_timestamp(&queue), None);
}
#[test]
fn unbounded_queue_retains_all_frames() {
let queue = VideoFrameQueue::new(None);
for timestamp_us in 1..=4 {
queue.push(test_frame(timestamp_us));
}
for timestamp_us in 1..=4 {
assert_eq!(pop_timestamp(&queue), Some(timestamp_us));
}
assert_eq!(pop_timestamp(&queue), None);
assert_eq!(queue.dropped_frames.load(Ordering::Relaxed), 0);
}
#[test]
fn close_clears_buffer_and_rejects_future_pushes() {
let queue = VideoFrameQueue::new(Some(2));
queue.push(test_frame(1));
queue.close();
queue.push(test_frame(2));
assert_eq!(pop_timestamp(&queue), None);
}
}
@@ -0,0 +1,56 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use cxx::SharedPtr;
use parking_lot::Mutex;
use sys_vt::ffi::video_to_media;
use webrtc_sys::video_track as sys_vt;
use super::media_stream_track::impl_media_stream_track;
use super::packet_trailer::PacketTrailerHandler;
use crate::media_stream_track::RtcTrackState;
#[derive(Clone)]
pub struct RtcVideoTrack {
pub(crate) sys_handle: SharedPtr<sys_vt::ffi::VideoTrack>,
packet_trailer_handler: Arc<Mutex<Option<PacketTrailerHandler>>>,
}
impl RtcVideoTrack {
impl_media_stream_track!(video_to_media);
pub(crate) fn new(sys_handle: SharedPtr<sys_vt::ffi::VideoTrack>) -> Self {
Self { sys_handle, packet_trailer_handler: Arc::new(Mutex::new(None)) }
}
pub fn sys_handle(&self) -> SharedPtr<sys_vt::ffi::MediaStreamTrack> {
video_to_media(self.sys_handle.clone())
}
/// Set the packet trailer handler for this track.
///
/// When set, any `NativeVideoStream` created from this track will
/// automatically use this handler to populate `user_timestamp`
/// on each decoded frame.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.packet_trailer_handler.lock().replace(handler);
}
/// Get the packet trailer handler, if one has been set.
pub fn packet_trailer_handler(&self) -> Option<PacketTrailerHandler> {
self.packet_trailer_handler.lock().clone()
}
}
@@ -0,0 +1,868 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(clippy::too_many_arguments)]
use webrtc_sys::yuv_helper as yuv_sys;
fn argb_assert_safety(src: &[u8], src_stride: u32, _width: i32, height: i32) {
let height_abs = height.unsigned_abs();
let min = (src_stride * height_abs) as usize;
assert!(src.len() >= min, "src isn't large enough");
}
fn i420_assert_safety(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
_width: i32,
height: i32,
) {
let height_abs = height.unsigned_abs();
let chroma_height = (height_abs + 1) / 2;
let min_y = (src_stride_y * height_abs) as usize;
let min_u = (src_stride_u * chroma_height) as usize;
let min_v = (src_stride_v * chroma_height) as usize;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_u.len() >= min_u, "src_u isn't large enough");
assert!(src_v.len() >= min_v, "src_v isn't large enough");
}
fn nv12_assert_safety(
src_y: &[u8],
src_stride_y: u32,
src_uv: &[u8],
src_stride_uv: u32,
_width: i32,
height: i32,
) {
let height_abs = height.unsigned_abs();
let chroma_height = (height_abs + 1) / 2;
let min_y = (src_stride_y * height_abs) as usize;
let min_uv = (src_stride_uv * chroma_height) as usize;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_uv.len() >= min_uv, "src_uv isn't large enough");
}
fn i444_assert_safety(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
_width: i32,
height: i32,
) {
let height_abs = height.unsigned_abs();
let min_y = (src_stride_y * height_abs) as usize;
let min_u = (src_stride_u * height_abs) as usize;
let min_v = (src_stride_v * height_abs) as usize;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_u.len() >= min_u, "src_u isn't large enough");
assert!(src_v.len() >= min_v, "src_v isn't large enough");
}
fn i422_assert_safety(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
_width: i32,
height: i32,
) {
let height_abs = height.unsigned_abs();
let min_y = (src_stride_y * height_abs) as usize;
let min_u = (src_stride_u * height_abs) as usize;
let min_v = (src_stride_v * height_abs) as usize;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_u.len() >= min_u, "src_u isn't large enough");
assert!(src_v.len() >= min_v, "src_v isn't large enough");
}
fn i010_assert_safety(
src_y: &[u16],
src_stride_y: u32,
src_u: &[u16],
src_stride_u: u32,
src_v: &[u16],
src_stride_v: u32,
_width: i32,
height: i32,
) {
let height_abs: u32 = height.unsigned_abs();
let chroma_height = height_abs / 2;
let min_y = (src_stride_y * height_abs) as usize / 2;
let min_u = (src_stride_u * chroma_height) as usize / 2;
let min_v = (src_stride_v * chroma_height) as usize / 2;
assert!(src_y.len() >= min_y, "src_y isn't large enough");
assert!(src_u.len() >= min_u, "src_u isn't large enough");
assert!(src_v.len() >= min_v, "src_v isn't large enough");
}
macro_rules! i420_to_rgba {
($x:ident) => {
pub fn $x(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst: &mut [u8],
dst_stride: u32,
width: i32,
height: i32,
) {
i420_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst, dst_stride, width, height);
unsafe {
yuv_sys::ffi::$x(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst.as_mut_ptr(),
dst_stride as i32,
width,
height,
)
.unwrap();
}
}
};
}
macro_rules! rgba_to_i420 {
($x:ident) => {
pub fn $x(
src_argb: &[u8],
src_stride_argb: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
argb_assert_safety(src_argb, src_stride_argb, width, height);
unsafe {
yuv_sys::ffi::$x(
src_argb.as_ptr(),
src_stride_argb as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap();
}
}
};
}
pub fn argb_to_rgb24(
src_argb: &[u8],
src_stride_argb: u32,
dst_rgb24: &mut [u8],
dst_stride_rgb24: u32,
width: i32,
height: i32,
) {
argb_assert_safety(src_argb, src_stride_argb, width, height);
argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height);
unsafe {
yuv_sys::ffi::argb_to_rgb24(
src_argb.as_ptr(),
src_stride_argb as i32,
dst_rgb24.as_mut_ptr(),
dst_stride_rgb24 as i32,
width,
height,
)
.unwrap();
}
}
// I420 <> RGB conversion
rgba_to_i420!(argb_to_i420);
rgba_to_i420!(abgr_to_i420);
i420_to_rgba!(i420_to_argb);
i420_to_rgba!(i420_to_bgra);
i420_to_rgba!(i420_to_abgr);
i420_to_rgba!(i420_to_rgba);
pub fn i420_to_nv12(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_uv: &mut [u8],
dst_stride_uv: u32,
width: i32,
height: i32,
) {
i420_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
unsafe {
yuv_sys::ffi::i420_to_nv12(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_uv.as_mut_ptr(),
dst_stride_uv as i32,
width,
height,
)
.unwrap();
}
}
pub fn nv12_to_i420(
src_y: &[u8],
src_stride_y: u32,
src_uv: &[u8],
src_stride_uv: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
unsafe {
yuv_sys::ffi::nv12_to_i420(
src_y.as_ptr(),
src_stride_y as i32,
src_uv.as_ptr(),
src_stride_uv as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap();
}
}
pub fn i444_to_i420(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
i444_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
unsafe {
yuv_sys::ffi::i444_to_i420(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap();
}
}
pub fn i422_to_i420(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
i422_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
unsafe {
yuv_sys::ffi::i422_to_i420(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap()
}
}
pub fn i010_to_i420(
src_y: &[u16],
src_stride_y: u32,
src_u: &[u16],
src_stride_u: u32,
src_v: &[u16],
src_stride_v: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_u: &mut [u8],
dst_stride_u: u32,
dst_v: &mut [u8],
dst_stride_v: u32,
width: i32,
height: i32,
) {
i010_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
i420_assert_safety(
dst_y,
dst_stride_y,
dst_u,
dst_stride_u,
dst_v,
dst_stride_v,
width,
height,
);
unsafe {
yuv_sys::ffi::i010_to_i420(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_u.as_mut_ptr(),
dst_stride_u as i32,
dst_v.as_mut_ptr(),
dst_stride_v as i32,
width,
height,
)
.unwrap()
}
}
pub fn nv12_to_argb(
src_y: &[u8],
src_stride_y: u32,
src_uv: &[u8],
src_stride_uv: u32,
dst_argb: &mut [u8],
dst_stride_argb: u32,
width: i32,
height: i32,
) {
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
unsafe {
yuv_sys::ffi::nv12_to_argb(
src_y.as_ptr(),
src_stride_y as i32,
src_uv.as_ptr(),
src_stride_uv as i32,
dst_argb.as_mut_ptr(),
dst_stride_argb as i32,
width,
height,
)
.unwrap();
}
}
pub fn nv12_to_abgr(
src_y: &[u8],
src_stride_y: u32,
src_uv: &[u8],
src_stride_uv: u32,
dst_abgr: &mut [u8],
dst_stride_abgr: u32,
width: i32,
height: i32,
) {
nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height);
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
unsafe {
yuv_sys::ffi::nv12_to_abgr(
src_y.as_ptr(),
src_stride_y as i32,
src_uv.as_ptr(),
src_stride_uv as i32,
dst_abgr.as_mut_ptr(),
dst_stride_abgr as i32,
width,
height,
)
.unwrap();
}
}
pub fn i444_to_argb(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_argb: &mut [u8],
dst_stride_argb: u32,
width: i32,
height: i32,
) {
i444_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
unsafe {
yuv_sys::ffi::i444_to_argb(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_argb.as_mut_ptr(),
dst_stride_argb as i32,
width,
height,
)
.unwrap();
}
}
pub fn i444_to_abgr(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_abgr: &mut [u8],
dst_stride_abgr: u32,
width: i32,
height: i32,
) {
i444_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
unsafe {
yuv_sys::ffi::i444_to_abgr(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_abgr.as_mut_ptr(),
dst_stride_abgr as i32,
width,
height,
)
.unwrap()
}
}
pub fn i422_to_argb(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_argb: &mut [u8],
dst_stride_argb: u32,
width: i32,
height: i32,
) {
i422_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
unsafe {
yuv_sys::ffi::i422_to_argb(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_argb.as_mut_ptr(),
dst_stride_argb as i32,
width,
height,
)
.unwrap();
}
}
pub fn i422_to_abgr(
src_y: &[u8],
src_stride_y: u32,
src_u: &[u8],
src_stride_u: u32,
src_v: &[u8],
src_stride_v: u32,
dst_abgr: &mut [u8],
dst_stride_abgr: u32,
width: i32,
height: i32,
) {
i422_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
unsafe {
yuv_sys::ffi::i422_to_abgr(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_abgr.as_mut_ptr(),
dst_stride_abgr as i32,
width,
height,
)
.unwrap()
}
}
pub fn i010_to_argb(
src_y: &[u16],
src_stride_y: u32,
src_u: &[u16],
src_stride_u: u32,
src_v: &[u16],
src_stride_v: u32,
dst_argb: &mut [u8],
dst_stride_argb: u32,
width: i32,
height: i32,
) {
i010_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_argb, dst_stride_argb, width, height);
unsafe {
yuv_sys::ffi::i010_to_argb(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_argb.as_mut_ptr(),
dst_stride_argb as i32,
width,
height,
)
.unwrap()
}
}
pub fn i010_to_abgr(
src_y: &[u16],
src_stride_y: u32,
src_u: &[u16],
src_stride_u: u32,
src_v: &[u16],
src_stride_v: u32,
dst_abgr: &mut [u8],
dst_stride_abgr: u32,
width: i32,
height: i32,
) {
i010_assert_safety(
src_y,
src_stride_y,
src_u,
src_stride_u,
src_v,
src_stride_v,
width,
height,
);
argb_assert_safety(dst_abgr, dst_stride_abgr, width, height);
unsafe {
yuv_sys::ffi::i010_to_abgr(
src_y.as_ptr(),
src_stride_y as i32,
src_u.as_ptr(),
src_stride_u as i32,
src_v.as_ptr(),
src_stride_v as i32,
dst_abgr.as_mut_ptr(),
dst_stride_abgr as i32,
width,
height,
)
.unwrap()
}
}
pub fn abgr_to_nv12(
src_abgr: &[u8],
src_stride_abgr: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_uv: &mut [u8],
dst_stride_uv: u32,
width: i32,
height: i32,
) {
argb_assert_safety(src_abgr, src_stride_abgr, width, height);
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
unsafe {
yuv_sys::ffi::abgr_to_nv12(
src_abgr.as_ptr(),
src_stride_abgr as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_uv.as_mut_ptr(),
dst_stride_uv as i32,
width,
height,
)
.unwrap()
}
}
pub fn argb_to_nv12(
src_argb: &[u8],
src_stride_argb: u32,
dst_y: &mut [u8],
dst_stride_y: u32,
dst_uv: &mut [u8],
dst_stride_uv: u32,
width: i32,
height: i32,
) {
argb_assert_safety(src_argb, src_stride_argb, width, height);
nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height);
unsafe {
yuv_sys::ffi::argb_to_nv12(
src_argb.as_ptr(),
src_stride_argb as i32,
dst_y.as_mut_ptr(),
dst_stride_y as i32,
dst_uv.as_mut_ptr(),
dst_stride_uv as i32,
width,
height,
)
.unwrap()
}
}
@@ -0,0 +1,345 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
data_channel::{DataChannel, DataChannelInit},
ice_candidate::IceCandidate,
imp::peer_connection as imp_pc,
media_stream::MediaStream,
media_stream_track::MediaStreamTrack,
peer_connection_factory::RtcConfiguration,
rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
rtp_transceiver::{RtpTransceiver, RtpTransceiverInit},
session_description::SessionDescription,
stats::RtcStats,
MediaType, RtcError,
};
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum PeerConnectionState {
New,
Connecting,
Connected,
Disconnected,
Failed,
Closed,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceConnectionState {
New,
Checking,
Connected,
Completed,
Failed,
Disconnected,
Closed,
Max,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceGatheringState {
New,
Gathering,
Complete,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SignalingState {
Stable,
HaveLocalOffer,
HaveLocalPrAnswer,
HaveRemoteOffer,
HaveRemotePrAnswer,
Closed,
}
#[derive(Debug, Clone, Default)]
pub struct OfferOptions {
pub ice_restart: bool,
pub offer_to_receive_audio: bool,
pub offer_to_receive_video: bool,
}
#[derive(Debug, Clone, Default)]
pub struct AnswerOptions {}
#[derive(Debug, Clone)]
pub struct IceCandidateError {
pub address: String,
pub port: i32,
pub url: String,
pub error_code: i32,
pub error_text: String,
}
#[derive(Debug, Clone)]
pub struct TrackEvent {
pub receiver: RtpReceiver,
pub streams: Vec<MediaStream>,
pub track: MediaStreamTrack,
pub transceiver: RtpTransceiver,
}
pub type OnConnectionChange = Box<dyn FnMut(PeerConnectionState) + Send + Sync>;
pub type OnDataChannel = Box<dyn FnMut(DataChannel) + Send + Sync>;
pub type OnIceCandidate = Box<dyn FnMut(IceCandidate) + Send + Sync>;
pub type OnIceCandidateError = Box<dyn FnMut(IceCandidateError) + Send + Sync>;
pub type OnIceConnectionChange = Box<dyn FnMut(IceConnectionState) + Send + Sync>;
pub type OnIceGatheringChange = Box<dyn FnMut(IceGatheringState) + Send + Sync>;
pub type OnNegotiationNeeded = Box<dyn FnMut(u32) + Send + Sync>;
pub type OnSignalingChange = Box<dyn FnMut(SignalingState) + Send + Sync>;
pub type OnTrack = Box<dyn FnMut(TrackEvent) + Send + Sync>;
#[derive(Clone)]
pub struct PeerConnection {
pub(crate) handle: imp_pc::PeerConnection,
}
impl PeerConnection {
pub fn set_configuration(&self, config: RtcConfiguration) -> Result<(), RtcError> {
self.handle.set_configuration(config)
}
pub async fn create_offer(
&self,
options: OfferOptions,
) -> Result<SessionDescription, RtcError> {
self.handle.create_offer(options).await
}
pub async fn create_answer(
&self,
options: AnswerOptions,
) -> Result<SessionDescription, RtcError> {
self.handle.create_answer(options).await
}
pub async fn set_local_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
self.handle.set_local_description(desc).await
}
pub async fn set_remote_description(&self, desc: SessionDescription) -> Result<(), RtcError> {
self.handle.set_remote_description(desc).await
}
pub async fn add_ice_candidate(&self, candidate: IceCandidate) -> Result<(), RtcError> {
self.handle.add_ice_candidate(candidate).await
}
pub fn create_data_channel(
&self,
label: &str,
init: DataChannelInit,
) -> Result<DataChannel, RtcError> {
self.handle.create_data_channel(label, init)
}
pub fn add_track<T: AsRef<str>>(
&self,
track: MediaStreamTrack,
streams_ids: &[T],
) -> Result<RtpSender, RtcError> {
self.handle.add_track(track, streams_ids)
}
pub fn remove_track(&self, sender: RtpSender) -> Result<(), RtcError> {
self.handle.remove_track(sender)
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
self.handle.get_stats().await
}
pub fn add_transceiver(
&self,
track: MediaStreamTrack,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
self.handle.add_transceiver(track, init)
}
pub fn add_transceiver_for_media(
&self,
media_type: MediaType,
init: RtpTransceiverInit,
) -> Result<RtpTransceiver, RtcError> {
self.handle.add_transceiver_for_media(media_type, init)
}
pub fn close(&self) {
self.handle.close()
}
pub fn restart_ice(&self) {
self.handle.restart_ice()
}
pub fn connection_state(&self) -> PeerConnectionState {
self.handle.connection_state()
}
pub fn ice_connection_state(&self) -> IceConnectionState {
self.handle.ice_connection_state()
}
pub fn ice_gathering_state(&self) -> IceGatheringState {
self.handle.ice_gathering_state()
}
pub fn signaling_state(&self) -> SignalingState {
self.handle.signaling_state()
}
pub fn current_local_description(&self) -> Option<SessionDescription> {
self.handle.current_local_description()
}
pub fn current_remote_description(&self) -> Option<SessionDescription> {
self.handle.current_remote_description()
}
pub fn senders(&self) -> Vec<RtpSender> {
self.handle.senders()
}
pub fn receivers(&self) -> Vec<RtpReceiver> {
self.handle.receivers()
}
pub fn transceivers(&self) -> Vec<RtpTransceiver> {
self.handle.transceivers()
}
pub fn on_connection_state_change(&self, f: Option<OnConnectionChange>) {
self.handle.on_connection_state_change(f)
}
pub fn on_data_channel(&self, f: Option<OnDataChannel>) {
self.handle.on_data_channel(f)
}
pub fn on_ice_candidate(&self, f: Option<OnIceCandidate>) {
self.handle.on_ice_candidate(f)
}
pub fn on_ice_candidate_error(&self, f: Option<OnIceCandidateError>) {
self.handle.on_ice_candidate_error(f)
}
pub fn on_ice_connection_state_change(&self, f: Option<OnIceConnectionChange>) {
self.handle.on_ice_connection_state_change(f)
}
pub fn on_ice_gathering_state_change(&self, f: Option<OnIceGatheringChange>) {
self.handle.on_ice_gathering_state_change(f)
}
pub fn on_negotiation_needed(&self, f: Option<OnNegotiationNeeded>) {
self.handle.on_negotiation_needed(f)
}
pub fn on_signaling_state_change(&self, f: Option<OnSignalingChange>) {
self.handle.on_signaling_state_change(f)
}
pub fn on_track(&self, f: Option<OnTrack>) {
self.handle.on_track(f)
}
}
impl Debug for PeerConnection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PeerConnection")
.field("state", &self.connection_state())
.field("ice_state", &self.ice_connection_state())
.finish()
}
}
#[cfg(test)]
mod tests {
use log::trace;
use tokio::sync::mpsc;
use crate::{peer_connection::*, peer_connection_factory::*};
#[tokio::test]
async fn create_pc() {
let _ = env_logger::builder().is_test(true).try_init();
let factory = PeerConnectionFactory::default();
let config = RtcConfiguration {
ice_servers: vec![IceServer {
urls: vec!["stun:stun1.l.google.com:19302".to_string()],
username: "".into(),
password: "".into(),
}],
continual_gathering_policy: ContinualGatheringPolicy::GatherOnce,
ice_transport_type: IceTransportsType::All,
};
let bob = factory.create_peer_connection(config.clone()).unwrap();
let alice = factory.create_peer_connection(config.clone()).unwrap();
let (bob_ice_tx, mut bob_ice_rx) = mpsc::unbounded_channel::<IceCandidate>();
let (alice_ice_tx, mut alice_ice_rx) = mpsc::unbounded_channel::<IceCandidate>();
let (alice_dc_tx, mut alice_dc_rx) = mpsc::unbounded_channel::<DataChannel>();
bob.on_ice_candidate(Some(Box::new(move |candidate| {
bob_ice_tx.send(candidate).unwrap();
})));
alice.on_ice_candidate(Some(Box::new(move |candidate| {
alice_ice_tx.send(candidate).unwrap();
})));
alice.on_data_channel(Some(Box::new(move |dc| {
alice_dc_tx.send(dc).unwrap();
})));
let bob_dc = bob.create_data_channel("test_dc", DataChannelInit::default()).unwrap();
let offer = bob.create_offer(OfferOptions::default()).await.unwrap();
trace!("Bob offer: {:?}", offer);
bob.set_local_description(offer.clone()).await.unwrap();
alice.set_remote_description(offer).await.unwrap();
let answer = alice.create_answer(AnswerOptions::default()).await.unwrap();
trace!("Alice answer: {:?}", answer);
alice.set_local_description(answer.clone()).await.unwrap();
bob.set_remote_description(answer).await.unwrap();
let bob_ice = bob_ice_rx.recv().await.unwrap();
let alice_ice = alice_ice_rx.recv().await.unwrap();
bob.add_ice_candidate(alice_ice).await.unwrap();
alice.add_ice_candidate(bob_ice).await.unwrap();
let (data_tx, mut data_rx) = mpsc::unbounded_channel::<String>();
let alice_dc = alice_dc_rx.recv().await.unwrap();
alice_dc.on_message(Some(Box::new(move |buffer| {
data_tx.send(String::from_utf8_lossy(buffer.data).to_string()).unwrap();
})));
bob_dc.send(b"This is a test", true).unwrap();
assert_eq!(data_rx.recv().await.unwrap(), "This is a test");
alice.close();
bob.close();
}
}
@@ -0,0 +1,316 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::peer_connection_factory as imp_pcf, peer_connection::PeerConnection,
rtp_parameters::RtpCapabilities, MediaType, RtcError,
};
#[derive(Debug, Clone)]
pub struct IceServer {
pub urls: Vec<String>,
pub username: String,
pub password: String,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum ContinualGatheringPolicy {
GatherOnce,
GatherContinually,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum IceTransportsType {
Relay,
NoHost,
All,
}
#[derive(Debug, Clone)]
pub struct RtcConfiguration {
pub ice_servers: Vec<IceServer>,
pub continual_gathering_policy: ContinualGatheringPolicy,
pub ice_transport_type: IceTransportsType,
}
impl Default for RtcConfiguration {
fn default() -> Self {
Self {
ice_servers: vec![],
continual_gathering_policy: ContinualGatheringPolicy::GatherContinually,
ice_transport_type: IceTransportsType::All,
}
}
}
#[derive(Clone, Default)]
pub struct PeerConnectionFactory {
pub(crate) handle: imp_pcf::PeerConnectionFactory,
}
impl Debug for PeerConnectionFactory {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.debug_struct("PeerConnectionFactory").finish()
}
}
impl PeerConnectionFactory {
pub fn create_peer_connection(
&self,
config: RtcConfiguration,
) -> Result<PeerConnection, RtcError> {
self.handle.create_peer_connection(config)
}
pub fn get_rtp_sender_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.handle.get_rtp_sender_capabilities(media_type)
}
pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
self.handle.get_rtp_receiver_capabilities(media_type)
}
}
pub mod native {
use super::PeerConnectionFactory;
use crate::{
audio_source::native::NativeAudioSource, audio_track::RtcAudioTrack,
video_source::native::NativeVideoSource, video_track::RtcVideoTrack,
};
pub trait PeerConnectionFactoryExt {
fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack;
fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack;
/// Create an audio track that uses the Platform ADM for capture.
/// The track will capture audio from the selected recording device.
fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack;
// Device enumeration
fn playout_devices(&self) -> i16;
fn recording_devices(&self) -> i16;
fn playout_device_name(&self, index: u16) -> String;
fn recording_device_name(&self, index: u16) -> String;
/// Get device GUID (platform-specific unique identifier, stable across hot-plug)
fn playout_device_guid(&self, index: u16) -> String;
fn recording_device_guid(&self, index: u16) -> String;
// Device selection by index
fn set_playout_device(&self, index: u16) -> bool;
fn set_recording_device(&self, index: u16) -> bool;
/// Device selection by GUID (preferred - stable across device changes)
fn set_playout_device_by_guid(&self, guid: &str) -> bool;
fn set_recording_device_by_guid(&self, guid: &str) -> bool;
// Recording control (for device switching while active)
fn stop_recording(&self) -> bool;
fn init_recording(&self) -> bool;
fn start_recording(&self) -> bool;
fn recording_is_initialized(&self) -> bool;
// Playout control (for device switching while active)
fn stop_playout(&self) -> bool;
fn init_playout(&self) -> bool;
fn start_playout(&self) -> bool;
fn playout_is_initialized(&self) -> bool;
// Built-in audio processing (hardware AEC/AGC/NS)
// Only available on iOS and some Android devices
fn builtin_aec_is_available(&self) -> bool;
fn builtin_agc_is_available(&self) -> bool;
fn builtin_ns_is_available(&self) -> bool;
fn enable_builtin_aec(&self, enable: bool) -> bool;
fn enable_builtin_agc(&self, enable: bool) -> bool;
fn enable_builtin_ns(&self, enable: bool) -> bool;
// ADM recording control
// Use this to disable microphone when only using NativeAudioSource
fn set_adm_recording_enabled(&self, enabled: bool);
fn adm_recording_enabled(&self) -> bool;
// ADM playout control
// When disabled (default), playout uses synthetic mode - remote audio is
// delivered via FFI callbacks. When enabled, plays through platform speakers.
fn set_adm_playout_enabled(&self, enabled: bool);
fn adm_playout_enabled(&self) -> bool;
// Platform ADM lifecycle management
// Call acquire_platform_adm when creating PlatformAudio.
// Call release_platform_adm when disposing PlatformAudio.
// The Platform ADM is only created when first acquired, and terminated
// when the last reference is released.
fn acquire_platform_adm(&self) -> bool;
fn release_platform_adm(&self);
fn platform_adm_ref_count(&self) -> i32;
fn is_platform_adm_active(&self) -> bool;
// Ensures the Platform ADM exists, retrying creation if an earlier
// attempt failed (e.g. the OS audio stack was still starting up).
fn ensure_platform_adm(&self) -> bool;
// Distinguishes "audio stack unavailable" from "zero audio devices".
fn platform_adm_available(&self) -> bool;
}
impl PeerConnectionFactoryExt for PeerConnectionFactory {
fn create_video_track(&self, label: &str, source: NativeVideoSource) -> RtcVideoTrack {
self.handle.create_video_track(label, source)
}
fn create_audio_track(&self, label: &str, source: NativeAudioSource) -> RtcAudioTrack {
self.handle.create_audio_track(label, source)
}
fn create_device_audio_track(&self, label: &str) -> RtcAudioTrack {
self.handle.create_device_audio_track(label)
}
fn playout_devices(&self) -> i16 {
self.handle.playout_devices()
}
fn recording_devices(&self) -> i16 {
self.handle.recording_devices()
}
fn playout_device_name(&self, index: u16) -> String {
self.handle.playout_device_name(index)
}
fn recording_device_name(&self, index: u16) -> String {
self.handle.recording_device_name(index)
}
fn playout_device_guid(&self, index: u16) -> String {
self.handle.playout_device_guid(index)
}
fn recording_device_guid(&self, index: u16) -> String {
self.handle.recording_device_guid(index)
}
fn set_playout_device(&self, index: u16) -> bool {
self.handle.set_playout_device(index)
}
fn set_recording_device(&self, index: u16) -> bool {
self.handle.set_recording_device(index)
}
fn set_playout_device_by_guid(&self, guid: &str) -> bool {
self.handle.set_playout_device_by_guid(guid)
}
fn set_recording_device_by_guid(&self, guid: &str) -> bool {
self.handle.set_recording_device_by_guid(guid)
}
fn stop_recording(&self) -> bool {
self.handle.stop_recording()
}
fn init_recording(&self) -> bool {
self.handle.init_recording()
}
fn start_recording(&self) -> bool {
self.handle.start_recording()
}
fn recording_is_initialized(&self) -> bool {
self.handle.recording_is_initialized()
}
fn stop_playout(&self) -> bool {
self.handle.stop_playout()
}
fn init_playout(&self) -> bool {
self.handle.init_playout()
}
fn start_playout(&self) -> bool {
self.handle.start_playout()
}
fn playout_is_initialized(&self) -> bool {
self.handle.playout_is_initialized()
}
fn builtin_aec_is_available(&self) -> bool {
self.handle.builtin_aec_is_available()
}
fn builtin_agc_is_available(&self) -> bool {
self.handle.builtin_agc_is_available()
}
fn builtin_ns_is_available(&self) -> bool {
self.handle.builtin_ns_is_available()
}
fn enable_builtin_aec(&self, enable: bool) -> bool {
self.handle.enable_builtin_aec(enable)
}
fn enable_builtin_agc(&self, enable: bool) -> bool {
self.handle.enable_builtin_agc(enable)
}
fn enable_builtin_ns(&self, enable: bool) -> bool {
self.handle.enable_builtin_ns(enable)
}
fn set_adm_recording_enabled(&self, enabled: bool) {
self.handle.set_adm_recording_enabled(enabled)
}
fn adm_recording_enabled(&self) -> bool {
self.handle.adm_recording_enabled()
}
fn set_adm_playout_enabled(&self, enabled: bool) {
self.handle.set_adm_playout_enabled(enabled)
}
fn adm_playout_enabled(&self) -> bool {
self.handle.adm_playout_enabled()
}
fn acquire_platform_adm(&self) -> bool {
self.handle.acquire_platform_adm()
}
fn release_platform_adm(&self) {
self.handle.release_platform_adm()
}
fn platform_adm_ref_count(&self) -> i32 {
self.handle.platform_adm_ref_count()
}
fn is_platform_adm_active(&self) -> bool {
self.handle.is_platform_adm_active()
}
fn ensure_platform_adm(&self) -> bool {
self.handle.ensure_platform_adm()
}
fn platform_adm_available(&self) -> bool {
self.handle.platform_adm_available()
}
}
}
@@ -0,0 +1,43 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub use crate::{
audio_frame::AudioFrame,
audio_source::{AudioSourceOptions, RtcAudioSource},
audio_track::RtcAudioTrack,
data_channel::{DataBuffer, DataChannel, DataChannelError, DataChannelInit, DataChannelState},
ice_candidate::IceCandidate,
media_stream::MediaStream,
media_stream_track::{MediaStreamTrack, RtcTrackState},
peer_connection::{
AnswerOptions, IceConnectionState, IceGatheringState, OfferOptions, PeerConnection,
PeerConnectionState, SignalingState,
},
peer_connection_factory::{
ContinualGatheringPolicy, IceServer, IceTransportsType, PeerConnectionFactory,
RtcConfiguration,
},
rtp_parameters::*,
rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
rtp_transceiver::{RtpTransceiver, RtpTransceiverDirection, RtpTransceiverInit},
session_description::{SdpType, SessionDescription},
video_frame::{
BoxVideoBuffer, BoxVideoFrame, I010Buffer, I420ABuffer, I420Buffer, I422Buffer, I444Buffer,
NV12Buffer, VideoBuffer, VideoBufferType, VideoFormatType, VideoFrame, VideoRotation,
},
video_source::{RtcVideoSource, VideoResolution},
video_track::RtcVideoTrack,
MediaType, RtcError, RtcErrorType,
};
@@ -0,0 +1,44 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use webrtc_sys::recorded_audio_tap::ffi as sys;
use webrtc_sys::recorded_audio_tap::RecordedAudioSinkWrapper;
/// Handle to an installed recorded-audio sink. Dropping it does not clear
/// the sink; call [`clear_recorded_audio_sink`] with this generation.
pub type RecordedAudioSinkGeneration = u64;
/// Installs a process-global tap on platform-ADM recorded microphone audio.
///
/// `callback` is invoked on the ADM capture thread with one 48kHz mono
/// 10ms frame (480 samples) per call: `(samples, sample_rate_hz,
/// num_channels, samples_per_channel)`. It must be wait-free: do no
/// allocation or blocking work, only hand the frame to a bounded queue.
/// Returns a generation token to pass to [`clear_recorded_audio_sink`].
pub fn set_recorded_audio_sink<F>(callback: F) -> RecordedAudioSinkGeneration
where
F: Fn(&[i16], i32, usize, usize) + Send + Sync + 'static,
{
sys::set_recorded_audio_sink(Box::new(RecordedAudioSinkWrapper::new(Box::new(callback))))
}
/// Removes the recorded-audio sink, but only if `generation` is still the
/// installed one. A stale token is a no-op, so a late teardown cannot
/// clobber a sink a newer caller installed.
pub fn clear_recorded_audio_sink(generation: RecordedAudioSinkGeneration) {
sys::clear_recorded_audio_sink(generation);
}
}
@@ -0,0 +1,98 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::rtp_transceiver::RtpTransceiverDirection;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum Priority {
VeryLow,
Low,
Medium,
High,
}
#[derive(Debug, Clone)]
pub struct RtpHeaderExtensionParameters {
pub uri: String,
pub id: i32,
pub encrypted: bool,
}
#[derive(Debug, Clone, Default)]
pub struct RtpParameters {
pub codecs: Vec<RtpCodecParameters>,
pub header_extensions: Vec<RtpHeaderExtensionParameters>,
pub rtcp: RtcpParameters,
}
#[derive(Debug, Clone, Default)]
pub struct RtpCodecParameters {
pub payload_type: u8,
pub mime_type: String, // read-only
pub clock_rate: Option<u64>,
pub channels: Option<u16>,
}
#[derive(Debug, Clone, Default)]
pub struct RtcpParameters {
pub cname: String,
pub reduced_size: bool,
}
#[derive(Debug, Clone)]
pub struct RtpEncodingParameters {
pub active: bool,
pub max_bitrate: Option<u64>,
pub max_framerate: Option<f64>,
pub priority: Priority,
pub rid: String,
pub scale_resolution_down_by: Option<f64>,
/// RTP scalability mode (e.g. "L3T3_KEY"). Required to enable true
/// SVC for codecs that support it (VP9, AV1).
pub scalability_mode: Option<String>,
}
#[derive(Debug, Clone)]
pub struct RtpCodecCapability {
pub channels: Option<u16>,
pub clock_rate: Option<u64>,
pub mime_type: String,
pub sdp_fmtp_line: Option<String>,
}
#[derive(Debug, Clone)]
pub struct RtpHeaderExtensionCapability {
pub uri: String,
pub direction: RtpTransceiverDirection,
}
#[derive(Debug, Clone)]
pub struct RtpCapabilities {
pub codecs: Vec<RtpCodecCapability>,
pub header_extensions: Vec<RtpHeaderExtensionCapability>,
}
impl Default for RtpEncodingParameters {
fn default() -> Self {
Self {
active: true,
max_bitrate: None,
max_framerate: None,
priority: Priority::Low,
rid: String::default(),
scale_resolution_down_by: None,
scalability_mode: None,
}
}
}
@@ -0,0 +1,48 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::rtp_receiver as imp_rr, media_stream_track::MediaStreamTrack,
rtp_parameters::RtpParameters, stats::RtcStats, RtcError,
};
#[derive(Clone)]
pub struct RtpReceiver {
pub(crate) handle: imp_rr::RtpReceiver,
}
impl RtpReceiver {
pub fn track(&self) -> Option<MediaStreamTrack> {
self.handle.track()
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
self.handle.get_stats().await
}
pub fn parameters(&self) -> RtpParameters {
self.handle.parameters()
}
}
impl Debug for RtpReceiver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpReceiver")
.field("track", &self.track())
.field("cname", &self.parameters().rtcp.cname)
.finish()
}
}
@@ -0,0 +1,53 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::rtp_sender as imp_rs, media_stream_track::MediaStreamTrack, rtp_parameters::RtpParameters,
stats::RtcStats, RtcError,
};
#[derive(Clone)]
pub struct RtpSender {
pub(crate) handle: imp_rs::RtpSender,
}
impl RtpSender {
pub fn track(&self) -> Option<MediaStreamTrack> {
self.handle.track()
}
pub async fn get_stats(&self) -> Result<Vec<RtcStats>, RtcError> {
self.handle.get_stats().await
}
pub fn set_track(&self, track: Option<MediaStreamTrack>) -> Result<(), RtcError> {
self.handle.set_track(track)
}
pub fn parameters(&self) -> RtpParameters {
self.handle.parameters()
}
pub fn set_parameters(&self, parameters: RtpParameters) -> Result<(), RtcError> {
self.handle.set_parameters(parameters)
}
}
impl Debug for RtpSender {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpReceiver").field("cname", &self.parameters().rtcp.cname).finish()
}
}
@@ -0,0 +1,85 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::rtp_transceiver as imp_rt,
rtp_parameters::{RtpCodecCapability, RtpEncodingParameters},
rtp_receiver::RtpReceiver,
rtp_sender::RtpSender,
RtcError,
};
#[derive(Debug, Clone)]
pub struct RtpTransceiverInit {
pub direction: RtpTransceiverDirection,
pub stream_ids: Vec<String>,
pub send_encodings: Vec<RtpEncodingParameters>,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum RtpTransceiverDirection {
SendRecv,
SendOnly,
RecvOnly,
Inactive,
Stopped,
}
#[derive(Clone)]
pub struct RtpTransceiver {
pub(crate) handle: imp_rt::RtpTransceiver,
}
impl RtpTransceiver {
pub fn mid(&self) -> Option<String> {
self.handle.mid()
}
pub fn current_direction(&self) -> Option<RtpTransceiverDirection> {
self.handle.current_direction()
}
pub fn direction(&self) -> RtpTransceiverDirection {
self.handle.direction()
}
pub fn sender(&self) -> RtpSender {
self.handle.sender()
}
pub fn receiver(&self) -> RtpReceiver {
self.handle.receiver()
}
pub fn set_codec_preferences(&self, codecs: Vec<RtpCodecCapability>) -> Result<(), RtcError> {
self.handle.set_codec_preferences(codecs)
}
pub fn stop(&self) -> Result<(), RtcError> {
self.handle.stop()
}
}
impl Debug for RtpTransceiver {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtpTransceiver")
.field("mid", &self.mid())
.field("direction", &self.direction())
.field("sender", &self.sender())
.field("receiver", &self.receiver())
.finish()
}
}
@@ -0,0 +1,90 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
fmt::{Debug, Display},
str::FromStr,
};
use thiserror::Error;
use crate::imp::session_description as sd_imp;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SdpType {
Offer,
PrAnswer,
Answer,
Rollback,
}
impl FromStr for SdpType {
type Err = &'static str;
fn from_str(sdp_type: &str) -> Result<Self, Self::Err> {
match sdp_type {
"offer" => Ok(Self::Offer),
"pranswer" => Ok(Self::PrAnswer),
"answer" => Ok(Self::Answer),
"rollback" => Ok(Self::Rollback),
_ => Err("invalid SdpType"),
}
}
}
impl Display for SdpType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let s = match self {
SdpType::Offer => "offer",
SdpType::PrAnswer => "pranswer",
SdpType::Answer => "answer",
SdpType::Rollback => "rollback",
};
write!(f, "{}", s)
}
}
#[derive(Clone)]
pub struct SessionDescription {
pub(crate) handle: sd_imp::SessionDescription,
}
#[derive(Clone, Error, Debug)]
#[error("Failed to parse sdp: {line} - {description}")]
pub struct SdpParseError {
pub line: String,
pub description: String,
}
impl SessionDescription {
pub fn parse(sdp: &str, sdp_type: SdpType) -> Result<Self, SdpParseError> {
sd_imp::SessionDescription::parse(sdp, sdp_type)
}
pub fn sdp_type(&self) -> SdpType {
self.handle.sdp_type()
}
}
impl ToString for SessionDescription {
fn to_string(&self) -> String {
self.handle.to_string()
}
}
impl Debug for SessionDescription {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SessionDescription").field("sdp_type", &self.sdp_type()).finish()
}
}
@@ -0,0 +1,624 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use serde::Deserialize;
use crate::data_channel::DataChannelState;
/// Values from https://www.w3.org/TR/webrtc-stats/ (NOTE: Some of the structs are not in the SPEC
/// but inside libwebrtc)
/// serde will handle the magic of correctly deserializing the json into our structs.
/// The enums values are inside encapsulated inside option because we're not sure about their
/// default values (So we default to None instead of an arbitrary value)
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "kebab-case")]
pub enum RtcStats {
Codec(CodecStats),
InboundRtp(InboundRtpStats),
OutboundRtp(OutboundRtpStats),
RemoteInboundRtp(RemoteInboundRtpStats),
RemoteOutboundRtp(RemoteOutboundRtpStats),
MediaSource(MediaSourceStats),
MediaPlayout(MediaPlayoutStats),
PeerConnection(PeerConnectionStats),
DataChannel(DataChannelStats),
Transport(TransportStats),
CandidatePair(CandidatePairStats),
LocalCandidate(LocalCandidateStats),
RemoteCandidate(RemoteCandidateStats),
Certificate(CertificateStats),
Stream(StreamStats),
Track, // Deprecated
}
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum QualityLimitationReason {
#[default]
None,
Cpu,
Bandwidth,
Other,
}
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceRole {
#[default]
Unknown,
Controlling,
Controlled,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DtlsTransportState {
New,
Connecting,
Connected,
Closed,
Failed,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceTransportState {
New,
Checking,
Connected,
Completed,
Disconnected,
Failed,
Closed,
}
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DtlsRole {
Client,
Server,
#[default]
Unknown,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum IceCandidatePairState {
Frozen,
Waiting,
InProgress, // in-progress
Failed,
Succeeded,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceCandidateType {
Host,
Srflx,
Prflx,
Relay,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceServerTransportProtocol {
Udp,
Tcp,
Tls,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum IceTcpCandidateType {
Active,
Passive,
So,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct CodecStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub codec: dictionaries::CodecStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct InboundRtpStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::RtpStreamStats,
#[serde(flatten)]
pub received: dictionaries::ReceivedRtpStreamStats,
#[serde(flatten)]
pub inbound: dictionaries::InboundRtpStreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct OutboundRtpStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::RtpStreamStats,
#[serde(flatten)]
pub sent: dictionaries::SentRtpStreamStats,
#[serde(flatten)]
pub outbound: dictionaries::OutboundRtpStreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct RemoteInboundRtpStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::RtpStreamStats,
#[serde(flatten)]
pub received: dictionaries::ReceivedRtpStreamStats,
#[serde(flatten)]
pub remote_inbound: dictionaries::RemoteInboundRtpStreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct RemoteOutboundRtpStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::RtpStreamStats,
#[serde(flatten)]
pub sent: dictionaries::SentRtpStreamStats,
#[serde(flatten)]
pub remote_outbound: dictionaries::RemoteOutboundRtpStreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct MediaSourceStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub source: dictionaries::MediaSourceStats,
#[serde(flatten)]
pub audio: dictionaries::AudioSourceStats,
#[serde(flatten)]
pub video: dictionaries::VideoSourceStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct MediaPlayoutStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub audio_playout: dictionaries::AudioPlayoutStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct PeerConnectionStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub pc: dictionaries::PeerConnectionStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct DataChannelStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub dc: dictionaries::DataChannelStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct TransportStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub transport: dictionaries::TransportStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct CandidatePairStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub candidate_pair: dictionaries::CandidatePairStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct LocalCandidateStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub local_candidate: dictionaries::IceCandidateStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct RemoteCandidateStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub remote_candidate: dictionaries::IceCandidateStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct CertificateStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub certificate: dictionaries::CertificateStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct StreamStats {
#[serde(flatten)]
pub rtc: dictionaries::RtcStats,
#[serde(flatten)]
pub stream: dictionaries::StreamStats,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct TrackStats {}
pub mod dictionaries {
use super::*;
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct RtcStats {
pub id: String,
pub timestamp: i64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct CodecStats {
pub payload_type: u32,
pub transport_id: String,
pub mime_type: String,
pub clock_rate: u32,
pub channels: u32,
pub sdp_fmtp_line: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct RtpStreamStats {
pub ssrc: u32,
pub kind: String,
pub transport_id: String,
pub codec_id: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct ReceivedRtpStreamStats {
pub packets_received: u64,
pub packets_lost: i64,
pub jitter: f64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct InboundRtpStreamStats {
pub track_identifier: String,
pub mid: String,
pub remote_id: String,
pub frames_decoded: u32,
pub key_frames_decoded: u32,
pub frames_rendered: u32,
pub frames_dropped: u32,
pub frame_width: u32,
pub frame_height: u32,
pub frames_per_second: f64,
pub qp_sum: u64,
pub total_decode_time: f64,
pub total_inter_frame_delay: f64,
pub total_squared_inter_frame_delay: f64,
pub pause_count: u32,
pub total_pause_duration: f64,
pub freeze_count: u32,
pub total_freeze_duration: f64,
pub last_packet_received_timestamp: f64,
pub header_bytes_received: u64,
pub packets_discarded: u64,
pub fec_bytes_received: u64,
pub fec_packets_received: u64,
pub fec_packets_discarded: u64,
pub bytes_received: u64,
pub nack_count: u32,
pub fir_count: u32,
pub pli_count: u32,
pub total_processing_delay: f64,
pub estimated_playout_timestamp: f64,
pub jitter_buffer_delay: f64,
pub jitter_buffer_target_delay: f64,
pub jitter_buffer_emitted_count: u64,
pub jitter_buffer_minimum_delay: f64,
pub total_samples_received: u64,
pub concealed_samples: u64,
pub silent_concealed_samples: u64,
pub concealment_events: u64,
pub inserted_samples_for_deceleration: u64,
pub removed_samples_for_acceleration: u64,
pub audio_level: f64,
pub total_audio_energy: f64,
pub total_samples_duration: f64,
pub frames_received: u64,
pub decoder_implementation: String,
pub playout_id: String,
pub power_efficient_decoder: bool,
pub frames_assembled_from_multiple_packets: u64,
pub total_assembly_time: f64,
pub retransmitted_packets_received: u64,
pub retransmitted_bytes_received: u64,
pub rtx_ssrc: u32,
pub fec_ssrc: u32,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct SentRtpStreamStats {
pub packets_sent: u64,
pub bytes_sent: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct OutboundRtpStreamStats {
pub mid: String,
pub media_source_id: String,
pub remote_id: String,
pub rid: String,
pub header_bytes_sent: u64,
pub retransmitted_packets_sent: u64,
pub retransmitted_bytes_sent: u64,
pub rtx_ssrc: u32,
pub target_bitrate: f64,
pub total_encoded_bytes_target: u64,
pub frame_width: u32,
pub frame_height: u32,
pub frames_per_second: f64,
pub frames_sent: u32,
pub huge_frames_sent: u32,
pub frames_encoded: u32,
pub key_frames_encoded: u32,
pub qp_sum: u64,
pub total_encode_time: f64,
pub total_packet_send_delay: f64,
pub quality_limitation_reason: QualityLimitationReason,
pub quality_limitation_durations: HashMap<String, f64>,
pub quality_limitation_resolution_changes: u32,
pub nack_count: u32,
pub fir_count: u32,
pub pli_count: u32,
pub encoder_implementation: String,
pub power_efficient_encoder: bool,
pub active: bool,
pub scalibility_mode: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct RemoteInboundRtpStreamStats {
pub local_id: String,
pub round_trip_time: f64,
pub total_round_trip_time: f64,
pub fraction_lost: f64,
pub round_trip_time_measurements: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct RemoteOutboundRtpStreamStats {
pub local_id: String,
pub remote_timestamp: f64,
pub reports_sent: u64,
pub round_trip_time: f64,
pub total_round_trip_time: f64,
pub round_trip_time_measurements: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct MediaSourceStats {
pub track_identifier: String,
pub kind: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct AudioSourceStats {
pub audio_level: f64,
pub total_audio_energy: f64,
pub total_samples_duration: f64,
pub echo_return_loss: f64,
pub echo_return_loss_enhancement: f64,
pub dropped_samples_duration: f64,
pub dropped_samples_events: u32,
pub total_capture_delay: f64,
pub total_samples_captured: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct VideoSourceStats {
pub width: u32,
pub height: u32,
pub frames: u32,
pub frames_per_second: f64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct AudioPlayoutStats {
pub kind: String,
pub synthesized_samples_duration: f64,
pub synthesized_samples_events: u32,
pub total_samples_duration: f64,
pub total_playout_delay: f64,
pub total_samples_count: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct PeerConnectionStats {
pub data_channels_opened: u32,
pub data_channels_closed: u32,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct DataChannelStats {
pub label: String,
pub protocol: String,
pub data_channel_identifier: i32,
pub state: Option<DataChannelState>,
pub messages_sent: u32,
pub bytes_sent: u64,
pub messages_received: u32,
pub bytes_received: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct TransportStats {
pub packets_sent: u64,
pub packets_received: u64,
pub bytes_sent: u64,
pub bytes_received: u64,
pub ice_role: IceRole,
pub ice_local_username_fragment: String,
pub dtls_state: Option<DtlsTransportState>,
pub ice_state: Option<IceTransportState>,
pub selected_candidate_pair_id: String,
pub local_certificate_id: String,
pub remote_certificate_id: String,
pub tls_version: String,
pub dtls_cipher: String,
pub dtls_role: DtlsRole,
pub srtp_cipher: String,
pub selected_candidate_pair_changes: u32,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct CandidatePairStats {
pub transport_id: String,
pub local_candidate_id: String,
pub remote_candidate_id: String,
pub state: Option<IceCandidatePairState>,
pub nominated: bool,
pub packets_sent: u64,
pub packets_received: u64,
pub bytes_sent: u64,
pub bytes_received: u64,
pub last_packet_sent_timestamp: f64,
pub last_packet_received_timestamp: f64,
pub total_round_trip_time: f64,
pub current_round_trip_time: f64,
pub available_outgoing_bitrate: f64,
pub available_incoming_bitrate: f64,
pub requests_received: u64,
pub requests_sent: u64,
pub responses_received: u64,
pub responses_sent: u64,
pub consent_requests_sent: u64,
pub packets_discarded_on_send: u32,
pub bytes_discarded_on_send: u64,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct IceCandidateStats {
pub transport_id: String,
pub address: String,
pub port: i32,
pub protocol: String,
pub candidate_type: Option<IceCandidateType>,
pub priority: i32,
pub url: String,
pub relay_protocol: Option<IceServerTransportProtocol>,
pub foundation: String,
pub related_address: String,
pub related_port: i32,
pub username_fragment: String,
pub tcp_type: Option<IceTcpCandidateType>,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct CertificateStats {
pub fingerprint: String,
pub fingerprint_algorithm: String,
pub base64_certificate: String,
pub issuer_certificate_id: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
#[serde(default)]
pub struct StreamStats {
pub id: String,
pub stream_identifier: String,
// pub timestamp: i64,
}
}
@@ -0,0 +1,596 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use thiserror::Error;
use crate::imp::video_frame as vf_imp;
#[derive(Debug, Error)]
pub enum SinkError {
#[error("platform error: {0}")]
Platform(String),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum VideoRotation {
VideoRotation0 = 0,
VideoRotation90 = 90,
VideoRotation180 = 180,
VideoRotation270 = 270,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum VideoFormatType {
ARGB,
BGRA,
ABGR,
RGBA,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum VideoBufferType {
Native,
I420,
I420A,
I422,
I444,
I010,
NV12,
}
/// Metadata carried alongside a video frame via the packet trailer mechanism.
///
/// Each field corresponds to an independently negotiable packet trailer feature
/// (`PTF_USER_TIMESTAMP`, `PTF_FRAME_ID`), so individual fields are `Option`.
#[derive(Debug, Clone, Copy)]
pub struct FrameMetadata {
/// Wall-clock capture time in microseconds, when `PTF_USER_TIMESTAMP` is enabled.
pub user_timestamp: Option<u64>,
/// Monotonically increasing frame identifier, when `PTF_FRAME_ID` is enabled.
pub frame_id: Option<u32>,
}
#[derive(Debug)]
pub struct VideoFrame<T>
where
T: AsRef<dyn VideoBuffer>,
{
pub rotation: VideoRotation,
pub timestamp_us: i64, // When the frame was captured in microseconds
/// Packet-trailer metadata, if any trailer features are active.
pub frame_metadata: Option<FrameMetadata>,
pub buffer: T,
}
impl<T: AsRef<dyn VideoBuffer>> VideoFrame<T> {
pub fn new(rotation: VideoRotation, buffer: T) -> Self {
Self { rotation, timestamp_us: 0, frame_metadata: None, buffer }
}
}
pub type BoxVideoBuffer = Box<dyn VideoBuffer>;
pub type BoxVideoFrame = VideoFrame<BoxVideoBuffer>;
pub(crate) mod internal {
use super::{I420Buffer, VideoFormatType};
pub trait BufferSealed: Send + Sync {
#[cfg(not(target_arch = "wasm32"))]
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer;
#[cfg(not(target_arch = "wasm32"))]
fn to_i420(&self) -> I420Buffer;
#[cfg(not(target_arch = "wasm32"))]
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
);
}
}
pub trait VideoBuffer: internal::BufferSealed + Debug {
fn width(&self) -> u32;
fn height(&self) -> u32;
fn buffer_type(&self) -> VideoBufferType;
#[cfg(not(target_arch = "wasm32"))]
fn as_native(&self) -> Option<&native::NativeBuffer> {
None
}
fn as_i420(&self) -> Option<&I420Buffer> {
None
}
fn as_i420a(&self) -> Option<&I420ABuffer> {
None
}
fn as_i422(&self) -> Option<&I422Buffer> {
None
}
fn as_i444(&self) -> Option<&I444Buffer> {
None
}
fn as_i010(&self) -> Option<&I010Buffer> {
None
}
fn as_nv12(&self) -> Option<&NV12Buffer> {
None
}
}
macro_rules! new_buffer_type {
($type:ident, $variant:ident, $as:ident) => {
pub struct $type {
pub(crate) handle: vf_imp::$type,
}
impl $crate::video_frame::internal::BufferSealed for $type {
#[cfg(not(target_arch = "wasm32"))]
fn sys_handle(&self) -> &webrtc_sys::video_frame_buffer::ffi::VideoFrameBuffer {
self.handle.sys_handle()
}
#[cfg(not(target_arch = "wasm32"))]
fn to_i420(&self) -> I420Buffer {
I420Buffer { handle: self.handle.to_i420() }
}
#[cfg(not(target_arch = "wasm32"))]
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
stride: u32,
width: i32,
height: i32,
) {
self.handle.to_argb(format, dst, stride, width, height)
}
}
impl VideoBuffer for $type {
fn width(&self) -> u32 {
self.handle.width()
}
fn height(&self) -> u32 {
self.handle.height()
}
fn buffer_type(&self) -> VideoBufferType {
VideoBufferType::$variant
}
fn $as(&self) -> Option<&$type> {
Some(self)
}
}
impl Debug for $type {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct(stringify!($type))
.field("width", &self.width())
.field("height", &self.height())
.finish()
}
}
impl AsRef<dyn VideoBuffer> for $type {
fn as_ref(&self) -> &(dyn VideoBuffer + 'static) {
self
}
}
};
}
new_buffer_type!(I420Buffer, I420, as_i420);
new_buffer_type!(I420ABuffer, I420A, as_i420a);
new_buffer_type!(I422Buffer, I422, as_i422);
new_buffer_type!(I444Buffer, I444, as_i444);
new_buffer_type!(I010Buffer, I010, as_i010);
new_buffer_type!(NV12Buffer, NV12, as_nv12);
impl I420Buffer {
pub fn with_strides(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> I420Buffer {
vf_imp::I420Buffer::new(width, height, stride_y, stride_u, stride_v)
}
pub fn new(width: u32, height: u32) -> I420Buffer {
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32) {
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl I420ABuffer {
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32, u32) {
(
self.handle.stride_y(),
self.handle.stride_u(),
self.handle.stride_v(),
self.handle.stride_a(),
)
}
#[allow(clippy::type_complexity)]
pub fn data(&self) -> (&[u8], &[u8], &[u8], Option<&[u8]>) {
self.handle.data()
}
#[allow(clippy::type_complexity)]
pub fn data_mut(&self) -> (&mut [u8], &mut [u8], &mut [u8], Option<&mut [u8]>) {
let (data_y, data_u, data_v, data_a) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
data_a.map(|data_a| {
std::slice::from_raw_parts_mut(data_a.as_ptr() as *mut u8, data_a.len())
}),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I420ABuffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl I422Buffer {
pub fn with_strides(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> I422Buffer {
vf_imp::I422Buffer::new(width, height, stride_y, stride_u, stride_v)
}
pub fn new(width: u32, height: u32) -> I422Buffer {
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32) {
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I422Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl I444Buffer {
pub fn with_strides(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> I444Buffer {
vf_imp::I444Buffer::new(width, height, stride_y, stride_u, stride_v)
}
pub fn new(width: u32, height: u32) -> I444Buffer {
Self::with_strides(width, height, width, width, width)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32) {
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
}
pub fn data(&self) -> (&[u8], &[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8], &mut [u8]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u8, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u8, data_v.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I444Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl I010Buffer {
pub fn with_strides(
width: u32,
height: u32,
stride_y: u32,
stride_u: u32,
stride_v: u32,
) -> I010Buffer {
vf_imp::I010Buffer::new(width, height, stride_y, stride_u, stride_v)
}
pub fn new(width: u32, height: u32) -> I010Buffer {
Self::with_strides(width, height, width, (width + 1) / 2, (width + 1) / 2)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32, u32) {
(self.handle.stride_y(), self.handle.stride_u(), self.handle.stride_v())
}
pub fn data(&self) -> (&[u16], &[u16], &[u16]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u16], &mut [u16], &mut [u16]) {
let (data_y, data_u, data_v) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u16, data_y.len()),
std::slice::from_raw_parts_mut(data_u.as_ptr() as *mut u16, data_u.len()),
std::slice::from_raw_parts_mut(data_v.as_ptr() as *mut u16, data_v.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> I010Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
impl NV12Buffer {
pub fn with_strides(width: u32, height: u32, stride_y: u32, stride_uv: u32) -> NV12Buffer {
vf_imp::NV12Buffer::new(width, height, stride_y, stride_uv)
}
pub fn new(width: u32, height: u32) -> NV12Buffer {
Self::with_strides(width, height, width, width + width % 2)
}
pub fn chroma_width(&self) -> u32 {
self.handle.chroma_width()
}
pub fn chroma_height(&self) -> u32 {
self.handle.chroma_height()
}
pub fn strides(&self) -> (u32, u32) {
(self.handle.stride_y(), self.handle.stride_uv())
}
pub fn data(&self) -> (&[u8], &[u8]) {
self.handle.data()
}
pub fn data_mut(&mut self) -> (&mut [u8], &mut [u8]) {
let (data_y, data_uv) = self.handle.data();
unsafe {
(
std::slice::from_raw_parts_mut(data_y.as_ptr() as *mut u8, data_y.len()),
std::slice::from_raw_parts_mut(data_uv.as_ptr() as *mut u8, data_uv.len()),
)
}
}
pub fn scale(&mut self, scaled_width: i32, scaled_height: i32) -> NV12Buffer {
self.handle.scale(scaled_width, scaled_height)
}
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::fmt::Debug;
use super::{vf_imp, I420Buffer, VideoBuffer, VideoBufferType, VideoFormatType};
new_buffer_type!(NativeBuffer, Native, as_native);
impl NativeBuffer {
pub fn from_fluxer_d3d11_texture(
handle: u64,
width: u32,
height: u32,
dxgi_format: u32,
) -> Option<Self> {
vf_imp::NativeBuffer::from_fluxer_d3d11_texture(
handle,
width,
height,
dxgi_format,
)
}
#[allow(clippy::too_many_arguments)]
pub fn from_fluxer_dmabuf_texture(
fds: [i32; 4],
plane_count: u32,
width: u32,
height: u32,
drm_format: u32,
modifier: u64,
strides: [u32; 4],
offsets: [u32; 4],
device_uuid_hi: u64,
device_uuid_lo: u64,
) -> Option<Self> {
vf_imp::NativeBuffer::from_fluxer_dmabuf_texture(
fds,
plane_count,
width,
height,
drm_format,
modifier,
strides,
offsets,
device_uuid_hi,
device_uuid_lo,
)
}
/// Creates a `NativeBuffer` from a `CVPixelBufferRef` pointer.
///
/// This function does not bump the reference count of the pixel buffer.
///
/// Safety: The given pointer must be a valid `CVPixelBufferRef`.
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub unsafe fn from_cv_pixel_buffer(cv_pixel_buffer: *mut std::ffi::c_void) -> Self {
vf_imp::NativeBuffer::from_cv_pixel_buffer(cv_pixel_buffer)
}
/// Returns the `CVPixelBufferRef` that backs this buffer, or `null` if
/// this buffer is not currently backed by a `CVPixelBufferRef`.
///
/// This function does not bump the reference count of the pixel buffer.
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void {
self.handle.get_cv_pixel_buffer()
}
}
pub trait VideoFrameBufferExt: VideoBuffer {
fn to_i420(&self) -> I420Buffer;
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
);
}
impl<T: VideoBuffer> VideoFrameBufferExt for T {
fn to_i420(&self) -> I420Buffer {
self.to_i420()
}
fn to_argb(
&self,
format: VideoFormatType,
dst: &mut [u8],
dst_stride: u32,
dst_width: i32,
dst_height: i32,
) {
self.to_argb(format, dst, dst_stride, dst_width, dst_height)
}
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {
use super::VideoFrameBuffer;
#[derive(Debug)]
pub struct WebGlBuffer {}
impl VideoFrameBuffer for WebGlBuffer {}
}
@@ -0,0 +1,97 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{enum_dispatch, imp::video_source as vs_imp};
#[derive(Debug, Clone)]
pub struct VideoResolution {
pub width: u32,
pub height: u32,
}
impl Default for VideoResolution {
// Default to 720p
fn default() -> Self {
VideoResolution { width: 1280, height: 720 }
}
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub enum RtcVideoSource {
// TODO(theomonnom): Web video sources (eq. to tracks on browsers?)
#[cfg(not(target_arch = "wasm32"))]
Native(native::NativeVideoSource),
}
// TODO(theomonnom): Support enum dispatch with conditional compilation?
impl RtcVideoSource {
enum_dispatch!(
[Native];
pub fn video_resolution(self: &Self) -> VideoResolution;
);
}
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::fmt::{Debug, Formatter};
use super::*;
use crate::native::packet_trailer::PacketTrailerHandler;
use crate::video_frame::{VideoBuffer, VideoFrame};
#[derive(Clone)]
pub struct NativeVideoSource {
pub(crate) handle: vs_imp::NativeVideoSource,
}
impl Debug for NativeVideoSource {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeVideoSource").finish()
}
}
impl Default for NativeVideoSource {
fn default() -> Self {
Self::new(VideoResolution::default(), false)
}
}
impl NativeVideoSource {
pub fn new(resolution: VideoResolution, is_screencast: bool) -> Self {
Self { handle: vs_imp::NativeVideoSource::new(resolution, is_screencast) }
}
pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {
self.handle.capture_frame(frame)
}
/// Set the packet trailer handler used by this source.
///
/// When set, any frame captured with a `user_timestamp` value will
/// automatically have its timestamp stored in the handler (keyed by
/// the TimestampAligner-adjusted capture timestamp) so the
/// `PacketTrailerTransformer` can embed it into the encoded frame.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.handle.set_packet_trailer_handler(handler)
}
pub fn video_resolution(&self) -> VideoResolution {
self.handle.video_resolution()
}
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {}
@@ -0,0 +1,123 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::imp::video_stream as stream_imp;
// There is no shared sink between native and web platforms.
// Each platform requires different configuration (e.g: WebGlContext, ..)
#[cfg(not(target_arch = "wasm32"))]
pub mod native {
use std::{
fmt::{Debug, Formatter},
pin::Pin,
task::{Context, Poll},
};
use super::stream_imp;
use crate::{
native::packet_trailer::PacketTrailerHandler, video_frame::BoxVideoFrame,
video_track::RtcVideoTrack,
};
use livekit_runtime::Stream;
const DEFAULT_QUEUE_SIZE_FRAMES: usize = 1;
#[derive(Clone, Debug, Default)]
pub struct NativeVideoStreamOptions {
/// Maximum number of queued WebRTC sink frames after the video callback.
///
/// `None` uses the default bounded queue size of 1 frame. `Some(0)`
/// opts into unbounded buffering. Positive values bound the queue, and
/// the stream drops the oldest queued frames on overflow so render
/// latency stays bounded.
///
/// If your application consumes both audio and video, keep the queue
/// sizing strategy coordinated across both streams. Using a much larger
/// queue, or unbounded buffering, for only one of them can increase
/// end-to-end latency for that stream and cause audio/video drift.
pub queue_size_frames: Option<usize>,
}
pub struct NativeVideoStream {
pub(crate) handle: stream_imp::NativeVideoStream,
}
impl Debug for NativeVideoStream {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
f.debug_struct("NativeVideoStream").field("track", &self.track()).finish()
}
}
impl NativeVideoStream {
pub fn new(video_track: RtcVideoTrack) -> Self {
Self {
handle: stream_imp::NativeVideoStream::new(
video_track,
Some(DEFAULT_QUEUE_SIZE_FRAMES),
),
}
}
pub fn with_options(video_track: RtcVideoTrack, options: NativeVideoStreamOptions) -> Self {
Self {
handle: stream_imp::NativeVideoStream::new(
video_track,
normalize_queue_size_frames(options.queue_size_frames),
),
}
}
/// Set the packet trailer handler for this stream.
///
/// When set, each frame produced by this stream will have its
/// `user_timestamp` field populated by looking up the user
/// timestamp for each frame's RTP timestamp.
///
/// Note: If the handler was already set on the `RtcVideoTrack`
/// before creating this stream, it is automatically wired up.
/// This method is only needed to override or set the handler
/// after construction.
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.handle.set_packet_trailer_handler(handler);
}
pub fn track(&self) -> RtcVideoTrack {
self.handle.track()
}
pub fn close(&mut self) {
self.handle.close();
}
}
impl Stream for NativeVideoStream {
type Item = BoxVideoFrame;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.get_mut().handle).poll_next(cx)
}
}
fn normalize_queue_size_frames(queue_size_frames: Option<usize>) -> Option<usize> {
match queue_size_frames {
None => Some(DEFAULT_QUEUE_SIZE_FRAMES),
Some(0) => None,
Some(value) => Some(value),
}
}
}
#[cfg(target_arch = "wasm32")]
pub mod web {}
@@ -0,0 +1,58 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use crate::{
imp::video_track as imp_vt,
media_stream_track::{media_stream_track, RtcTrackState},
};
#[cfg(not(target_arch = "wasm32"))]
use crate::native::packet_trailer::PacketTrailerHandler;
#[derive(Clone)]
pub struct RtcVideoTrack {
pub(crate) handle: imp_vt::RtcVideoTrack,
}
impl RtcVideoTrack {
media_stream_track!();
/// Set the packet trailer handler for this track.
///
/// When set, any `NativeVideoStream` created from this track will
/// automatically use this handler to populate `user_timestamp`
/// on each decoded frame.
#[cfg(not(target_arch = "wasm32"))]
pub fn set_packet_trailer_handler(&self, handler: PacketTrailerHandler) {
self.handle.set_packet_trailer_handler(handler);
}
/// Get the packet trailer handler, if one has been set.
#[cfg(not(target_arch = "wasm32"))]
pub fn packet_trailer_handler(&self) -> Option<PacketTrailerHandler> {
self.handle.packet_trailer_handler()
}
}
impl Debug for RtcVideoTrack {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RtcVideoTrack")
.field("id", &self.id())
.field("enabled", &self.enabled())
.field("state", &self.state())
.finish()
}
}
@@ -0,0 +1,117 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use core::str;
use std::cell::RefCell;
use std::rc::Rc;
use wasm_bindgen::prelude::*;
use web_sys::{MessageEvent, RtcDataChannelEvent, RtcDataChannelState};
use crate::data_channel::{
DataChannelError, DataChannelTrait, DataState, OnBufferedAmountChange, OnMessage, OnStateChange,
};
impl From<RtcDataChannelState> for DataState {
fn from(value: RtcDataChannelState) -> Self {
match value {
RtcDataChannelState::Connecting => Self::Connecting,
RtcDataChannelState::Open => Self::Open,
RtcDataChannelState::Closing => Self::Closing,
RtcDataChannelState::Closed => Self::Closed,
_ => panic!("unknown data channel state"),
}
}
}
#[derive(Clone)]
pub struct DataChannel {
sys_handle: web_sys::RtcDataChannel,
on_closing: Rc<RefCell<Option<JsValue>>>,
}
impl DataChannelTrait for DataChannel {
fn send(&self, data: &[u8], binary: bool) -> Result<(), DataChannelError> {
if binary {
self.sys_handle
.send_with_u8_array(data)
.map_err(|_| DataChannelError::Send)
} else {
let utf8 = str::from_utf8(data)?;
self.sys_handle
.send_with_str(utf8)
.map_err(|_| DataChannelError::Send)
}
}
fn label(&self) -> String {
self.sys_handle.label()
}
fn state(&self) -> DataState {
self.sys_handle.ready_state().into()
}
fn close(&self) {
self.sys_handle.close();
}
fn on_state_change(&self, callback: Option<OnStateChange>) {
if let Some(mut callback) = callback {
let dc = self.clone();
let js_callback = Closure::new(move |_: RtcDataChannelEvent| {
callback(dc.state());
});
let js_callback = js_callback.into_js_value();
self.sys_handle
.set_onopen(Some(js_callback.unchecked_ref()));
self.sys_handle
.set_onclose(Some(js_callback.unchecked_ref()));
self.sys_handle
.add_event_listener_with_callback("closing", js_callback.unchecked_ref())
.unwrap();
self.on_closing.replace(Some(js_callback));
} else {
self.sys_handle.set_onopen(None);
self.sys_handle.set_onclose(None);
if let Some(on_closing) = self.on_closing.take() {
self.sys_handle
.remove_event_listener_with_callback("closing", on_closing.unchecked_ref())
.unwrap();
}
self.on_closing.replace(None);
}
}
fn on_message(&self, callback: Option<OnMessage>) {
let js_callback = callback.map(|mut callback| {
Closure::new(move |event: MessageEvent| {
if let Some(str) = event.as_string() {
callback(str.as_bytes(), false);
}
})
.into_js_value()
});
self.sys_handle.set_onmessage(
js_callback
.as_ref()
.map(|callback| callback.unchecked_ref()),
);
}
fn on_buffered_amount_change(&self, _callback: Option<OnBufferedAmountChange>) {
todo!("onbufferedamountlow instead?")
}
}
@@ -0,0 +1,372 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::VideoTrack;
use crate::{
media_stream::{
BiplanarYuv8Buffer, BiplanarYuvBuffer, PlanarYuv16BBuffer, PlanarYuv8Buffer,
PlanarYuvBuffer, VideoFrameBuffer,
},
video_frame::{BiplanarYuv8Buffer, I420Buffer, SinkError, VideoFrame, VideoFrameBuffer},
I010Buffer, I420ABuffer, I422Buffer, I444Buffer, NV12Buffer,
};
use std::sync::mpsc;
use web_sys::{WebGlRenderingContext, WebGlTexture};
#[derive(Debug)]
pub struct WebGlVideoSink {
track: Arc<VideoTrack>,
gl_ctx: WebGlRenderingContext,
tex: WebGlTexture,
}
/// Create a new WebGL texture and update it inside requestAnimationFrame
impl WebGlVideoSink {
pub fn new(
track: Arc<VideoTrack>,
gl_ctx: WebGlRenderingContext,
) -> Result<(Self, mpsc::Receiver<VideoFrame<WebGlBuffer>>), SinkError> {
let (sender, receiver) = mpsc::channel();
let tex = gl_ctx.create_texture()?;
Ok((Self { track, gl_ctx, tex }, receiver))
}
}
#[derive(Debug, Clone)]
pub struct WebGlBuffer {
width: i32,
height: i32,
tex: WebGlTexture,
}
impl VideoFrameBuffer for WebGlBuffer {
fn width(&self) -> i32 {
self.width
}
fn height(&self) -> i32 {
self.height
}
}
/// The following types could be implemented if we want
/// to support VideoFrame with WebCodecs
#[derive(Debug)]
pub struct I420Buffer {}
#[derive(Debug)]
pub struct I420ABuffer {}
#[derive(Debug)]
pub struct I422Buffer {}
#[derive(Debug)]
pub struct I444Buffer {}
#[derive(Debug)]
pub struct I010Buffer {}
#[derive(Debug)]
pub struct NV12Buffer {}
impl VideoFrameBuffer for I420Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I420ABuffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I422Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I444Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for I010Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl VideoFrameBuffer for NV12Buffer {
fn width(&self) -> i32 {
unimplemented!()
}
fn height(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I420Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I420ABuffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I422Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I444Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for I010Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuvBuffer for NV12Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_u(&self) -> i32 {
unimplemented!()
}
fn stride_v(&self) -> i32 {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I420Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I420ABuffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I422Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv8Buffer for I444Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_u(&self) -> &[u8] {
unimplemented!()
}
fn data_v(&self) -> &[u8] {
unimplemented!()
}
}
impl PlanarYuv16BBuffer for I010Buffer {
fn data_y(&self) -> &[u16] {
unimplemented!()
}
fn data_u(&self) -> &[u16] {
unimplemented!()
}
fn data_v(&self) -> &[u16] {
unimplemented!()
}
}
impl BiplanarYuvBuffer for NV12Buffer {
fn chroma_width(&self) -> i32 {
unimplemented!()
}
fn chroma_height(&self) -> i32 {
unimplemented!()
}
fn stride_y(&self) -> i32 {
unimplemented!()
}
fn stride_uv(&self) -> i32 {
unimplemented!()
}
}
impl BiplanarYuv8Buffer for NV12Buffer {
fn data_y(&self) -> &[u8] {
unimplemented!()
}
fn data_uv(&self) -> &[u8] {
unimplemented!()
}
}
@@ -0,0 +1,15 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
unimplemented!();