Add native self-hosted instance connection to fluxer_desktop

Trimmed monorepo checkout (fluxer_desktop + packages/voice_engine_v2 +
tools/ci) with a "Connect to a Different Server" menu item and popout
that lets the desktop app switch to any self-hosted Fluxer instance,
plus fixes for well-known discovery on single-domain self-hosted
deployments and a false-positive ERR_ABORTED on same-origin client
redirects during the switch. Defaults to chat.fluxr.chat and uses an
isolated userData directory from the official build.
This commit is contained in:
2026-07-01 18:22:43 -04:00
commit 682afacd30
1763 changed files with 613720 additions and 0 deletions
@@ -0,0 +1,926 @@
/*
* Copyright 2026 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/adm_proxy.h"
#include "api/audio/audio_device.h"
#include "api/audio/create_audio_device_module.h"
#include "api/make_ref_counted.h"
#include "rtc_base/logging.h"
#include "rtc_base/thread.h"
#if defined(__ANDROID__)
#include <jni.h>
#include "sdk/android/native_api/audio_device_module/audio_device_android.h"
#include "sdk/android/native_api/base/init.h"
#endif
namespace livekit_ffi {
AdmProxy::AdmProxy(const webrtc::Environment& env, webrtc::Thread* worker_thread)
: env_(env),
worker_thread_(worker_thread) {
// Create the synthetic ADM for synthetic mode. SyntheticAudioDevice pumps
// the WebRTC audio pipeline without platform audio, allowing FFI callbacks
// to receive decoded remote audio.
synthetic_adm_ = webrtc::make_ref_counted<SyntheticAudioDevice>(env_);
if (synthetic_adm_->Init() != 0) {
RTC_LOG(LS_ERROR) << "AdmProxy: Failed to initialize synthetic ADM";
}
// Attempt to create the Platform ADM for real audio I/O.
// The eager attempt exists for iOS compatibility: the iOS audio session
// requires early setup to avoid KVO race conditions.
// On Android, we defer Platform ADM creation to AcquirePlatformAdm().
// This is because:
// 1. CreateAudioDeviceModule requires JNI to be fully initialized
// 2. The JNI initialization (via JNI_OnLoad or manual init) may not have
// completed by the time the AdmProxy constructor runs
// 3. Deferring creation ensures JNI is ready when we actually need the ADM
// On all platforms a failed attempt is retried later by
// EnsurePlatformAdmCreated(): desktop apps launched at login can race the
// OS audio stack (coreaudiod, Windows audio services), and a one-shot
// creation would leave audio permanently broken for the process lifetime.
#if defined(__ANDROID__)
// platform_adm_ stays nullptr, will be created in EnsurePlatformAdmCreated()
#else
webrtc::MutexLock lock(&mutex_);
if (!EnsurePlatformAdmCreated()) {
RTC_LOG(LS_WARNING)
<< "AdmProxy: Platform ADM unavailable at construction; will retry on demand";
}
#endif
}
AdmProxy::~AdmProxy() {
RTC_LOG(LS_VERBOSE) << "AdmProxy::~AdmProxy()";
if (synthetic_adm_) {
synthetic_adm_->Terminate();
synthetic_adm_ = nullptr;
}
if (platform_adm_) {
platform_adm_->Terminate();
platform_adm_ = nullptr;
}
}
// =============================================================================
// Helper Methods
// =============================================================================
bool AdmProxy::is_platform_playout_active() const {
// Platform playout is active when: ref_count > 0 AND playout explicitly enabled.
// Otherwise, synthetic mode handles playout via the internal pumping task.
return platform_adm_ && platform_adm_ref_count_ > 0 && playout_enabled_;
}
webrtc::AudioDeviceModule* AdmProxy::recording_adm() const {
// Recording only available through platform ADM when enabled.
// Synthetic mode doesn't support recording (no microphone).
if (platform_adm_ && platform_adm_ref_count_ > 0 && recording_enabled_) {
return platform_adm_.get();
}
return nullptr;
}
// =============================================================================
// Platform ADM Lifecycle Management
// =============================================================================
// Lazily creates the Platform ADM. Must be called with mutex held.
// Returns true if ADM is available (either already existed or successfully created).
// A failed attempt leaves platform_adm_ null so the next call retries; the OS
// audio stack may simply not be ready yet (cold boot, login launch).
bool AdmProxy::EnsurePlatformAdmCreated() {
if (platform_adm_) {
return true; // Already created
}
#if defined(__ANDROID__)
// Use CreateAndroidAudioDeviceModule which properly uses GetAppContext()
// to get the application context set via ContextUtils.initialize().
platform_adm_ = webrtc::CreateAndroidAudioDeviceModule(
env_, webrtc::AudioDeviceModule::kPlatformDefaultAudio);
if (!platform_adm_) {
RTC_LOG(LS_ERROR) << "AdmProxy: CreateAndroidAudioDeviceModule returned nullptr. "
<< "Ensure ContextUtils.initialize() was called.";
return false;
}
#else
platform_adm_ = webrtc::CreateAudioDeviceModule(
env_, webrtc::AudioDeviceModule::kPlatformDefaultAudio);
if (!platform_adm_) {
RTC_LOG(LS_ERROR) << "AdmProxy: CreateAudioDeviceModule returned nullptr";
return false;
}
#endif
int32_t init_result = platform_adm_->Init();
if (init_result != 0) {
RTC_LOG(LS_ERROR) << "AdmProxy: Platform ADM Init() failed with error=" << init_result;
platform_adm_ = nullptr;
return false;
}
RestorePlatformAdmStateLocked();
return true;
}
// Re-applies state that may have been recorded before the Platform ADM
// existed: the audio transport registered by WebRTC and any device selection.
// Without this, an ADM created after RegisterAudioCallback() would never
// receive audio data.
void AdmProxy::RestorePlatformAdmStateLocked() {
RTC_DCHECK(platform_adm_);
if (audio_transport_) {
recording_transport_proxy_.set_real_transport(audio_transport_);
platform_adm_->RegisterAudioCallback(&recording_transport_proxy_);
}
if (!selected_playout_guid_.empty()) {
int16_t count = platform_adm_->PlayoutDevices();
for (int16_t i = 0; i < count; i++) {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char guid[webrtc::kAdmMaxGuidSize] = {0};
if (platform_adm_->PlayoutDeviceName(static_cast<uint16_t>(i), name, guid) == 0 &&
selected_playout_guid_ == guid) {
platform_adm_->SetPlayoutDevice(static_cast<uint16_t>(i));
break;
}
}
}
if (!selected_recording_guid_.empty()) {
int16_t count = platform_adm_->RecordingDevices();
for (int16_t i = 0; i < count; i++) {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char guid[webrtc::kAdmMaxGuidSize] = {0};
if (platform_adm_->RecordingDeviceName(static_cast<uint16_t>(i), name, guid) == 0 &&
selected_recording_guid_ == guid) {
platform_adm_->SetRecordingDevice(static_cast<uint16_t>(i));
break;
}
}
}
}
bool AdmProxy::EnsurePlatformAdm() {
webrtc::MutexLock lock(&mutex_);
return EnsurePlatformAdmCreated();
}
bool AdmProxy::platform_adm_available() const {
webrtc::MutexLock lock(&mutex_);
return platform_adm_ != nullptr;
}
bool AdmProxy::AcquirePlatformAdm() {
webrtc::MutexLock lock(&mutex_);
// Lazily create the Platform ADM on first acquire (and retry after any
// earlier failed attempt, e.g. when the OS audio stack was still starting).
if (!EnsurePlatformAdmCreated()) {
RTC_LOG(LS_ERROR) << "AdmProxy::AcquirePlatformAdm() - Failed to create Platform ADM";
return false;
}
int old_ref_count = platform_adm_ref_count_;
platform_adm_ref_count_++;
// If this is the first acquisition and playout/recording is enabled,
// we may need to switch from synthetic mode to platform ADM
if (old_ref_count == 0) {
SwitchPlayoutModeIfNeeded();
SwitchRecordingAdmIfNeeded();
}
return true;
}
void AdmProxy::ReleasePlatformAdm() {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_ref_count_ <= 0) {
RTC_LOG(LS_WARNING) << "AdmProxy::ReleasePlatformAdm() called with ref_count="
<< platform_adm_ref_count_;
return;
}
platform_adm_ref_count_--;
// If ref_count reaches 0, switch back from platform ADM to synthetic mode
// Note: We do NOT terminate the Platform ADM - it stays alive until destructor.
// This avoids iOS KVO race conditions from re-creating the ADM.
if (platform_adm_ref_count_ == 0) {
SwitchPlayoutModeIfNeeded();
SwitchRecordingAdmIfNeeded();
}
}
int AdmProxy::platform_adm_ref_count() const {
webrtc::MutexLock lock(&mutex_);
return platform_adm_ref_count_;
}
bool AdmProxy::is_platform_adm_active() const {
webrtc::MutexLock lock(&mutex_);
// Platform ADM is considered active when there are users and playout/recording is enabled
return platform_adm_ != nullptr && platform_adm_ref_count_ > 0;
}
// =============================================================================
// Recording/Playout Control
// =============================================================================
void AdmProxy::set_recording_enabled(bool enabled) {
webrtc::MutexLock lock(&mutex_);
if (recording_enabled_ == enabled) {
return;
}
recording_enabled_ = enabled;
SwitchRecordingAdmIfNeeded();
}
bool AdmProxy::recording_enabled() const {
webrtc::MutexLock lock(&mutex_);
return recording_enabled_;
}
void AdmProxy::set_playout_enabled(bool enabled) {
webrtc::MutexLock lock(&mutex_);
if (playout_enabled_ == enabled) {
return;
}
playout_enabled_ = enabled;
SwitchPlayoutModeIfNeeded();
}
bool AdmProxy::playout_enabled() const {
webrtc::MutexLock lock(&mutex_);
return playout_enabled_;
}
// =============================================================================
// Mode Switching Helpers (called with mutex held)
// =============================================================================
void AdmProxy::SwitchPlayoutModeIfNeeded() {
if (!playing_) return;
bool use_platform = is_platform_playout_active();
if (use_platform) {
// Switch to platform mode - stop synthetic, start platform ADM
if (synthetic_adm_) {
synthetic_adm_->StopPlayout();
}
if (platform_adm_) {
platform_adm_->InitPlayout();
platform_adm_->StartPlayout();
}
} else {
// Switch to synthetic mode - stop platform ADM, start synthetic ADM
if (platform_adm_) {
platform_adm_->StopPlayout();
}
if (synthetic_adm_) {
synthetic_adm_->StartPlayout();
}
}
}
void AdmProxy::SwitchRecordingAdmIfNeeded() {
if (!recording_) return;
// Stop platform ADM recording (only one that supports recording)
if (platform_adm_) platform_adm_->StopRecording();
// Start if new ADM supports recording
auto* adm = recording_adm();
if (adm) {
adm->InitRecording();
adm->StartRecording();
} else {
recording_ = false;
}
}
// =============================================================================
// AudioDeviceModule Interface Implementation
// =============================================================================
int32_t AdmProxy::ActiveAudioLayer(AudioLayer* audioLayer) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->ActiveAudioLayer(audioLayer);
}
*audioLayer = AudioLayer::kDummyAudio;
return 0;
}
int32_t AdmProxy::RegisterAudioCallback(webrtc::AudioTransport* transport) {
webrtc::MutexLock lock(&mutex_);
audio_transport_ = transport;
recording_transport_proxy_.set_real_transport(transport);
// Register the interposing proxy with both ADMs so they're ready when we
// switch modes. The proxy tees recorded frames and forwards to `transport`.
webrtc::AudioTransport* proxy = &recording_transport_proxy_;
if (synthetic_adm_) {
synthetic_adm_->RegisterAudioCallback(proxy);
}
if (platform_adm_) {
platform_adm_->RegisterAudioCallback(proxy);
}
return 0;
}
int32_t AdmProxy::Init() {
// Init is a no-op - Platform ADM is created lazily via AcquirePlatformAdm()
return 0;
}
int32_t AdmProxy::Terminate() {
webrtc::MutexLock lock(&mutex_);
int32_t result = 0;
if (synthetic_adm_) {
result = synthetic_adm_->Terminate();
}
if (platform_adm_) {
int32_t platform_result = platform_adm_->Terminate();
if (result == 0) result = platform_result;
}
return result;
}
bool AdmProxy::Initialized() const {
webrtc::MutexLock lock(&mutex_);
// We're initialized if at least one ADM is initialized
bool synthetic_init = synthetic_adm_ && synthetic_adm_->Initialized();
bool platform_init = platform_adm_ && platform_adm_->Initialized();
return synthetic_init || platform_init;
}
int16_t AdmProxy::PlayoutDevices() {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->PlayoutDevices();
}
// In synthetic mode, return 0 devices (no platform audio)
return 0;
}
int16_t AdmProxy::RecordingDevices() {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->RecordingDevices();
}
// In synthetic mode, return 0 devices (no platform audio)
return 0;
}
int32_t AdmProxy::PlayoutDeviceName(uint16_t index,
char name[webrtc::kAdmMaxDeviceNameSize],
char guid[webrtc::kAdmMaxGuidSize]) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->PlayoutDeviceName(index, name, guid);
}
return -1;
}
int32_t AdmProxy::RecordingDeviceName(uint16_t index,
char name[webrtc::kAdmMaxDeviceNameSize],
char guid[webrtc::kAdmMaxGuidSize]) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->RecordingDeviceName(index, name, guid);
}
return -1;
}
int32_t AdmProxy::SetPlayoutDevice(uint16_t index) {
webrtc::MutexLock lock(&mutex_);
selected_playout_device_ = index;
// Also store the GUID for this device for robust restoration
if (platform_adm_) {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char guid[webrtc::kAdmMaxGuidSize] = {0};
if (platform_adm_->PlayoutDeviceName(index, name, guid) == 0) {
selected_playout_guid_ = guid;
}
return platform_adm_->SetPlayoutDevice(index);
}
return 0;
}
int32_t AdmProxy::SetPlayoutDevice(WindowsDeviceType device) {
webrtc::MutexLock lock(&mutex_);
// Note: When using WindowsDeviceType, we can't easily get the GUID
// The GUID will be populated on next CreatePlatformAdm if needed
selected_playout_guid_.clear();
if (platform_adm_) {
return platform_adm_->SetPlayoutDevice(device);
}
return 0;
}
int32_t AdmProxy::SetRecordingDevice(uint16_t index) {
webrtc::MutexLock lock(&mutex_);
selected_recording_device_ = index;
// Also store the GUID for this device for robust restoration
if (platform_adm_) {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char guid[webrtc::kAdmMaxGuidSize] = {0};
if (platform_adm_->RecordingDeviceName(index, name, guid) == 0) {
selected_recording_guid_ = guid;
}
return platform_adm_->SetRecordingDevice(index);
}
return 0;
}
int32_t AdmProxy::SetRecordingDevice(WindowsDeviceType device) {
webrtc::MutexLock lock(&mutex_);
// Note: When using WindowsDeviceType, we can't easily get the GUID
// The GUID will be populated on next CreatePlatformAdm if needed
selected_recording_guid_.clear();
if (platform_adm_) {
return platform_adm_->SetRecordingDevice(device);
}
return 0;
}
int32_t AdmProxy::PlayoutIsAvailable(bool* available) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->PlayoutIsAvailable(available);
}
*available = true; // Synthetic playout is always available
return 0;
}
int32_t AdmProxy::InitPlayout() {
webrtc::MutexLock lock(&mutex_);
if (is_platform_playout_active()) {
if (platform_adm_) {
int32_t result = platform_adm_->InitPlayout();
if (result == 0) {
playout_initialized_ = true;
}
return result;
}
return -1;
}
// Synthetic mode
if (synthetic_adm_) {
int32_t result = synthetic_adm_->InitPlayout();
if (result == 0) {
playout_initialized_ = true;
}
return result;
}
return -1;
}
bool AdmProxy::PlayoutIsInitialized() const {
webrtc::MutexLock lock(&mutex_);
if (is_platform_playout_active()) {
return platform_adm_ && platform_adm_->PlayoutIsInitialized();
}
// Synthetic mode
return synthetic_adm_ && synthetic_adm_->PlayoutIsInitialized();
}
int32_t AdmProxy::RecordingIsAvailable(bool* available) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->RecordingIsAvailable(available);
}
*available = false; // Recording not available in synthetic mode
return 0;
}
int32_t AdmProxy::InitRecording() {
webrtc::MutexLock lock(&mutex_);
auto* adm = recording_adm();
if (!adm) {
// Recording not available (no platform ADM or recording disabled)
// Return success to avoid breaking WebRTC's initialization flow
return 0;
}
int32_t result = adm->InitRecording();
if (result == 0) {
recording_initialized_ = true;
}
return result;
}
bool AdmProxy::RecordingIsInitialized() const {
webrtc::MutexLock lock(&mutex_);
auto* adm = recording_adm();
if (adm) {
return adm->RecordingIsInitialized();
}
return false; // Recording not available
}
int32_t AdmProxy::StartPlayout() {
webrtc::MutexLock lock(&mutex_);
playing_ = true;
if (is_platform_playout_active()) {
if (platform_adm_) {
return platform_adm_->StartPlayout();
}
return -1;
}
// Synthetic mode
if (synthetic_adm_) {
return synthetic_adm_->StartPlayout();
}
return -1;
}
int32_t AdmProxy::StopPlayout() {
webrtc::MutexLock lock(&mutex_);
playing_ = false;
// Stop both ADMs
if (synthetic_adm_) {
synthetic_adm_->StopPlayout();
}
if (platform_adm_) {
platform_adm_->StopPlayout();
}
return 0;
}
bool AdmProxy::Playing() const {
webrtc::MutexLock lock(&mutex_);
if (is_platform_playout_active()) {
return platform_adm_ && platform_adm_->Playing();
}
return synthetic_adm_ && synthetic_adm_->Playing();
}
int32_t AdmProxy::StartRecording() {
webrtc::MutexLock lock(&mutex_);
auto* adm = recording_adm();
if (!adm) {
// Recording not available - return success to avoid breaking WebRTC
return 0;
}
recording_ = true;
return adm->StartRecording();
}
int32_t AdmProxy::StopRecording() {
webrtc::MutexLock lock(&mutex_);
recording_ = false;
auto* adm = recording_adm();
if (adm) {
return adm->StopRecording();
}
return 0;
}
bool AdmProxy::Recording() const {
webrtc::MutexLock lock(&mutex_);
auto* adm = recording_adm();
if (adm) {
return adm->Recording();
}
return false;
}
int32_t AdmProxy::InitSpeaker() {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->InitSpeaker();
}
return 0;
}
bool AdmProxy::SpeakerIsInitialized() const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SpeakerIsInitialized();
}
return true;
}
int32_t AdmProxy::InitMicrophone() {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->InitMicrophone();
}
return 0;
}
bool AdmProxy::MicrophoneIsInitialized() const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MicrophoneIsInitialized();
}
return false;
}
int32_t AdmProxy::SpeakerVolumeIsAvailable(bool* available) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SpeakerVolumeIsAvailable(available);
}
*available = false;
return 0;
}
int32_t AdmProxy::SetSpeakerVolume(uint32_t volume) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SetSpeakerVolume(volume);
}
return -1;
}
int32_t AdmProxy::SpeakerVolume(uint32_t* volume) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SpeakerVolume(volume);
}
return -1;
}
int32_t AdmProxy::MaxSpeakerVolume(uint32_t* maxVolume) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MaxSpeakerVolume(maxVolume);
}
return -1;
}
int32_t AdmProxy::MinSpeakerVolume(uint32_t* minVolume) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MinSpeakerVolume(minVolume);
}
return -1;
}
int32_t AdmProxy::MicrophoneVolumeIsAvailable(bool* available) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MicrophoneVolumeIsAvailable(available);
}
*available = false;
return 0;
}
int32_t AdmProxy::SetMicrophoneVolume(uint32_t volume) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SetMicrophoneVolume(volume);
}
return -1;
}
int32_t AdmProxy::MicrophoneVolume(uint32_t* volume) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MicrophoneVolume(volume);
}
return -1;
}
int32_t AdmProxy::MaxMicrophoneVolume(uint32_t* maxVolume) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MaxMicrophoneVolume(maxVolume);
}
return -1;
}
int32_t AdmProxy::MinMicrophoneVolume(uint32_t* minVolume) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MinMicrophoneVolume(minVolume);
}
return -1;
}
int32_t AdmProxy::SpeakerMuteIsAvailable(bool* available) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SpeakerMuteIsAvailable(available);
}
*available = false;
return 0;
}
int32_t AdmProxy::SetSpeakerMute(bool enable) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SetSpeakerMute(enable);
}
return -1;
}
int32_t AdmProxy::SpeakerMute(bool* enabled) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SpeakerMute(enabled);
}
return -1;
}
int32_t AdmProxy::MicrophoneMuteIsAvailable(bool* available) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MicrophoneMuteIsAvailable(available);
}
*available = false;
return 0;
}
int32_t AdmProxy::SetMicrophoneMute(bool enable) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SetMicrophoneMute(enable);
}
return -1;
}
int32_t AdmProxy::MicrophoneMute(bool* enabled) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->MicrophoneMute(enabled);
}
return -1;
}
int32_t AdmProxy::StereoPlayoutIsAvailable(bool* available) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->StereoPlayoutIsAvailable(available);
}
*available = true;
return 0;
}
int32_t AdmProxy::SetStereoPlayout(bool enable) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SetStereoPlayout(enable);
}
return 0;
}
int32_t AdmProxy::StereoPlayout(bool* enabled) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->StereoPlayout(enabled);
}
*enabled = true;
return 0;
}
int32_t AdmProxy::StereoRecordingIsAvailable(bool* available) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->StereoRecordingIsAvailable(available);
}
*available = false;
return 0;
}
int32_t AdmProxy::SetStereoRecording(bool enable) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SetStereoRecording(enable);
}
return 0;
}
int32_t AdmProxy::StereoRecording(bool* enabled) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->StereoRecording(enabled);
}
*enabled = false;
return 0;
}
int32_t AdmProxy::PlayoutDelay(uint16_t* delayMS) const {
webrtc::MutexLock lock(&mutex_);
if (is_platform_playout_active()) {
if (platform_adm_) {
return platform_adm_->PlayoutDelay(delayMS);
}
} else if (synthetic_adm_) {
return synthetic_adm_->PlayoutDelay(delayMS);
}
*delayMS = 0;
return 0;
}
bool AdmProxy::BuiltInAECIsAvailable() const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->BuiltInAECIsAvailable();
}
return false;
}
bool AdmProxy::BuiltInAGCIsAvailable() const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->BuiltInAGCIsAvailable();
}
return false;
}
bool AdmProxy::BuiltInNSIsAvailable() const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->BuiltInNSIsAvailable();
}
return false;
}
int32_t AdmProxy::EnableBuiltInAEC(bool enable) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->EnableBuiltInAEC(enable);
}
return -1;
}
int32_t AdmProxy::EnableBuiltInAGC(bool enable) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->EnableBuiltInAGC(enable);
}
return -1;
}
int32_t AdmProxy::EnableBuiltInNS(bool enable) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->EnableBuiltInNS(enable);
}
return -1;
}
#if defined(WEBRTC_IOS)
int AdmProxy::GetPlayoutAudioParameters(webrtc::AudioParameters* params) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->GetPlayoutAudioParameters(params);
}
return -1;
}
int AdmProxy::GetRecordAudioParameters(webrtc::AudioParameters* params) const {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->GetRecordAudioParameters(params);
}
return -1;
}
#endif
int32_t AdmProxy::SetObserver(webrtc::AudioDeviceObserver* observer) {
webrtc::MutexLock lock(&mutex_);
if (platform_adm_) {
return platform_adm_->SetObserver(observer);
}
return 0;
}
} // namespace livekit_ffi
@@ -0,0 +1,150 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/android.h"
#include <atomic>
#include <jni.h>
#include <stdio.h>
#include <unistd.h>
#include <memory>
#include "api/video_codecs/video_decoder_factory.h"
#include "rtc_base/logging.h"
#include "sdk/android/native_api/base/init.h"
#include "sdk/android/native_api/codecs/wrapper.h"
#include "sdk/android/native_api/jni/class_loader.h"
#include "sdk/android/native_api/jni/scoped_java_ref.h"
#include "sdk/android/src/jni/jni_helpers.h"
// When compiling the examples app on Android, the linker complains that
// `stdout` and `stderr` symbols cannot be found. The previous workaround
// referenced `__sF`, but that symbol was removed in NDK 28+.
// Use POSIX `fdopen()` with the standard file descriptors instead — works
// across all NDK versions.
#undef stdout
FILE *stdout = fdopen(STDOUT_FILENO, "w");
#undef stderr
FILE *stderr = fdopen(STDERR_FILENO, "w");
namespace livekit_ffi {
// Track whether Android WebRTC has been initialized to prevent crashes on double-init.
static std::atomic<bool> g_android_initialized{false};
void init_android(JavaVM* jvm) {
// Idempotent - safe to call multiple times
if (g_android_initialized.exchange(true)) {
RTC_LOG(LS_INFO) << "livekit_ffi::init_android() - already initialized, skipping";
return;
}
RTC_LOG(LS_INFO) << "livekit_ffi::init_android() called with jvm=" << (jvm ? "valid" : "null");
if (!jvm) {
RTC_LOG(LS_ERROR) << "livekit_ffi::init_android() - JavaVM is null! Cannot initialize Android WebRTC.";
g_android_initialized.store(false);
return;
}
webrtc::InitAndroid(jvm);
RTC_LOG(LS_INFO) << "livekit_ffi::init_android() - webrtc::InitAndroid() completed";
}
bool init_android_context(JavaVM* jvm, uintptr_t context_ptr) {
RTC_LOG(LS_INFO) << "livekit_ffi::init_android_context() called";
if (!jvm || !context_ptr) {
RTC_LOG(LS_ERROR) << "livekit_ffi::init_android_context() - jvm or context is null";
return false;
}
// Initialize JVM first (idempotent - WebRTC handles double-init internally)
init_android(jvm);
// Cast uintptr_t back to jobject
jobject context = reinterpret_cast<jobject>(context_ptr);
JNIEnv* env = webrtc::AttachCurrentThreadIfNeeded();
if (!env) {
RTC_LOG(LS_ERROR) << "livekit_ffi::init_android_context() - Failed to attach to JNI";
return false;
}
// Find livekit.org.webrtc.ContextUtils class
jclass context_utils_class = env->FindClass("livekit/org/webrtc/ContextUtils");
if (!context_utils_class) {
RTC_LOG(LS_ERROR) << "livekit_ffi::init_android_context() - Failed to find ContextUtils class";
env->ExceptionClear();
return false;
}
// Get the initialize method
jmethodID initialize_method = env->GetStaticMethodID(
context_utils_class, "initialize", "(Landroid/content/Context;)V");
if (!initialize_method) {
RTC_LOG(LS_ERROR) << "livekit_ffi::init_android_context() - Failed to find initialize method";
env->ExceptionClear();
env->DeleteLocalRef(context_utils_class);
return false;
}
// Call ContextUtils.initialize(context)
env->CallStaticVoidMethod(context_utils_class, initialize_method, context);
// Check for exceptions
if (env->ExceptionCheck()) {
RTC_LOG(LS_ERROR) << "livekit_ffi::init_android_context() - Exception during initialize";
env->ExceptionDescribe();
env->ExceptionClear();
env->DeleteLocalRef(context_utils_class);
return false;
}
env->DeleteLocalRef(context_utils_class);
RTC_LOG(LS_INFO) << "livekit_ffi::init_android_context() - ContextUtils initialized successfully";
return true;
}
std::unique_ptr<webrtc::VideoEncoderFactory>
CreateAndroidVideoEncoderFactory() {
JNIEnv* env = webrtc::AttachCurrentThreadIfNeeded();
webrtc::ScopedJavaLocalRef<jclass> factory_class =
webrtc::GetClass(env, "livekit/org/webrtc/DefaultVideoEncoderFactory");
jmethodID ctor = env->GetMethodID(factory_class.obj(), "<init>",
"(Llivekit/org/webrtc/EglBase$Context;ZZ)V");
jobject encoder_factory =
env->NewObject(factory_class.obj(), ctor, nullptr, true, false);
return webrtc::JavaToNativeVideoEncoderFactory(env, encoder_factory);
}
std::unique_ptr<webrtc::VideoDecoderFactory>
CreateAndroidVideoDecoderFactory() {
JNIEnv* env = webrtc::AttachCurrentThreadIfNeeded();
webrtc::ScopedJavaLocalRef<jclass> factory_class =
webrtc::GetClass(env, "livekit/org/webrtc/WrappedVideoDecoderFactory");
jmethodID ctor = env->GetMethodID(factory_class.obj(), "<init>",
"(Llivekit/org/webrtc/EglBase$Context;)V");
jobject decoder_factory = env->NewObject(factory_class.obj(), ctor, nullptr);
return webrtc::JavaToNativeVideoDecoderFactory(env, decoder_factory);
}
} // namespace livekit_ffi
@@ -0,0 +1,43 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(target_os = "android")]
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
unsafe extern "C++" {
include!("livekit/android.h");
type JavaVM;
/// Initialize Android WebRTC with the JVM.
/// Called automatically by init_android_context(), so only call directly
/// in JNI_OnLoad or when you don't have an Android Context.
/// Idempotent - safe to call multiple times.
unsafe fn init_android(vm: *mut JavaVM);
/// Initialize Android WebRTC with the application context.
/// This is the main init function - calls init_android() internally,
/// then initializes ContextUtils for PlatformAudio.
/// Idempotent - safe to call multiple times.
///
/// # Arguments
/// * `jvm` - The JavaVM pointer
/// * `context` - The Android application context (jobject as usize)
///
/// # Returns
/// true if context init succeeded, false otherwise.
/// Note: JVM init always happens regardless of return value.
unsafe fn init_android_context(jvm: *mut JavaVM, context: usize) -> bool;
}
}
@@ -0,0 +1,73 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/apm.h"
#include "api/audio/builtin_audio_processing_builder.h"
#include "api/environment/environment_factory.h"
#include <iostream>
#include <memory>
namespace livekit_ffi {
AudioProcessingModule::AudioProcessingModule(
const AudioProcessingConfig& config) {
apm_ = webrtc::BuiltinAudioProcessingBuilder()
.Build(webrtc::CreateEnvironment());
apm_->ApplyConfig(config.ToWebrtcConfig());
apm_->Initialize();
}
int AudioProcessingModule::process_stream(const int16_t* src,
size_t src_len,
int16_t* dst,
size_t dst_len,
int sample_rate,
int num_channels) {
webrtc::StreamConfig stream_cfg(sample_rate, num_channels);
return apm_->ProcessStream(src, stream_cfg, stream_cfg, dst);
}
int AudioProcessingModule::process_reverse_stream(const int16_t* src,
size_t src_len,
int16_t* dst,
size_t dst_len,
int sample_rate,
int num_channels) {
webrtc::StreamConfig stream_cfg(sample_rate, num_channels);
return apm_->ProcessReverseStream(src, stream_cfg, stream_cfg, dst);
}
int AudioProcessingModule::set_stream_delay_ms(int delay_ms) {
return apm_->set_stream_delay_ms(delay_ms);
}
std::unique_ptr<AudioProcessingModule> create_apm(
bool echo_canceller_enabled,
bool gain_controller_enabled,
bool high_pass_filter_enabled,
bool noise_suppression_enabled) {
AudioProcessingConfig config;
config.echo_canceller_enabled = echo_canceller_enabled;
config.gain_controller_enabled = gain_controller_enabled;
config.high_pass_filter_enabled = high_pass_filter_enabled;
config.noise_suppression_enabled = noise_suppression_enabled;
return std::make_unique<AudioProcessingModule>(config);
}
} // namespace livekit_ffi
@@ -0,0 +1,55 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
unsafe extern "C++" {
include!("livekit/apm.h");
type AudioProcessingModule;
unsafe fn process_stream(
self: Pin<&mut AudioProcessingModule>,
src: *const i16,
src_len: usize,
dst: *mut i16,
dst_len: usize,
sample_rate: i32,
num_channels: i32,
) -> i32;
unsafe fn process_reverse_stream(
self: Pin<&mut AudioProcessingModule>,
src: *const i16,
src_len: usize,
dst: *mut i16,
dst_len: usize,
sample_rate: i32,
num_channels: i32,
) -> i32;
fn set_stream_delay_ms(self: Pin<&mut AudioProcessingModule>, delay: i32) -> i32;
fn create_apm(
echo_canceller_enabled: bool,
gain_controller_enabled: bool,
high_pass_filter_enabled: bool,
noise_suppression_enabled: bool,
) -> UniquePtr<AudioProcessingModule>;
}
}
impl_thread_safety!(ffi::AudioProcessingModule, Send + Sync);
@@ -0,0 +1,216 @@
/*
* Copyright 2026 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/audio_device_controller.h"
#include <string>
#include <utility>
namespace livekit_ffi {
AudioDeviceController::AudioDeviceController(
webrtc::scoped_refptr<AdmProxy> adm_proxy)
: adm_proxy_(std::move(adm_proxy)) {}
int16_t AudioDeviceController::playout_devices() const {
return adm_proxy_->PlayoutDevices();
}
int16_t AudioDeviceController::recording_devices() const {
return adm_proxy_->RecordingDevices();
}
rust::String AudioDeviceController::playout_device_name(uint16_t index) const {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char guid[webrtc::kAdmMaxGuidSize] = {0};
adm_proxy_->PlayoutDeviceName(index, name, guid);
return rust::String(name);
}
rust::String AudioDeviceController::recording_device_name(uint16_t index) const {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char guid[webrtc::kAdmMaxGuidSize] = {0};
adm_proxy_->RecordingDeviceName(index, name, guid);
return rust::String(name);
}
rust::String AudioDeviceController::playout_device_guid(uint16_t index) const {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char guid[webrtc::kAdmMaxGuidSize] = {0};
adm_proxy_->PlayoutDeviceName(index, name, guid);
return rust::String(guid);
}
rust::String AudioDeviceController::recording_device_guid(uint16_t index) const {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char guid[webrtc::kAdmMaxGuidSize] = {0};
adm_proxy_->RecordingDeviceName(index, name, guid);
return rust::String(guid);
}
bool AudioDeviceController::set_playout_device(uint16_t index) const {
return adm_proxy_->SetPlayoutDevice(index) == 0;
}
bool AudioDeviceController::set_recording_device(uint16_t index) const {
return adm_proxy_->SetRecordingDevice(index) == 0;
}
bool AudioDeviceController::set_playout_device_by_guid(rust::String guid) const {
int16_t count = adm_proxy_->PlayoutDevices();
// Try to find a device matching the GUID
for (int16_t i = 0; i < count; i++) {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char device_guid[webrtc::kAdmMaxGuidSize] = {0};
if (adm_proxy_->PlayoutDeviceName(i, name, device_guid) == 0) {
if (std::string(guid.c_str()) == std::string(device_guid)) {
return adm_proxy_->SetPlayoutDevice(i) == 0;
}
}
}
// No match found - fall back to default device (index 0).
// This handles mobile platforms (iOS/Android) where:
// - GUIDs may be empty or not meaningful
// - Device selection is a no-op (system handles routing)
if (count > 0) {
return adm_proxy_->SetPlayoutDevice(0) == 0;
}
return false;
}
bool AudioDeviceController::set_recording_device_by_guid(rust::String guid) const {
int16_t count = adm_proxy_->RecordingDevices();
// Try to find a device matching the GUID
for (int16_t i = 0; i < count; i++) {
char name[webrtc::kAdmMaxDeviceNameSize] = {0};
char device_guid[webrtc::kAdmMaxGuidSize] = {0};
if (adm_proxy_->RecordingDeviceName(i, name, device_guid) == 0) {
if (std::string(guid.c_str()) == std::string(device_guid)) {
return adm_proxy_->SetRecordingDevice(i) == 0;
}
}
}
// No match found - fall back to default device (index 0).
// This handles mobile platforms (iOS/Android) where:
// - GUIDs may be empty or not meaningful
// - Device selection is a no-op (system handles routing)
if (count > 0) {
return adm_proxy_->SetRecordingDevice(0) == 0;
}
return false;
}
bool AudioDeviceController::stop_recording() const {
return adm_proxy_->StopRecording() == 0;
}
bool AudioDeviceController::init_recording() const {
return adm_proxy_->InitRecording() == 0;
}
bool AudioDeviceController::start_recording() const {
return adm_proxy_->StartRecording() == 0;
}
bool AudioDeviceController::recording_is_initialized() const {
return adm_proxy_->RecordingIsInitialized();
}
bool AudioDeviceController::stop_playout() const {
return adm_proxy_->StopPlayout() == 0;
}
bool AudioDeviceController::init_playout() const {
return adm_proxy_->InitPlayout() == 0;
}
bool AudioDeviceController::start_playout() const {
return adm_proxy_->StartPlayout() == 0;
}
bool AudioDeviceController::playout_is_initialized() const {
return adm_proxy_->PlayoutIsInitialized();
}
bool AudioDeviceController::builtin_aec_is_available() const {
return adm_proxy_->BuiltInAECIsAvailable();
}
bool AudioDeviceController::builtin_agc_is_available() const {
return adm_proxy_->BuiltInAGCIsAvailable();
}
bool AudioDeviceController::builtin_ns_is_available() const {
return adm_proxy_->BuiltInNSIsAvailable();
}
bool AudioDeviceController::enable_builtin_aec(bool enable) const {
return adm_proxy_->EnableBuiltInAEC(enable) == 0;
}
bool AudioDeviceController::enable_builtin_agc(bool enable) const {
return adm_proxy_->EnableBuiltInAGC(enable) == 0;
}
bool AudioDeviceController::enable_builtin_ns(bool enable) const {
return adm_proxy_->EnableBuiltInNS(enable) == 0;
}
void AudioDeviceController::set_adm_recording_enabled(bool enabled) const {
adm_proxy_->set_recording_enabled(enabled);
}
bool AudioDeviceController::adm_recording_enabled() const {
return adm_proxy_->recording_enabled();
}
void AudioDeviceController::set_adm_playout_enabled(bool enabled) const {
adm_proxy_->set_playout_enabled(enabled);
}
bool AudioDeviceController::adm_playout_enabled() const {
return adm_proxy_->playout_enabled();
}
bool AudioDeviceController::acquire_platform_adm() const {
return adm_proxy_->AcquirePlatformAdm();
}
void AudioDeviceController::release_platform_adm() const {
adm_proxy_->ReleasePlatformAdm();
}
int AudioDeviceController::platform_adm_ref_count() const {
return adm_proxy_->platform_adm_ref_count();
}
bool AudioDeviceController::is_platform_adm_active() const {
return adm_proxy_->is_platform_adm_active();
}
bool AudioDeviceController::ensure_platform_adm() const {
return adm_proxy_->EnsurePlatformAdm();
}
bool AudioDeviceController::platform_adm_available() const {
return adm_proxy_->platform_adm_available();
}
} // namespace livekit_ffi
@@ -0,0 +1,121 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub use cxx::SharedPtr;
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
unsafe extern "C++" {
include!("livekit/audio_device_controller.h");
include!("livekit/peer_connection_factory.h");
type AudioDeviceController;
type PeerConnectionFactory = crate::peer_connection_factory::ffi::PeerConnectionFactory;
fn audio_device(self: &PeerConnectionFactory) -> SharedPtr<AudioDeviceController>;
fn playout_devices(self: &AudioDeviceController) -> i16;
fn recording_devices(self: &AudioDeviceController) -> i16;
fn playout_device_name(self: &AudioDeviceController, index: u16) -> String;
fn recording_device_name(self: &AudioDeviceController, index: u16) -> String;
fn playout_device_guid(self: &AudioDeviceController, index: u16) -> String;
fn recording_device_guid(self: &AudioDeviceController, index: u16) -> String;
fn set_playout_device(self: &AudioDeviceController, index: u16) -> bool;
fn set_recording_device(self: &AudioDeviceController, index: u16) -> bool;
fn set_playout_device_by_guid(self: &AudioDeviceController, guid: String) -> bool;
fn set_recording_device_by_guid(self: &AudioDeviceController, guid: String) -> bool;
fn stop_recording(self: &AudioDeviceController) -> bool;
fn init_recording(self: &AudioDeviceController) -> bool;
fn start_recording(self: &AudioDeviceController) -> bool;
fn recording_is_initialized(self: &AudioDeviceController) -> bool;
fn stop_playout(self: &AudioDeviceController) -> bool;
fn init_playout(self: &AudioDeviceController) -> bool;
fn start_playout(self: &AudioDeviceController) -> bool;
fn playout_is_initialized(self: &AudioDeviceController) -> bool;
fn builtin_aec_is_available(self: &AudioDeviceController) -> bool;
fn builtin_agc_is_available(self: &AudioDeviceController) -> bool;
fn builtin_ns_is_available(self: &AudioDeviceController) -> bool;
fn enable_builtin_aec(self: &AudioDeviceController, enable: bool) -> bool;
fn enable_builtin_agc(self: &AudioDeviceController, enable: bool) -> bool;
fn enable_builtin_ns(self: &AudioDeviceController, enable: bool) -> bool;
fn set_adm_recording_enabled(self: &AudioDeviceController, enabled: bool);
fn adm_recording_enabled(self: &AudioDeviceController) -> bool;
fn set_adm_playout_enabled(self: &AudioDeviceController, enabled: bool);
fn adm_playout_enabled(self: &AudioDeviceController) -> bool;
fn acquire_platform_adm(self: &AudioDeviceController) -> bool;
fn release_platform_adm(self: &AudioDeviceController);
fn platform_adm_ref_count(self: &AudioDeviceController) -> i32;
fn is_platform_adm_active(self: &AudioDeviceController) -> bool;
fn ensure_platform_adm(self: &AudioDeviceController) -> bool;
fn platform_adm_available(self: &AudioDeviceController) -> bool;
}
}
impl_thread_safety!(ffi::AudioDeviceController, Send + Sync);
#[cfg(test)]
mod tests {
use crate::peer_connection_factory::ffi::create_peer_connection_factory;
use std::sync::Mutex;
static TEST_MUTEX: Mutex<()> = Mutex::new(());
#[test]
fn test_audio_device_controller_bridge() {
let _guard = TEST_MUTEX.lock().expect("test mutex poisoned");
let factory = create_peer_connection_factory();
let audio = factory.audio_device();
let recording_count = audio.recording_devices();
let playout_count = audio.playout_devices();
assert!(recording_count >= 0);
assert!(playout_count >= 0);
if recording_count > 0 {
let _ = audio.recording_device_name(0);
let guid = audio.recording_device_guid(0);
let _ = audio.set_recording_device_by_guid(guid);
}
if playout_count > 0 {
let _ = audio.playout_device_name(0);
let guid = audio.playout_device_guid(0);
let _ = audio.set_playout_device_by_guid(guid);
}
let initial_recording = audio.adm_recording_enabled();
audio.set_adm_recording_enabled(!initial_recording);
assert_eq!(audio.adm_recording_enabled(), !initial_recording);
audio.set_adm_recording_enabled(initial_recording);
assert_eq!(audio.adm_recording_enabled(), initial_recording);
let initial_playout = audio.adm_playout_enabled();
audio.set_adm_playout_enabled(!initial_playout);
assert_eq!(audio.adm_playout_enabled(), !initial_playout);
audio.set_adm_playout_enabled(initial_playout);
assert_eq!(audio.adm_playout_enabled(), initial_playout);
assert!(audio.platform_adm_ref_count() >= 0);
let _ = audio.is_platform_adm_active();
}
}
@@ -0,0 +1,105 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/audio_mixer.h"
#include <iostream>
#include <memory>
#include "api/audio/audio_frame.h"
#include "api/audio/audio_mixer.h"
#include "modules/audio_mixer/audio_mixer_impl.h"
#include "webrtc-sys/src/audio_mixer.rs.h"
namespace livekit_ffi {
AudioMixer::AudioMixer() {
audio_mixer_ = webrtc::AudioMixerImpl::Create();
}
void AudioMixer::add_source(rust::Box<AudioMixerSourceWrapper> source) {
auto native_source = std::make_shared<AudioMixerSource>(std::move(source));
webrtc::MutexLock lock(&sources_mutex_);
audio_mixer_->AddSource(native_source.get());
sources_.push_back(native_source);
}
void AudioMixer::remove_source(int source_ssrc) {
webrtc::MutexLock lock(&sources_mutex_);
auto it = std::find_if(
sources_.begin(), sources_.end(),
[source_ssrc](const auto& s) { return s->Ssrc() == source_ssrc; });
if (it != sources_.end()) {
audio_mixer_->RemoveSource(it->get());
sources_.erase(it);
}
}
size_t AudioMixer::mix(size_t number_of_channels) {
audio_mixer_->Mix(number_of_channels, &frame_);
return frame_.num_channels() * frame_.samples_per_channel();
}
const int16_t* AudioMixer::data() const {
return frame_.data();
}
std::unique_ptr<AudioMixer> create_audio_mixer() {
return std::make_unique<AudioMixer>();
}
AudioMixerSource::AudioMixerSource(rust::Box<AudioMixerSourceWrapper> source)
: source_(std::move(source)) {}
int AudioMixerSource::Ssrc() const {
return source_->ssrc();
}
int AudioMixerSource::PreferredSampleRate() const {
return source_->preferred_sample_rate();
}
webrtc::AudioMixer::Source::AudioFrameInfo
AudioMixerSource::GetAudioFrameWithInfo(int sample_rate,
webrtc::AudioFrame* audio_frame) {
NativeAudioFrame frame(audio_frame);
livekit_ffi::AudioFrameInfo result =
source_->get_audio_frame_with_info(sample_rate, frame);
if (result == livekit_ffi::AudioFrameInfo::Normal) {
return webrtc::AudioMixer::Source::AudioFrameInfo::kNormal;
} else if (result == livekit_ffi::AudioFrameInfo::Muted) {
return webrtc::AudioMixer::Source::AudioFrameInfo::kMuted;
} else {
return webrtc::AudioMixer::Source::AudioFrameInfo::kError;
}
}
void NativeAudioFrame::update_frame(uint32_t timestamp,
const int16_t* data,
size_t samples_per_channel,
int sample_rate_hz,
size_t num_channels) {
frame_->UpdateFrame(timestamp, data, samples_per_channel, sample_rate_hz,
webrtc::AudioFrame::SpeechType::kNormalSpeech,
webrtc::AudioFrame::VADActivity::kVadUnknown,
num_channels);
}
} // namespace livekit_ffi
@@ -0,0 +1,106 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{pin::Pin, sync::Arc};
use ffi::AudioFrameInfo;
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
unsafe extern "C++" {
include!("livekit/audio_mixer.h");
type AudioMixer;
unsafe fn add_source(self: Pin<&mut AudioMixer>, src: Box<AudioMixerSourceWrapper>);
unsafe fn remove_source(self: Pin<&mut AudioMixer>, ssrc: i32);
unsafe fn mix(self: Pin<&mut AudioMixer>, num_channels: usize) -> usize;
unsafe fn data(self: &AudioMixer) -> *const i16;
fn create_audio_mixer() -> UniquePtr<AudioMixer>;
type NativeAudioFrame;
unsafe fn update_frame(
self: Pin<&mut NativeAudioFrame>,
timestamp: u32,
data: *const i16,
samples_per_channel: usize,
sample_rate_hz: i32,
num_channels: usize,
);
}
pub enum AudioFrameInfo {
Normal,
Muted,
Error,
}
extern "Rust" {
type AudioMixerSourceWrapper;
fn ssrc(self: &AudioMixerSourceWrapper) -> i32;
fn preferred_sample_rate(self: &AudioMixerSourceWrapper) -> i32;
fn get_audio_frame_with_info(
self: &AudioMixerSourceWrapper,
target_sample_rate: i32,
frame: Pin<&mut NativeAudioFrame>,
) -> AudioFrameInfo;
}
}
pub trait AudioMixerSource {
fn ssrc(&self) -> i32;
fn preferred_sample_rate(&self) -> i32;
fn get_audio_frame_with_info(
&self,
target_sample_rate: i32,
frame: NativeAudioFrame,
) -> AudioFrameInfo;
}
pub struct AudioMixerSourceWrapper {
source: Arc<dyn AudioMixerSource>,
}
pub type NativeAudioFrame<'a> = Pin<&'a mut ffi::NativeAudioFrame>;
impl AudioMixerSourceWrapper {
pub fn new(source: Arc<dyn AudioMixerSource>) -> Self {
Self { source }
}
pub fn ssrc(&self) -> i32 {
self.source.ssrc()
}
pub fn preferred_sample_rate(&self) -> i32 {
self.source.preferred_sample_rate()
}
pub fn get_audio_frame_with_info(
&self,
target_sample_rate: i32,
frame: Pin<&mut ffi::NativeAudioFrame>,
) -> AudioFrameInfo {
self.source.get_audio_frame_with_info(target_sample_rate, frame)
}
}
impl_thread_safety!(ffi::AudioMixer, Send + Sync);
@@ -0,0 +1,52 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/audio_resampler.h"
#include <memory>
#include "audio/remix_resample.h"
#include "api/audio/audio_view.h"
#include "api/audio/audio_frame.h"
namespace livekit_ffi {
size_t AudioResampler::remix_and_resample(const int16_t* src,
size_t samples_per_channel,
size_t num_channels,
int sample_rate,
size_t dest_num_channels,
int dest_sample_rate) {
frame_.num_channels_ = dest_num_channels;
frame_.sample_rate_hz_ = dest_sample_rate;
frame_.samples_per_channel_ = webrtc::SampleRateToDefaultChannelSize(dest_sample_rate);
webrtc::InterleavedView<const int16_t> source(static_cast<const int16_t*>(src),
samples_per_channel,
num_channels);
webrtc::voe::RemixAndResample(source, sample_rate, &resampler_, &frame_);
return frame_.num_channels() * frame_.samples_per_channel() * sizeof(int16_t);
}
const int16_t* AudioResampler::data() const {
return frame_.data();
}
std::unique_ptr<AudioResampler> create_audio_resampler() {
return std::make_unique<AudioResampler>();
}
} // namespace livekit_ffi
@@ -0,0 +1,40 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
unsafe extern "C++" {
include!("livekit/audio_resampler.h");
type AudioResampler;
unsafe fn remix_and_resample(
self: Pin<&mut AudioResampler>,
src: *const i16,
samples_per_channel: usize,
num_channels: usize,
sample_rate: i32,
dst_num_channels: usize,
dst_sample_rate: i32,
) -> usize;
unsafe fn data(self: &AudioResampler) -> *const i16;
fn create_audio_resampler() -> UniquePtr<AudioResampler>;
}
}
impl_thread_safety!(ffi::AudioResampler, Send + Sync);
@@ -0,0 +1,319 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/audio_track.h"
#include <algorithm>
#include <iostream>
#include <iterator>
#include <memory>
#include "api/audio_options.h"
#include "api/audio/audio_frame.h"
#include "api/media_stream_interface.h"
#include "api/task_queue/task_queue_base.h"
#include "audio/remix_resample.h"
#include "common_audio/include/audio_util.h"
#include "livekit/global_task_queue.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/ref_counted_object.h"
#include "rtc_base/synchronization/mutex.h"
#include "rust/cxx.h"
#include "webrtc-sys/src/audio_track.rs.h"
namespace livekit_ffi {
inline webrtc::AudioOptions to_native_audio_options(
const AudioSourceOptions& options) {
webrtc::AudioOptions rtc_options{};
rtc_options.echo_cancellation = options.echo_cancellation;
rtc_options.noise_suppression = options.noise_suppression;
rtc_options.auto_gain_control = options.auto_gain_control;
return rtc_options;
}
inline AudioSourceOptions to_rust_audio_options(
const webrtc::AudioOptions& rtc_options) {
AudioSourceOptions options{};
options.echo_cancellation = rtc_options.echo_cancellation.value_or(false);
options.noise_suppression = rtc_options.noise_suppression.value_or(false);
options.auto_gain_control = rtc_options.auto_gain_control.value_or(false);
return options;
}
AudioTrack::AudioTrack(std::shared_ptr<RtcRuntime> rtc_runtime,
webrtc::scoped_refptr<webrtc::AudioTrackInterface> track)
: MediaStreamTrack(rtc_runtime, std::move(track)) {}
AudioTrack::~AudioTrack() {
webrtc::MutexLock lock(&mutex_);
for (auto& sink : sinks_) {
track()->RemoveSink(sink.get());
}
}
void AudioTrack::add_sink(const std::shared_ptr<NativeAudioSink>& sink) const {
webrtc::MutexLock lock(&mutex_);
track()->AddSink(sink.get());
sinks_.push_back(sink);
}
void AudioTrack::remove_sink(
const std::shared_ptr<NativeAudioSink>& sink) const {
webrtc::MutexLock lock(&mutex_);
track()->RemoveSink(sink.get());
sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end());
}
NativeAudioSink::NativeAudioSink(rust::Box<AudioSinkWrapper> observer,
int sample_rate,
int num_channels)
: observer_(std::move(observer)),
sample_rate_(sample_rate),
num_channels_(num_channels) {
frame_.sample_rate_hz_ = sample_rate;
frame_.num_channels_ = num_channels;
frame_.samples_per_channel_ = webrtc::SampleRateToDefaultChannelSize(sample_rate);
}
void NativeAudioSink::OnData(const void* audio_data,
int bits_per_sample,
int sample_rate,
size_t number_of_channels,
size_t number_of_frames) {
RTC_CHECK_EQ(16, bits_per_sample);
const int16_t* data = static_cast<const int16_t*>(audio_data);
if (sample_rate_ != sample_rate || num_channels_ != number_of_channels) {
webrtc::InterleavedView<const int16_t> source(data,
number_of_frames,
number_of_channels);
// resample/remix before capturing
webrtc::voe::RemixAndResample(source, sample_rate, &resampler_, &frame_);
rust::Slice<const int16_t> rust_slice(
frame_.data(), frame_.num_channels() * frame_.samples_per_channel());
observer_->on_data(rust_slice, frame_.sample_rate_hz(),
frame_.num_channels(), frame_.samples_per_channel());
} else {
rust::Slice<const int16_t> rust_slice(
data, number_of_channels * number_of_frames);
observer_->on_data(rust_slice, sample_rate, number_of_channels,
number_of_frames);
}
}
std::shared_ptr<NativeAudioSink> new_native_audio_sink(
rust::Box<AudioSinkWrapper> observer,
int sample_rate,
int num_channels) {
return std::make_shared<NativeAudioSink>(std::move(observer), sample_rate,
num_channels);
}
AudioTrackSource::InternalSource::InternalSource(
const webrtc::AudioOptions& options,
int sample_rate,
int num_channels,
int queue_size_ms, // must be a multiple of 10ms
webrtc::TaskQueueFactory* task_queue_factory)
: options_(options),
sample_rate_(sample_rate),
num_channels_(num_channels),
capture_userdata_(nullptr),
on_complete_(nullptr) {
if (!queue_size_ms) {
// Set queue_size_samples_ to 0 so that capture_frame() will get to the fast path.
queue_size_samples_ = 0;
return; // no audio queue
}
int samples10ms = sample_rate / 100 * num_channels;
silence_buffer_.assign(samples10ms, 0);
queue_size_samples_ = queue_size_ms / 10 * samples10ms;
notify_threshold_samples_ = queue_size_samples_; // TODO: this is currently
// using x2 the queue size
buffer_.reserve(queue_size_samples_ + notify_threshold_samples_);
audio_queue_ =
task_queue_factory->CreateTaskQueue(
"AudioSourceCapture", webrtc::TaskQueueFactory::Priority::NORMAL);
audio_task_ = webrtc::RepeatingTaskHandle::Start(
audio_queue_.get(),
[this, samples10ms]() {
webrtc::MutexLock lock(&mutex_);
constexpr int kBitsPerSample = sizeof(int16_t) * 8;
if (buffer_.size() >= samples10ms) {
for (auto sink : sinks_)
sink->OnData(buffer_.data(), kBitsPerSample, sample_rate_,
num_channels_, samples10ms / num_channels_);
buffer_.erase(buffer_.begin(), buffer_.begin() + samples10ms);
} else {
// Always provide a 10ms frame to avoid playout underruns.
for (auto sink : sinks_)
sink->OnData(silence_buffer_.data(), kBitsPerSample, sample_rate_,
num_channels_, samples10ms / num_channels_);
}
if (on_complete_ && buffer_.size() <= notify_threshold_samples_) {
on_complete_(capture_userdata_);
on_complete_ = nullptr;
capture_userdata_ = nullptr;
}
return webrtc::TimeDelta::Millis(10);
},
webrtc::TaskQueueBase::DelayPrecision::kHigh);
}
AudioTrackSource::InternalSource::~InternalSource() {
}
bool AudioTrackSource::InternalSource::capture_frame(
rust::Slice<const int16_t> data,
uint32_t sample_rate,
uint32_t number_of_channels,
size_t number_of_frames,
const SourceContext* ctx,
void (*on_complete)(const SourceContext*)) {
webrtc::MutexLock lock(&mutex_);
if (queue_size_samples_) {
int available =
(queue_size_samples_ + notify_threshold_samples_) - buffer_.size();
if (available < data.size())
return false;
if (on_complete_ || capture_userdata_)
return false;
buffer_.insert(buffer_.end(), data.begin(), data.end());
if (buffer_.size() <= notify_threshold_samples_) {
on_complete(ctx); // complete directly
} else {
on_complete_ = on_complete;
capture_userdata_ = ctx;
}
} else {
// Fast path: capture directly when the queue buffer is 0 (frame size must be 10ms)
for (auto sink : sinks_)
sink->OnData(data.data(), sizeof(int16_t) * 8, sample_rate,
number_of_channels, number_of_frames);
}
return true;
}
void AudioTrackSource::InternalSource::clear_buffer() {
webrtc::MutexLock lock(&mutex_);
buffer_.clear();
}
webrtc::MediaSourceInterface::SourceState
AudioTrackSource::InternalSource::state() const {
return webrtc::MediaSourceInterface::SourceState::kLive;
}
bool AudioTrackSource::InternalSource::remote() const {
return false;
}
const webrtc::AudioOptions AudioTrackSource::InternalSource::options() const {
webrtc::MutexLock lock(&mutex_);
return options_;
}
void AudioTrackSource::InternalSource::set_options(
const webrtc::AudioOptions& options) {
webrtc::MutexLock lock(&mutex_);
options_ = options;
}
void AudioTrackSource::InternalSource::AddSink(
webrtc::AudioTrackSinkInterface* sink) {
webrtc::MutexLock lock(&mutex_);
sinks_.push_back(sink);
}
void AudioTrackSource::InternalSource::RemoveSink(
webrtc::AudioTrackSinkInterface* sink) {
webrtc::MutexLock lock(&mutex_);
sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end());
}
AudioTrackSource::AudioTrackSource(AudioSourceOptions options,
int sample_rate,
int num_channels,
int queue_size_ms,
webrtc::TaskQueueFactory* task_queue_factory)
: source_(webrtc::make_ref_counted<InternalSource>(
to_native_audio_options(options),
sample_rate,
num_channels,
queue_size_ms,
task_queue_factory)) {}
AudioSourceOptions AudioTrackSource::audio_options() const {
return to_rust_audio_options(source_->options());
}
void AudioTrackSource::set_audio_options(
const AudioSourceOptions& options) const {
source_->set_options(to_native_audio_options(options));
}
bool AudioTrackSource::capture_frame(
rust::Slice<const int16_t> audio_data,
uint32_t sample_rate,
uint32_t number_of_channels,
size_t number_of_frames,
const SourceContext* ctx,
void (*on_complete)(const SourceContext*)) const {
return source_->capture_frame(audio_data, sample_rate, number_of_channels,
number_of_frames, ctx, on_complete);
}
void AudioTrackSource::clear_buffer() const {
source_->clear_buffer();
}
std::shared_ptr<AudioTrackSource> new_audio_track_source(
AudioSourceOptions options,
int sample_rate,
int num_channels,
int queue_size_ms) {
return std::make_shared<AudioTrackSource>(options, sample_rate, num_channels,
queue_size_ms,
GetGlobalTaskQueueFactory());
}
webrtc::scoped_refptr<AudioTrackSource::InternalSource> AudioTrackSource::get()
const {
return source_;
}
} // namespace livekit_ffi
@@ -0,0 +1,124 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::type_id;
use cxx::ExternType;
use std::any::Any;
use std::sync::Arc;
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
pub struct AudioSourceOptions {
pub echo_cancellation: bool,
pub noise_suppression: bool,
pub auto_gain_control: bool,
}
extern "C++" {
include!("livekit/media_stream_track.h");
type MediaStreamTrack = crate::media_stream_track::ffi::MediaStreamTrack;
type CompleteCallback = crate::audio_track::CompleteCallback;
}
unsafe extern "C++" {
include!("livekit/audio_track.h");
type AudioTrack;
type NativeAudioSink;
type AudioTrackSource;
fn add_sink(self: &AudioTrack, sink: &SharedPtr<NativeAudioSink>);
fn remove_sink(self: &AudioTrack, sink: &SharedPtr<NativeAudioSink>);
fn new_native_audio_sink(
observer: Box<AudioSinkWrapper>,
sample_rate: i32,
num_channels: i32,
) -> SharedPtr<NativeAudioSink>;
unsafe fn capture_frame(
self: &AudioTrackSource,
data: &[i16],
sample_rate: u32,
nb_channels: u32,
nb_frames: usize,
userdata: *const SourceContext,
on_complete: CompleteCallback,
) -> bool;
fn clear_buffer(self: &AudioTrackSource);
fn audio_options(self: &AudioTrackSource) -> AudioSourceOptions;
fn set_audio_options(self: &AudioTrackSource, options: &AudioSourceOptions);
fn new_audio_track_source(
options: AudioSourceOptions,
sample_rate: i32,
num_channels: i32,
queue_size_ms: i32,
) -> SharedPtr<AudioTrackSource>;
fn audio_to_media(track: SharedPtr<AudioTrack>) -> SharedPtr<MediaStreamTrack>;
unsafe fn media_to_audio(track: SharedPtr<MediaStreamTrack>) -> SharedPtr<AudioTrack>;
fn _shared_audio_track() -> SharedPtr<AudioTrack>;
fn _shared_audio_track_source() -> SharedPtr<AudioTrackSource>;
}
extern "Rust" {
type AudioSinkWrapper;
type SourceContext;
fn on_data(
self: &AudioSinkWrapper,
data: &[i16],
sample_rate: i32,
nb_channels: usize,
nb_frames: usize,
);
}
}
impl_thread_safety!(ffi::AudioTrack, Send + Sync);
impl_thread_safety!(ffi::NativeAudioSink, Send + Sync);
impl_thread_safety!(ffi::AudioTrackSource, Send + Sync);
#[repr(transparent)]
pub struct SourceContext(pub Box<dyn Any + Send>);
#[repr(transparent)]
pub struct CompleteCallback(pub extern "C" fn(ctx: *const SourceContext));
unsafe impl ExternType for CompleteCallback {
type Id = type_id!("livekit_ffi::CompleteCallback");
type Kind = cxx::kind::Trivial;
}
pub trait AudioSink: Send {
fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize);
}
pub struct AudioSinkWrapper {
observer: Arc<dyn AudioSink>,
}
impl AudioSinkWrapper {
pub fn new(observer: Arc<dyn AudioSink>) -> Self {
Self { observer }
}
fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) {
self.observer.on_data(data, sample_rate, nb_channels, nb_frames);
}
}
@@ -0,0 +1,22 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/candidate.h"
namespace livekit_ffi {
Candidate::Candidate(const webrtc::Candidate& candidate)
: candidate_(candidate) {}
} // namespace livekit_ffi
@@ -0,0 +1,24 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
unsafe extern "C++" {
include!("livekit/candidate.h");
type Candidate; // webrtc::Candidate
fn _shared_candidate() -> SharedPtr<Candidate>;
}
}
@@ -0,0 +1,121 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/data_channel.h"
#include <utility>
#include "rtc_base/synchronization/mutex.h"
#include "webrtc-sys/src/data_channel.rs.h"
namespace livekit_ffi {
webrtc::DataChannelInit to_native_data_channel_init(DataChannelInit init) {
webrtc::DataChannelInit rtc_init{};
rtc_init.id = init.id;
rtc_init.negotiated = init.negotiated;
rtc_init.ordered = init.ordered;
rtc_init.protocol = init.protocol.c_str();
if (init.has_max_retransmit_time)
rtc_init.maxRetransmitTime = init.max_retransmit_time;
if (init.has_max_retransmits)
rtc_init.maxRetransmits = init.max_retransmits;
if (init.has_priority)
rtc_init.priority = webrtc::PriorityValue(static_cast<webrtc::Priority>(init.priority));
return rtc_init;
}
DataChannel::DataChannel(
std::shared_ptr<RtcRuntime> rtc_runtime,
webrtc::scoped_refptr<webrtc::DataChannelInterface> data_channel)
: rtc_runtime_(rtc_runtime), data_channel_(std::move(data_channel)) {
RTC_LOG(LS_VERBOSE) << "DataChannel::DataChannel()";
}
DataChannel::~DataChannel() {
RTC_LOG(LS_VERBOSE) << "DataChannel::~DataChannel()";
unregister_observer();
}
void DataChannel::register_observer(
rust::Box<DataChannelObserverWrapper> observer) const {
webrtc::MutexLock lock(&mutex_);
data_channel_->UnregisterObserver();
observer_ =
std::make_unique<NativeDataChannelObserver>(std::move(observer), this);
data_channel_->RegisterObserver(observer_.get());
}
void DataChannel::unregister_observer() const {
webrtc::MutexLock lock(&mutex_);
data_channel_->UnregisterObserver();
observer_ = nullptr;
}
bool DataChannel::send(const DataBuffer& buffer) const {
return data_channel_->Send(webrtc::DataBuffer{
webrtc::CopyOnWriteBuffer(buffer.ptr, buffer.len), buffer.binary});
}
int DataChannel::id() const {
return data_channel_->id();
}
rust::String DataChannel::label() const {
return data_channel_->label();
}
DataState DataChannel::state() const {
return static_cast<DataState>(data_channel_->state());
}
void DataChannel::close() const {
return data_channel_->Close();
}
uint64_t DataChannel::buffered_amount() const {
return data_channel_->buffered_amount();
}
NativeDataChannelObserver::NativeDataChannelObserver(
rust::Box<DataChannelObserverWrapper> observer,
const DataChannel* dc)
: observer_(std::move(observer)), dc_(dc) {}
void NativeDataChannelObserver::OnStateChange() {
observer_->on_state_change(dc_->state());
}
void NativeDataChannelObserver::OnMessage(const webrtc::DataBuffer& buffer) {
DataBuffer data{};
data.ptr = buffer.data.data();
data.len = buffer.data.size();
data.binary = buffer.binary;
observer_->on_message(data);
}
void NativeDataChannelObserver::OnBufferedAmountChange(
uint64_t sent_data_size) {
observer_->on_buffered_amount_change(sent_data_size);
}
} // namespace livekit_ffi
@@ -0,0 +1,118 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
#[derive(Debug)]
#[repr(i32)]
pub enum Priority {
VeryLow,
Low,
Medium,
High,
}
#[derive(Debug)]
pub struct DataChannelInit {
pub ordered: bool,
pub has_max_retransmit_time: bool,
pub max_retransmit_time: i32,
pub has_max_retransmits: bool,
pub max_retransmits: i32,
pub protocol: String,
pub negotiated: bool,
pub id: i32,
pub has_priority: bool,
pub priority: Priority,
}
#[derive(Debug)]
pub struct DataBuffer {
pub ptr: *const u8,
pub len: usize,
pub binary: bool,
}
#[derive(Debug)]
#[repr(i32)]
pub enum DataState {
Connecting,
Open,
Closing,
Closed,
}
unsafe extern "C++" {
include!("livekit/data_channel.h");
type DataChannel;
fn register_observer(self: &DataChannel, observer: Box<DataChannelObserverWrapper>);
fn unregister_observer(self: &DataChannel);
fn send(self: &DataChannel, data: &DataBuffer) -> bool;
fn id(self: &DataChannel) -> i32;
fn label(self: &DataChannel) -> String;
fn state(self: &DataChannel) -> DataState;
fn close(self: &DataChannel);
fn buffered_amount(self: &DataChannel) -> u64;
fn _shared_data_channel() -> SharedPtr<DataChannel>; // Ignore
}
extern "Rust" {
type DataChannelObserverWrapper;
fn on_state_change(self: &DataChannelObserverWrapper, state: DataState);
fn on_message(self: &DataChannelObserverWrapper, buffer: DataBuffer);
fn on_buffered_amount_change(self: &DataChannelObserverWrapper, sent_data_size: u64);
}
}
impl_thread_safety!(ffi::DataChannel, Send + Sync);
pub trait DataChannelObserver: Send + Sync {
fn on_state_change(&self, state: ffi::DataState);
fn on_message(&self, data: &[u8], is_binary: bool);
fn on_buffered_amount_change(&self, sent_data_size: u64);
}
pub struct DataChannelObserverWrapper {
observer: Arc<dyn DataChannelObserver>,
}
impl DataChannelObserverWrapper {
pub fn new(observer: Arc<dyn DataChannelObserver>) -> Self {
Self { observer }
}
fn on_state_change(&self, state: ffi::DataState) {
self.observer.on_state_change(state);
}
fn on_message(&self, buffer: ffi::DataBuffer) {
unsafe {
let data = std::slice::from_raw_parts(buffer.ptr, buffer.len);
self.observer.on_message(data, buffer.binary);
}
}
fn on_buffered_amount_change(&self, sent_data_size: u64) {
self.observer.on_buffered_amount_change(sent_data_size);
}
}
@@ -0,0 +1,118 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/desktop_capturer.h"
#include "modules/desktop_capture/desktop_capture_options.h"
using SourceList = webrtc::DesktopCapturer::SourceList;
namespace livekit_ffi {
std::unique_ptr<DesktopCapturer> new_desktop_capturer(
DesktopCapturerOptions options) {
webrtc::DesktopCaptureOptions webrtc_options =
webrtc::DesktopCaptureOptions::CreateDefault();
#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
webrtc_options.set_allow_sck_capturer(true);
webrtc_options.set_allow_sck_system_picker(options.allow_sck_system_picker);
#endif /* defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) */
#ifdef _WIN64
switch (options.source_type) {
case SourceType::Screen:
webrtc_options.set_allow_wgc_screen_capturer(true);
break;
case SourceType::Window:
webrtc_options.set_allow_wgc_window_capturer(true);
// https://github.com/webrtc-sdk/webrtc/blob/m137_release/modules/desktop_capture/desktop_capture_options.h#L133-L142
webrtc_options.set_enumerate_current_process_windows(false);
break;
default:
break;
}
webrtc_options.set_allow_directx_capturer(true);
#endif /* _WIN64 */
#ifdef WEBRTC_USE_PIPEWIRE
webrtc_options.set_allow_pipewire(true);
#endif /* WEBRTC_USE_PIPEWIRE */
// prefer_cursor_embedded indicate that the capturer should try to include the
// cursor in the frame
webrtc_options.set_prefer_cursor_embedded(options.include_cursor);
std::unique_ptr<webrtc::DesktopCapturer> capturer = nullptr;
switch (options.source_type) {
case SourceType::Window:
capturer = webrtc::DesktopCapturer::CreateWindowCapturer(webrtc_options);
break;
case SourceType::Screen:
capturer = webrtc::DesktopCapturer::CreateScreenCapturer(webrtc_options);
break;
case SourceType::Generic:
capturer = webrtc::DesktopCapturer::CreateGenericCapturer(webrtc_options);
break;
default:
return nullptr;
}
if (!capturer) {
return nullptr;
}
return std::make_unique<DesktopCapturer>(std::move(capturer));
}
void DesktopCapturer::start(
rust::Box<DesktopCapturerCallbackWrapper> callback) {
this->callback = std::move(callback);
capturer->Start(this);
}
void DesktopCapturer::OnCaptureResult(
webrtc::DesktopCapturer::Result result,
std::unique_ptr<webrtc::DesktopFrame> frame) {
CaptureResult ret_result = CaptureResult::ErrorPermanent;
switch (result) {
case webrtc::DesktopCapturer::Result::SUCCESS:
ret_result = CaptureResult::Success;
break;
case webrtc::DesktopCapturer::Result::ERROR_PERMANENT:
ret_result = CaptureResult::ErrorPermanent;
break;
case webrtc::DesktopCapturer::Result::ERROR_TEMPORARY:
ret_result = CaptureResult::ErrorTemporary;
break;
default:
break;
}
if (callback) {
(*callback)->on_capture_result(
ret_result, std::make_unique<DesktopFrame>(std::move(frame)));
}
}
rust::Vec<Source> DesktopCapturer::get_source_list() const {
SourceList list{};
bool res = capturer->GetSourceList(&list);
rust::Vec<Source> source_list{};
if (res) {
for (auto& source : list) {
source_list.push_back(Source{static_cast<uint64_t>(source.id),
source.title, source.display_id});
}
}
return source_list;
}
} // namespace livekit_ffi
@@ -0,0 +1,113 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use cxx::UniquePtr;
use ffi::CaptureResult;
use crate::{desktop_capturer::ffi::DesktopFrame, impl_thread_safety};
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
#[derive(Clone)]
struct Source {
id: u64,
title: String,
display_id: i64,
}
#[derive(Debug, PartialEq)]
enum SourceType {
Screen,
Window,
Generic,
}
#[derive(Clone, Debug)]
struct DesktopCapturerOptions {
source_type: SourceType,
include_cursor: bool,
allow_sck_system_picker: bool,
}
enum CaptureResult {
Success,
ErrorTemporary,
ErrorPermanent,
}
unsafe extern "C++" {
include!("livekit/desktop_capturer.h");
type DesktopCapturer;
type DesktopFrame;
fn new_desktop_capturer(options: DesktopCapturerOptions) -> UniquePtr<DesktopCapturer>;
fn capture_frame(self: &DesktopCapturer);
fn get_source_list(self: &DesktopCapturer) -> Vec<Source>;
fn select_source(self: &DesktopCapturer, id: u64) -> bool;
fn start(self: Pin<&mut DesktopCapturer>, callback: Box<DesktopCapturerCallbackWrapper>);
fn width(self: &DesktopFrame) -> i32;
fn height(self: &DesktopFrame) -> i32;
fn stride(self: &DesktopFrame) -> i32;
fn left(self: &DesktopFrame) -> i32;
fn top(self: &DesktopFrame) -> i32;
fn data(self: &DesktopFrame) -> *const u8;
}
extern "Rust" {
type DesktopCapturerCallbackWrapper;
fn on_capture_result(
self: &mut DesktopCapturerCallbackWrapper,
result: CaptureResult,
frame: UniquePtr<DesktopFrame>,
);
}
}
impl_thread_safety!(ffi::DesktopCapturer, Send + Sync);
#[derive(Debug, PartialEq)]
pub enum CaptureError {
Temporary,
Permanent,
}
pub trait DesktopCapturerCallback: Send {
fn on_capture_result(&mut self, result: Result<UniquePtr<DesktopFrame>, CaptureError>);
}
pub struct DesktopCapturerCallbackWrapper {
callback: Box<dyn DesktopCapturerCallback>,
}
impl DesktopCapturerCallbackWrapper {
pub fn new(callback: Box<dyn DesktopCapturerCallback>) -> Self {
Self { callback }
}
fn on_capture_result(&mut self, result: CaptureResult, frame: UniquePtr<DesktopFrame>) {
match result {
CaptureResult::Success => self.callback.on_capture_result(Ok(frame)),
CaptureResult::ErrorTemporary => {
self.callback.on_capture_result(Err(CaptureError::Temporary))
}
CaptureResult::ErrorPermanent => {
self.callback.on_capture_result(Err(CaptureError::Permanent))
}
_ => self.callback.on_capture_result(Err(CaptureError::Permanent)),
}
}
}
@@ -0,0 +1,364 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/frame_cryptor.h"
#include <memory>
#include "absl/types/optional.h"
#include "api/make_ref_counted.h"
#include "livekit/peer_connection.h"
#include "livekit/peer_connection_factory.h"
#include "livekit/packet_trailer.h"
#include "livekit/webrtc.h"
#include "rtc_base/thread.h"
#include "webrtc-sys/src/frame_cryptor.rs.h"
namespace livekit_ffi {
class ChainedFrameTransformer : public webrtc::FrameTransformerInterface,
public webrtc::TransformedFrameCallback {
public:
ChainedFrameTransformer(
webrtc::scoped_refptr<webrtc::FrameTransformerInterface> first,
webrtc::scoped_refptr<webrtc::FrameTransformerInterface> second)
: first_(std::move(first)), second_(std::move(second)) {}
void Transform(
std::unique_ptr<webrtc::TransformableFrameInterface> frame) override {
first_->Transform(std::move(frame));
}
void RegisterTransformedFrameCallback(
webrtc::scoped_refptr<webrtc::TransformedFrameCallback> callback) override {
second_->RegisterTransformedFrameCallback(callback);
first_->RegisterTransformedFrameCallback(
webrtc::scoped_refptr<webrtc::TransformedFrameCallback>(this));
}
void RegisterTransformedFrameSinkCallback(
webrtc::scoped_refptr<webrtc::TransformedFrameCallback> callback,
uint32_t ssrc) override {
second_->RegisterTransformedFrameSinkCallback(callback, ssrc);
first_->RegisterTransformedFrameSinkCallback(
webrtc::scoped_refptr<webrtc::TransformedFrameCallback>(this), ssrc);
}
void UnregisterTransformedFrameCallback() override {
first_->UnregisterTransformedFrameCallback();
second_->UnregisterTransformedFrameCallback();
}
void UnregisterTransformedFrameSinkCallback(uint32_t ssrc) override {
first_->UnregisterTransformedFrameSinkCallback(ssrc);
second_->UnregisterTransformedFrameSinkCallback(ssrc);
}
void OnTransformedFrame(
std::unique_ptr<webrtc::TransformableFrameInterface> frame) override {
second_->Transform(std::move(frame));
}
private:
webrtc::scoped_refptr<webrtc::FrameTransformerInterface> first_;
webrtc::scoped_refptr<webrtc::FrameTransformerInterface> second_;
};
webrtc::FrameCryptorTransformer::Algorithm AlgorithmToFrameCryptorAlgorithm(
Algorithm algorithm) {
switch (algorithm) {
case Algorithm::AesGcm:
return webrtc::FrameCryptorTransformer::Algorithm::kAesGcm;
case Algorithm::AesCbc:
return webrtc::FrameCryptorTransformer::Algorithm::kAesCbc;
default:
return webrtc::FrameCryptorTransformer::Algorithm::kAesGcm;
}
}
webrtc::KeyDerivationAlgorithm
KeyDerivationAlgorithmToFrameCryptorKeyDerivationAlgorithm(
KeyDerivationAlgorithm algorithm) {
switch (algorithm) {
case KeyDerivationAlgorithm::PBKDF2:
return webrtc::KeyDerivationAlgorithm::kPBKDF2;
case KeyDerivationAlgorithm::HKDF:
return webrtc::KeyDerivationAlgorithm::kHKDF;
default:
return webrtc::KeyDerivationAlgorithm::kPBKDF2;
}
}
KeyProvider::KeyProvider(KeyProviderOptions options) {
webrtc::KeyProviderOptions rtc_options;
rtc_options.shared_key = options.shared_key;
std::vector<uint8_t> ratchet_salt;
std::copy(options.ratchet_salt.begin(), options.ratchet_salt.end(),
std::back_inserter(ratchet_salt));
rtc_options.ratchet_salt = ratchet_salt;
rtc_options.ratchet_window_size = options.ratchet_window_size;
rtc_options.failure_tolerance = options.failure_tolerance;
rtc_options.key_ring_size = options.key_ring_size;
rtc_options.key_derivation_algorithm =
KeyDerivationAlgorithmToFrameCryptorKeyDerivationAlgorithm(
options.key_derivation_algorithm);
impl_ =
new webrtc::RefCountedObject<webrtc::DefaultKeyProviderImpl>(rtc_options);
}
FrameCryptor::FrameCryptor(
std::shared_ptr<RtcRuntime> rtc_runtime,
const std::string participant_id,
webrtc::FrameCryptorTransformer::Algorithm algorithm,
webrtc::scoped_refptr<webrtc::KeyProvider> key_provider,
webrtc::scoped_refptr<webrtc::RtpSenderInterface> sender)
: rtc_runtime_(rtc_runtime),
participant_id_(participant_id),
key_provider_(key_provider),
sender_(sender) {
auto mediaType =
sender->track()->kind() == "audio"
? webrtc::FrameCryptorTransformer::MediaType::kAudioFrame
: webrtc::FrameCryptorTransformer::MediaType::kVideoFrame;
e2ee_transformer_ = webrtc::scoped_refptr<webrtc::FrameCryptorTransformer>(
new webrtc::FrameCryptorTransformer(rtc_runtime->signaling_thread(),
participant_id, mediaType, algorithm,
key_provider_));
sender->SetEncoderToPacketizerFrameTransformer(e2ee_transformer_);
e2ee_transformer_->SetEnabled(false);
}
FrameCryptor::FrameCryptor(
std::shared_ptr<RtcRuntime> rtc_runtime,
const std::string participant_id,
webrtc::FrameCryptorTransformer::Algorithm algorithm,
webrtc::scoped_refptr<webrtc::KeyProvider> key_provider,
webrtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver)
: rtc_runtime_(rtc_runtime),
participant_id_(participant_id),
key_provider_(key_provider),
receiver_(receiver) {
auto mediaType =
receiver->track()->kind() == "audio"
? webrtc::FrameCryptorTransformer::MediaType::kAudioFrame
: webrtc::FrameCryptorTransformer::MediaType::kVideoFrame;
e2ee_transformer_ = webrtc::scoped_refptr<webrtc::FrameCryptorTransformer>(
new webrtc::FrameCryptorTransformer(rtc_runtime->signaling_thread(),
participant_id, mediaType, algorithm,
key_provider_));
receiver->SetDepacketizerToDecoderFrameTransformer(e2ee_transformer_);
e2ee_transformer_->SetEnabled(false);
}
FrameCryptor::~FrameCryptor() {
if (observer_) {
unregister_observer();
}
}
void FrameCryptor::register_observer(
rust::Box<RtcFrameCryptorObserverWrapper> observer) const {
webrtc::MutexLock lock(&mutex_);
observer_ = webrtc::make_ref_counted<NativeFrameCryptorObserver>(
std::move(observer), this);
e2ee_transformer_->RegisterFrameCryptorTransformerObserver(observer_);
}
void FrameCryptor::unregister_observer() const {
webrtc::MutexLock lock(&mutex_);
observer_ = nullptr;
e2ee_transformer_->UnRegisterFrameCryptorTransformerObserver();
}
void FrameCryptor::set_packet_trailer_handler(
std::shared_ptr<PacketTrailerHandler> handler) const {
if (!handler) {
return;
}
auto timestamp_transformer = handler->transformer();
if (!timestamp_transformer) {
return;
}
webrtc::scoped_refptr<webrtc::FrameTransformerInterface> first;
webrtc::scoped_refptr<webrtc::FrameTransformerInterface> second;
if (sender_) {
first = e2ee_transformer_;
second = timestamp_transformer;
} else if (receiver_) {
first = timestamp_transformer;
second = e2ee_transformer_;
} else {
return;
}
chained_transformer_ =
webrtc::make_ref_counted<ChainedFrameTransformer>(first, second);
if (sender_) {
sender_->SetEncoderToPacketizerFrameTransformer(chained_transformer_);
}
if (receiver_) {
receiver_->SetDepacketizerToDecoderFrameTransformer(chained_transformer_);
}
}
NativeFrameCryptorObserver::NativeFrameCryptorObserver(
rust::Box<RtcFrameCryptorObserverWrapper> observer,
const FrameCryptor* fc)
: observer_(std::move(observer)), fc_(fc) {}
NativeFrameCryptorObserver::~NativeFrameCryptorObserver() {}
void NativeFrameCryptorObserver::OnFrameCryptionStateChanged(
const std::string participant_id,
webrtc::FrameCryptionState state) {
observer_->on_frame_cryption_state_change(
participant_id, static_cast<FrameCryptionState>(state));
}
void FrameCryptor::set_enabled(bool enabled) const {
webrtc::MutexLock lock(&mutex_);
e2ee_transformer_->SetEnabled(enabled);
}
bool FrameCryptor::enabled() const {
webrtc::MutexLock lock(&mutex_);
return e2ee_transformer_->enabled();
}
void FrameCryptor::set_key_index(int32_t index) const {
webrtc::MutexLock lock(&mutex_);
e2ee_transformer_->SetKeyIndex(index);
}
int32_t FrameCryptor::key_index() const {
webrtc::MutexLock lock(&mutex_);
return e2ee_transformer_->key_index();
}
DataPacketCryptor::DataPacketCryptor(
webrtc::FrameCryptorTransformer::Algorithm algorithm,
webrtc::scoped_refptr<webrtc::KeyProvider> key_provider)
: data_packet_cryptor_(
webrtc::make_ref_counted<webrtc::DataPacketCryptor>(algorithm,
key_provider)) {}
EncryptedPacket DataPacketCryptor::encrypt_data_packet(
const ::rust::String participant_id,
uint32_t key_index,
rust::Vec<::std::uint8_t> data) const {
std::vector<uint8_t> data_vec;
std::copy(data.begin(), data.end(), std::back_inserter(data_vec));
auto result = data_packet_cryptor_->Encrypt(
std::string(participant_id.data(), participant_id.size()), key_index,
data_vec);
if (!result.ok()) {
throw std::runtime_error(std::string("Failed to encrypt data packet: ") +
result.error().message());
}
auto& packet = result.value();
EncryptedPacket encrypted_packet;
encrypted_packet.data = rust::Vec<uint8_t>();
std::copy(packet->data.begin(), packet->data.end(),
std::back_inserter(encrypted_packet.data));
encrypted_packet.iv = rust::Vec<uint8_t>();
std::copy(packet->iv.begin(), packet->iv.end(),
std::back_inserter(encrypted_packet.iv));
encrypted_packet.key_index = packet->key_index;
return encrypted_packet;
}
rust::Vec<::std::uint8_t> DataPacketCryptor::decrypt_data_packet(
const ::rust::String participant_id,
const EncryptedPacket& encrypted_packet) const {
std::vector<uint8_t> data_vec;
std::copy(encrypted_packet.data.begin(), encrypted_packet.data.end(),
std::back_inserter(data_vec));
std::vector<uint8_t> iv_vec;
std::copy(encrypted_packet.iv.begin(), encrypted_packet.iv.end(),
std::back_inserter(iv_vec));
auto native_encrypted_packet =
webrtc::make_ref_counted<webrtc::EncryptedPacket>(
std::move(data_vec), std::move(iv_vec), encrypted_packet.key_index);
auto result = data_packet_cryptor_->Decrypt(
std::string(participant_id.data(), participant_id.size()),
native_encrypted_packet);
if (!result.ok()) {
throw std::runtime_error(std::string("Failed to decrypt data packet: ") +
result.error().message());
}
rust::Vec<uint8_t> decrypted_data;
auto& decrypted = result.value();
std::copy(decrypted.begin(), decrypted.end(),
std::back_inserter(decrypted_data));
return decrypted_data;
}
std::shared_ptr<KeyProvider> new_key_provider(KeyProviderOptions options) {
return std::make_shared<KeyProvider>(options);
}
std::shared_ptr<FrameCryptor> new_frame_cryptor_for_rtp_sender(
std::shared_ptr<PeerConnectionFactory> peer_factory,
const ::rust::String participant_id,
Algorithm algorithm,
std::shared_ptr<KeyProvider> key_provider,
std::shared_ptr<RtpSender> sender) {
return std::make_shared<FrameCryptor>(
peer_factory->rtc_runtime(),
std::string(participant_id.data(), participant_id.size()),
AlgorithmToFrameCryptorAlgorithm(algorithm),
key_provider->rtc_key_provider(), sender->rtc_sender());
}
std::shared_ptr<FrameCryptor> new_frame_cryptor_for_rtp_receiver(
std::shared_ptr<PeerConnectionFactory> peer_factory,
const ::rust::String participant_id,
Algorithm algorithm,
std::shared_ptr<KeyProvider> key_provider,
std::shared_ptr<RtpReceiver> receiver) {
return std::make_shared<FrameCryptor>(
peer_factory->rtc_runtime(),
std::string(participant_id.data(), participant_id.size()),
AlgorithmToFrameCryptorAlgorithm(algorithm),
key_provider->rtc_key_provider(), receiver->rtc_receiver());
}
std::shared_ptr<DataPacketCryptor> new_data_packet_cryptor(
Algorithm algorithm,
std::shared_ptr<KeyProvider> key_provider) {
return std::make_shared<DataPacketCryptor>(
AlgorithmToFrameCryptorAlgorithm(algorithm),
key_provider->rtc_key_provider());
}
} // namespace livekit_ffi
@@ -0,0 +1,275 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
#[derive(Debug)]
pub struct KeyProviderOptions {
pub shared_key: bool,
pub ratchet_window_size: i32,
pub ratchet_salt: Vec<u8>,
pub failure_tolerance: i32,
pub key_ring_size: i32,
pub key_derivation_algorithm: KeyDerivationAlgorithm,
}
#[derive(Debug)]
#[repr(i32)]
pub enum KeyDerivationAlgorithm {
PBKDF2 = 0,
HKDF,
}
#[derive(Debug)]
#[repr(i32)]
pub enum Algorithm {
AesGcm = 0,
AesCbc,
}
#[derive(Debug)]
#[repr(i32)]
pub enum FrameCryptionState {
New = 0,
Ok,
EncryptionFailed,
DecryptionFailed,
MissingKey,
KeyRatcheted,
InternalError,
}
#[derive(Debug)]
pub struct EncryptedPacket {
pub data: Vec<u8>,
pub iv: Vec<u8>,
pub key_index: u32,
}
unsafe extern "C++" {
include!("livekit/frame_cryptor.h");
pub type KeyProvider;
pub fn new_key_provider(options: KeyProviderOptions) -> SharedPtr<KeyProvider>;
pub fn set_shared_key(self: &KeyProvider, key_index: i32, key: Vec<u8>) -> bool;
pub fn ratchet_shared_key(self: &KeyProvider, key_index: i32) -> Result<Vec<u8>>;
pub fn get_shared_key(self: &KeyProvider, key_index: i32) -> Result<Vec<u8>>;
pub fn set_sif_trailer(&self, trailer: Vec<u8>);
pub fn set_key(
self: &KeyProvider,
participant_id: String,
key_index: i32,
key: Vec<u8>,
) -> bool;
pub fn ratchet_key(
self: &KeyProvider,
participant_id: String,
key_index: i32,
) -> Result<Vec<u8>>;
pub fn get_key(
self: &KeyProvider,
participant_id: String,
key_index: i32,
) -> Result<Vec<u8>>;
}
unsafe extern "C++" {
include!("livekit/frame_cryptor.h");
include!("livekit/rtp_sender.h");
include!("livekit/rtp_receiver.h");
include!("livekit/peer_connection_factory.h");
include!("livekit/packet_trailer.h");
type RtpSender = crate::rtp_sender::ffi::RtpSender;
type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver;
type PeerConnectionFactory = crate::peer_connection_factory::ffi::PeerConnectionFactory;
type PacketTrailerHandler = crate::packet_trailer::ffi::PacketTrailerHandler;
pub type FrameCryptor;
pub fn new_frame_cryptor_for_rtp_sender(
peer_factory: SharedPtr<PeerConnectionFactory>,
participant_id: String,
algorithm: Algorithm,
key_provider: SharedPtr<KeyProvider>,
sender: SharedPtr<RtpSender>,
) -> SharedPtr<FrameCryptor>;
pub fn new_frame_cryptor_for_rtp_receiver(
peer_factory: SharedPtr<PeerConnectionFactory>,
participant_id: String,
algorithm: Algorithm,
key_provider: SharedPtr<KeyProvider>,
receiver: SharedPtr<RtpReceiver>,
) -> SharedPtr<FrameCryptor>;
pub fn set_enabled(self: &FrameCryptor, enabled: bool);
pub fn enabled(self: &FrameCryptor) -> bool;
pub fn set_key_index(self: &FrameCryptor, index: i32);
pub fn key_index(self: &FrameCryptor) -> i32;
pub fn participant_id(self: &FrameCryptor) -> String;
pub fn register_observer(
self: &FrameCryptor,
observer: Box<RtcFrameCryptorObserverWrapper>,
);
pub fn unregister_observer(self: &FrameCryptor);
pub fn set_packet_trailer_handler(
self: &FrameCryptor,
handler: SharedPtr<PacketTrailerHandler>,
);
}
unsafe extern "C++" {
include!("livekit/frame_cryptor.h");
pub type DataPacketCryptor;
pub fn new_data_packet_cryptor(
algorithm: Algorithm,
key_provider: SharedPtr<KeyProvider>,
) -> SharedPtr<DataPacketCryptor>;
pub fn encrypt_data_packet(
self: &DataPacketCryptor,
participant_id: String,
key_index: u32,
data: Vec<u8>,
) -> Result<EncryptedPacket>;
pub fn decrypt_data_packet(
self: &DataPacketCryptor,
participant_id: String,
encrypted_packet: &EncryptedPacket,
) -> Result<Vec<u8>>;
}
extern "Rust" {
type RtcFrameCryptorObserverWrapper;
fn on_frame_cryption_state_change(
self: &RtcFrameCryptorObserverWrapper,
participant_id: String,
state: FrameCryptionState,
);
}
} // namespace livekit_ffi
impl_thread_safety!(ffi::FrameCryptor, Send + Sync);
impl_thread_safety!(ffi::KeyProvider, Send + Sync);
impl_thread_safety!(ffi::DataPacketCryptor, Send + Sync);
use ffi::FrameCryptionState;
// Re-export the EncryptedPacket for convenience
pub use ffi::EncryptedPacket;
pub trait RtcFrameCryptorObserver: Send + Sync {
fn on_frame_cryption_state_change(&self, participant_id: String, state: FrameCryptionState);
}
pub struct RtcFrameCryptorObserverWrapper {
observer: Arc<dyn RtcFrameCryptorObserver>,
}
impl RtcFrameCryptorObserverWrapper {
pub fn new(observer: Arc<dyn RtcFrameCryptorObserver>) -> Self {
Self { observer }
}
fn on_frame_cryption_state_change(
self: &RtcFrameCryptorObserverWrapper,
participant_id: String,
state: FrameCryptionState,
) {
self.observer.on_frame_cryption_state_change(participant_id, state);
}
}
/// High-level Rust wrapper for data packet cryptor functionality
pub struct DataPacketCryptor {
inner: cxx::SharedPtr<ffi::DataPacketCryptor>,
}
impl DataPacketCryptor {
/// Create a new data packet cryptor with the specified algorithm and key provider
pub fn new(algorithm: ffi::Algorithm, key_provider: cxx::SharedPtr<ffi::KeyProvider>) -> Self {
Self { inner: ffi::new_data_packet_cryptor(algorithm, key_provider) }
}
/// Encrypt data for a specific participant
pub fn encrypt(
&self,
participant_id: &str,
key_index: u32,
data: &[u8],
) -> Result<ffi::EncryptedPacket, Box<dyn std::error::Error>> {
let data_vec: Vec<u8> = data.to_vec();
match self.inner.encrypt_data_packet(participant_id.to_string(), key_index, data_vec) {
Ok(packet) => Ok(packet),
Err(e) => Err(format!("Encryption failed: {}", e).into()),
}
}
/// Decrypt an encrypted packet for a specific participant
pub fn decrypt(
&self,
participant_id: &str,
encrypted_packet: &ffi::EncryptedPacket,
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
match self.inner.decrypt_data_packet(participant_id.to_string(), encrypted_packet) {
Ok(data) => Ok(data.into_iter().collect()),
Err(e) => Err(format!("Decryption failed: {}", e).into()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_data_packet_cryptor_creation() {
let options = ffi::KeyProviderOptions {
shared_key: true,
ratchet_window_size: 16,
ratchet_salt: vec![],
failure_tolerance: -1,
key_ring_size: 16,
key_derivation_algorithm: ffi::KeyDerivationAlgorithm::HKDF,
};
let key_provider = ffi::new_key_provider(options);
let _cryptor = DataPacketCryptor::new(ffi::Algorithm::AesGcm, key_provider);
}
}
@@ -0,0 +1,30 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/global_task_queue.h"
#include "api/task_queue/default_task_queue_factory.h"
#include "api/task_queue/task_queue_factory.h"
namespace livekit_ffi {
webrtc::TaskQueueFactory* GetGlobalTaskQueueFactory() {
static std::unique_ptr<webrtc::TaskQueueFactory> global_task_queue_factory =
webrtc::CreateDefaultTaskQueueFactory();
return global_task_queue_factory.get();
}
} // namespace livekit_ffi
@@ -0,0 +1,67 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
// Wrapper to opaque C++ objects
// https://github.com/dtolnay/cxx/issues/741
// Used to allow SharedPtr/UniquePtr type inside a rust::Vec
pub struct MediaStreamPtr {
pub ptr: SharedPtr<MediaStream>,
}
pub struct CandidatePtr {
pub ptr: SharedPtr<Candidate>,
}
pub struct AudioTrackPtr {
pub ptr: SharedPtr<AudioTrack>,
}
pub struct VideoTrackPtr {
pub ptr: SharedPtr<VideoTrack>,
}
pub struct RtpSenderPtr {
pub ptr: SharedPtr<RtpSender>,
}
pub struct RtpReceiverPtr {
pub ptr: SharedPtr<RtpReceiver>,
}
pub struct RtpTransceiverPtr {
pub ptr: SharedPtr<RtpTransceiver>,
}
unsafe extern "C++" {
include!("livekit/helper.h");
type MediaStream = crate::media_stream::ffi::MediaStream;
type AudioTrack = crate::media_stream::ffi::AudioTrack;
type VideoTrack = crate::media_stream::ffi::VideoTrack;
type Candidate = crate::candidate::ffi::Candidate;
type RtpSender = crate::rtp_sender::ffi::RtpSender;
type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver;
type RtpTransceiver = crate::rtp_transceiver::ffi::RtpTransceiver;
fn _vec_media_stream_ptr() -> Vec<MediaStreamPtr>;
fn _vec_candidate_ptr() -> Vec<CandidatePtr>;
fn _vec_audio_track_ptr() -> Vec<AudioTrackPtr>;
fn _vec_video_track_ptr() -> Vec<VideoTrackPtr>;
fn _vec_rtp_sender_ptr() -> Vec<RtpSenderPtr>;
fn _vec_rtp_receiver_ptr() -> Vec<RtpReceiverPtr>;
fn _vec_rtp_transceiver_ptr() -> Vec<RtpTransceiverPtr>;
}
}
@@ -0,0 +1,183 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/jsep.h"
#include <iomanip>
#include <memory>
#include "livekit/rtc_error.h"
#include "rtc_base/ref_counted_object.h"
#include "rust/cxx.h"
namespace livekit_ffi {
std::string serialize_sdp_error(webrtc::SdpParseError error) {
std::stringstream ss;
ss << std::hex << std::setfill('0');
ss << std::setw(8) << (uint32_t)error.line.length();
ss << std::dec << std::setw(1) << error.line;
ss << std::dec << std::setw(1) << error.description;
return ss.str();
}
IceCandidate::IceCandidate(
std::unique_ptr<webrtc::IceCandidateInterface> ice_candidate)
: ice_candidate_(std::move(ice_candidate)) {}
rust::String IceCandidate::sdp_mid() const {
return ice_candidate_->sdp_mid();
}
int IceCandidate::sdp_mline_index() const {
return ice_candidate_->sdp_mline_index();
}
rust::String IceCandidate::candidate() const {
return stringify();
}
rust::String IceCandidate::stringify() const {
std::string str;
ice_candidate_->ToString(&str);
return rust::String::lossy(str);
}
std::unique_ptr<webrtc::IceCandidateInterface> IceCandidate::release() {
return std::move(ice_candidate_);
}
std::shared_ptr<IceCandidate> create_ice_candidate(rust::String sdp_mid,
int sdp_mline_index,
rust::String sdp) {
webrtc::SdpParseError error;
auto ice_rtc = webrtc::CreateIceCandidate(sdp_mid.c_str(), sdp_mline_index,
sdp.c_str(), &error);
if (!ice_rtc) {
throw std::runtime_error(serialize_sdp_error(error));
}
return std::make_shared<IceCandidate>(
std::unique_ptr<webrtc::IceCandidateInterface>(ice_rtc));
}
SessionDescription::SessionDescription(
std::unique_ptr<webrtc::SessionDescriptionInterface> session_description)
: session_description_(std::move(session_description)) {}
SdpType SessionDescription::sdp_type() const {
return static_cast<SdpType>(session_description_->GetType());
}
rust::String SessionDescription::stringify() const {
std::string str;
session_description_->ToString(&str);
return rust::String::lossy(str);
}
std::unique_ptr<SessionDescription> SessionDescription::clone() const {
return std::make_unique<SessionDescription>(session_description_->Clone());
}
std::unique_ptr<webrtc::SessionDescriptionInterface>
SessionDescription::release() {
return std::move(session_description_);
}
std::unique_ptr<SessionDescription> create_session_description(
SdpType type,
rust::String sdp) {
webrtc::SdpParseError error;
auto rtc_sdp = webrtc::CreateSessionDescription(
static_cast<webrtc::SdpType>(type), sdp.c_str(), &error);
if (!rtc_sdp) {
throw std::runtime_error(serialize_sdp_error(error));
}
return std::make_unique<SessionDescription>(std::move(rtc_sdp));
}
#ifdef LIVEKIT_TEST
rust::String serialize_sdp_parse_error_for_test() {
webrtc::SdpParseError error;
auto rtc_sdp = webrtc::CreateSessionDescription(
webrtc::SdpType::kOffer,
"v=0\n"
"o=- 6549709950142776241 2 IN IP4 127.0.0.1\n"
"s=-\n"
"t=0 0\n"
"======================== ERROR HERE\n"
"a=group:BUNDLE 0\n"
"a=extmap-allow-mixed\n"
"a=msid-semantic: WMS\n"
"m=application 9 UDP/DTLS/SCTP webrtc-datachannel\n"
"c=IN IP4 0.0.0.0\n"
"a=ice-ufrag:Tw7h\n"
"a=ice-pwd:6XOVUD6HpcB4c1M8EB8jXJE9\n"
"a=ice-options:trickle\n"
"a=fingerprint:sha-256 "
"4F:EC:23:59:5D:A5:E6:3E:3E:5D:8A:09:B6:FA:04:AA:19:99:49:67:BD:65:93:06:BB:EE:AC:D5:21:0F:57:D6\n"
"a=setup:actpass\n"
"a=mid:0\n"
"a=sctp-port:5000\n"
"a=max-message-size:262144\n",
&error);
if (rtc_sdp) {
return rust::String("");
}
return rust::String::lossy(serialize_sdp_error(error));
}
#endif
NativeCreateSdpObserver::NativeCreateSdpObserver(
rust::Box<PeerContext> ctx,
rust::Fn<void(rust::Box<PeerContext>, std::unique_ptr<SessionDescription>)>
on_success,
rust::Fn<void(rust::Box<PeerContext>, RtcError)> on_error)
: ctx_(std::move(ctx)), on_success_(on_success), on_error_(on_error) {}
void NativeCreateSdpObserver::OnSuccess(
webrtc::SessionDescriptionInterface* desc) {
// We have ownership of desc
on_success_(std::move(ctx_),
std::make_unique<SessionDescription>(
std::unique_ptr<webrtc::SessionDescriptionInterface>(desc)));
}
void NativeCreateSdpObserver::OnFailure(webrtc::RTCError error) {
on_error_(std::move(ctx_), to_error(error));
}
NativeSetLocalSdpObserver::NativeSetLocalSdpObserver(
rust::Box<PeerContext> ctx,
rust::Fn<void(rust::Box<PeerContext>, RtcError)> on_complete)
: ctx_(std::move(ctx)), on_complete_(on_complete) {}
void NativeSetLocalSdpObserver::OnSetLocalDescriptionComplete(
webrtc::RTCError error) {
on_complete_(std::move(ctx_), to_error(error));
}
NativeSetRemoteSdpObserver::NativeSetRemoteSdpObserver(
rust::Box<PeerContext> ctx,
rust::Fn<void(rust::Box<PeerContext>, RtcError)> on_complete)
: ctx_(std::move(ctx)), on_complete_(on_complete) {}
void NativeSetRemoteSdpObserver::OnSetRemoteDescriptionComplete(
webrtc::RTCError error) {
on_complete_(std::move(ctx_), to_error(error));
}
} // namespace livekit_ffi
@@ -0,0 +1,123 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{
error::Error,
fmt::{Display, Formatter},
};
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
#[derive(Debug)]
#[repr(i32)]
pub enum SdpType {
Offer,
PrAnswer,
Answer,
Rollback,
}
#[derive(Debug)]
pub struct SdpParseError {
pub line: String,
pub description: String,
}
extern "C++" {
include!("livekit/rtc_error.h");
type RtcError = crate::rtc_error::ffi::RtcError;
}
unsafe extern "C++" {
include!("livekit/jsep.h");
type IceCandidate;
type SessionDescription;
fn sdp_mid(self: &IceCandidate) -> String;
fn sdp_mline_index(self: &IceCandidate) -> i32;
fn candidate(self: &IceCandidate) -> String;
fn stringify(self: &IceCandidate) -> String;
fn sdp_type(self: &SessionDescription) -> SdpType;
fn stringify(self: &SessionDescription) -> String;
fn clone(self: &SessionDescription) -> UniquePtr<SessionDescription>;
fn create_ice_candidate(
sdp_mid: String,
sdp_mline_index: i32,
sdp: String,
) -> Result<SharedPtr<IceCandidate>>;
fn create_session_description(
sdp_type: SdpType,
sdp: String,
) -> Result<UniquePtr<SessionDescription>>;
fn _shared_ice_candidate() -> SharedPtr<IceCandidate>; // Ignore
fn _unique_session_description() -> UniquePtr<SessionDescription>; // Ignore
}
}
impl Error for ffi::SdpParseError {}
impl Display for ffi::SdpParseError {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
write!(f, "SdpParseError occurred {}: {}", self.line, self.description)
}
}
impl_thread_safety!(ffi::SessionDescription, Send + Sync);
impl_thread_safety!(ffi::IceCandidate, Send + Sync);
impl ffi::SdpParseError {
/// # Safety
/// The value must be correctly encoded
pub unsafe fn from(value: &str) -> Self {
// Parse the hex encoded error from c++
let line_length = u32::from_str_radix(&value[0..8], 16).unwrap() as usize + 8;
let line = String::from(&value[8..line_length]);
let description = String::from(&value[line_length..]);
Self { line, description }
}
}
#[cfg(test)]
mod tests {
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi_tests {
unsafe extern "C++" {
include!("livekit/jsep.h");
fn serialize_sdp_parse_error_for_test() -> String;
}
}
use crate::jsep::ffi;
/// Tests that SdpParseError can correctly deserialize the hex-encoded
/// error format produced by C++ when SDP parsing fails.
#[test]
fn sdp_parse_error_deserialization() {
let serialized = ffi_tests::serialize_sdp_parse_error_for_test();
let err = unsafe { ffi::SdpParseError::from(&serialized) };
assert!(!err.line.is_empty(), "error line should not be empty");
assert!(!err.description.is_empty(), "error description should not be empty");
}
}
@@ -0,0 +1,256 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXcomposite.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXcomposite.so.1");
CHECK(lib_handle, "failed to load library 'libXcomposite.so.1' via callback ''");
#else
lib_handle = dlopen("libXcomposite.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXcomposite.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"XCompositeCreateRegionFromBorderClip",
"XCompositeFindDisplay",
"XCompositeGetOverlayWindow",
"XCompositeNameWindowPixmap",
"XCompositeQueryExtension",
"XCompositeQueryVersion",
"XCompositeRedirectSubwindows",
"XCompositeRedirectWindow",
"XCompositeReleaseOverlayWindow",
"XCompositeUnredirectSubwindows",
"XCompositeUnredirectWindow",
"XCompositeVersion",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXcomposite_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXcomposite_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXcomposite_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXcomposite_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXcomposite_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXcomposite_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXcomposite_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXcomposite_so_tramp_table, 0, SYM_COUNT * sizeof(_libXcomposite_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,573 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#define lr x30
#define ip0 x16
.section .note.GNU-stack,"",@progbits
.data
.globl _libXcomposite_so_tramp_table
.hidden _libXcomposite_so_tramp_table
.align 8
_libXcomposite_so_tramp_table:
.zero 104
.text
.globl _libXcomposite_so_tramp_resolve
.hidden _libXcomposite_so_tramp_resolve
.globl _libXcomposite_so_save_regs_and_resolve
.hidden _libXcomposite_so_save_regs_and_resolve
.type _libXcomposite_so_save_regs_and_resolve, %function
_libXcomposite_so_save_regs_and_resolve:
.cfi_startproc
// Slow path which calls dlsym, taken only on first call.
// Registers are saved according to "Procedure Call Standard for the Arm® 64-bit Architecture".
// For DWARF directives, read https://www.imperialviolet.org/2017/01/18/cfi.html.
// Stack is aligned at 16 bytes
#define PUSH_PAIR(reg1, reg2) stp reg1, reg2, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset reg1, 0; .cfi_rel_offset reg2, 8
#define POP_PAIR(reg1, reg2) ldp reg1, reg2, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore reg2; .cfi_restore reg1
#define PUSH_WIDE_PAIR(reg1, reg2) stp reg1, reg2, [sp, #-32]!; .cfi_adjust_cfa_offset 32; .cfi_rel_offset reg1, 0; .cfi_rel_offset reg2, 16
#define POP_WIDE_PAIR(reg1, reg2) ldp reg1, reg2, [sp], #32; .cfi_adjust_cfa_offset -32; .cfi_restore reg2; .cfi_restore reg1
// Save only arguments (and lr)
PUSH_PAIR(x0, x1)
PUSH_PAIR(x2, x3)
PUSH_PAIR(x4, x5)
PUSH_PAIR(x6, x7)
PUSH_PAIR(x8, lr)
ldr x0, [sp, #80] // 16*5
PUSH_WIDE_PAIR(q0, q1)
PUSH_WIDE_PAIR(q2, q3)
PUSH_WIDE_PAIR(q4, q5)
PUSH_WIDE_PAIR(q6, q7)
// Stack is aligned at 16 bytes
bl _libXcomposite_so_tramp_resolve
mov ip0, x0
// TODO: pop pc?
POP_WIDE_PAIR(q6, q7)
POP_WIDE_PAIR(q4, q5)
POP_WIDE_PAIR(q2, q3)
POP_WIDE_PAIR(q0, q1)
POP_PAIR(x8, lr)
POP_PAIR(x6, x7)
POP_PAIR(x4, x5)
POP_PAIR(x2, x3)
POP_PAIR(x0, x1)
br lr
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeCreateRegionFromBorderClip
.p2align 4
.type XCompositeCreateRegionFromBorderClip, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeCreateRegionFromBorderClip
#endif
XCompositeCreateRegionFromBorderClip:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+0
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+0]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 0 & 0xffff
#if 0 > 0xffff
movk ip0, 0 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeFindDisplay
.p2align 4
.type XCompositeFindDisplay, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeFindDisplay
#endif
XCompositeFindDisplay:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+8
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+8]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 1 & 0xffff
#if 1 > 0xffff
movk ip0, 1 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeGetOverlayWindow
.p2align 4
.type XCompositeGetOverlayWindow, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeGetOverlayWindow
#endif
XCompositeGetOverlayWindow:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+16
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+16]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 2 & 0xffff
#if 2 > 0xffff
movk ip0, 2 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeNameWindowPixmap
.p2align 4
.type XCompositeNameWindowPixmap, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeNameWindowPixmap
#endif
XCompositeNameWindowPixmap:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+24
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+24]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 3 & 0xffff
#if 3 > 0xffff
movk ip0, 3 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeQueryExtension
.p2align 4
.type XCompositeQueryExtension, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeQueryExtension
#endif
XCompositeQueryExtension:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+32
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+32]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 4 & 0xffff
#if 4 > 0xffff
movk ip0, 4 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeQueryVersion
.p2align 4
.type XCompositeQueryVersion, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeQueryVersion
#endif
XCompositeQueryVersion:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+40
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+40]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 5 & 0xffff
#if 5 > 0xffff
movk ip0, 5 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeRedirectSubwindows
.p2align 4
.type XCompositeRedirectSubwindows, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeRedirectSubwindows
#endif
XCompositeRedirectSubwindows:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+48
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+48]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 6 & 0xffff
#if 6 > 0xffff
movk ip0, 6 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeRedirectWindow
.p2align 4
.type XCompositeRedirectWindow, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeRedirectWindow
#endif
XCompositeRedirectWindow:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+56
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+56]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 7 & 0xffff
#if 7 > 0xffff
movk ip0, 7 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeReleaseOverlayWindow
.p2align 4
.type XCompositeReleaseOverlayWindow, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeReleaseOverlayWindow
#endif
XCompositeReleaseOverlayWindow:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+64
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+64]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 8 & 0xffff
#if 8 > 0xffff
movk ip0, 8 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeUnredirectSubwindows
.p2align 4
.type XCompositeUnredirectSubwindows, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeUnredirectSubwindows
#endif
XCompositeUnredirectSubwindows:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+72
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+72]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 9 & 0xffff
#if 9 > 0xffff
movk ip0, 9 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeUnredirectWindow
.p2align 4
.type XCompositeUnredirectWindow, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeUnredirectWindow
#endif
XCompositeUnredirectWindow:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+80
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+80]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 10 & 0xffff
#if 10 > 0xffff
movk ip0, 10 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeVersion
.p2align 4
.type XCompositeVersion, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeVersion
#endif
XCompositeVersion:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXcomposite_so_tramp_table+88
ldr ip0, [ip0, #:lo12:_libXcomposite_so_tramp_table+88]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 11 & 0xffff
#if 11 > 0xffff
movk ip0, 11 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXcomposite_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
@@ -0,0 +1,251 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXdamage.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXdamage.so.1");
CHECK(lib_handle, "failed to load library 'libXdamage.so.1' via callback ''");
#else
lib_handle = dlopen("libXdamage.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXdamage.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"XDamageAdd",
"XDamageCreate",
"XDamageDestroy",
"XDamageFindDisplay",
"XDamageQueryExtension",
"XDamageQueryVersion",
"XDamageSubtract",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXdamage_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXdamage_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXdamage_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXdamage_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXdamage_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXdamage_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXdamage_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXdamage_so_tramp_table, 0, SYM_COUNT * sizeof(_libXdamage_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,368 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#define lr x30
#define ip0 x16
.section .note.GNU-stack,"",@progbits
.data
.globl _libXdamage_so_tramp_table
.hidden _libXdamage_so_tramp_table
.align 8
_libXdamage_so_tramp_table:
.zero 64
.text
.globl _libXdamage_so_tramp_resolve
.hidden _libXdamage_so_tramp_resolve
.globl _libXdamage_so_save_regs_and_resolve
.hidden _libXdamage_so_save_regs_and_resolve
.type _libXdamage_so_save_regs_and_resolve, %function
_libXdamage_so_save_regs_and_resolve:
.cfi_startproc
// Slow path which calls dlsym, taken only on first call.
// Registers are saved according to "Procedure Call Standard for the Arm® 64-bit Architecture".
// For DWARF directives, read https://www.imperialviolet.org/2017/01/18/cfi.html.
// Stack is aligned at 16 bytes
#define PUSH_PAIR(reg1, reg2) stp reg1, reg2, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset reg1, 0; .cfi_rel_offset reg2, 8
#define POP_PAIR(reg1, reg2) ldp reg1, reg2, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore reg2; .cfi_restore reg1
#define PUSH_WIDE_PAIR(reg1, reg2) stp reg1, reg2, [sp, #-32]!; .cfi_adjust_cfa_offset 32; .cfi_rel_offset reg1, 0; .cfi_rel_offset reg2, 16
#define POP_WIDE_PAIR(reg1, reg2) ldp reg1, reg2, [sp], #32; .cfi_adjust_cfa_offset -32; .cfi_restore reg2; .cfi_restore reg1
// Save only arguments (and lr)
PUSH_PAIR(x0, x1)
PUSH_PAIR(x2, x3)
PUSH_PAIR(x4, x5)
PUSH_PAIR(x6, x7)
PUSH_PAIR(x8, lr)
ldr x0, [sp, #80] // 16*5
PUSH_WIDE_PAIR(q0, q1)
PUSH_WIDE_PAIR(q2, q3)
PUSH_WIDE_PAIR(q4, q5)
PUSH_WIDE_PAIR(q6, q7)
// Stack is aligned at 16 bytes
bl _libXdamage_so_tramp_resolve
mov ip0, x0
// TODO: pop pc?
POP_WIDE_PAIR(q6, q7)
POP_WIDE_PAIR(q4, q5)
POP_WIDE_PAIR(q2, q3)
POP_WIDE_PAIR(q0, q1)
POP_PAIR(x8, lr)
POP_PAIR(x6, x7)
POP_PAIR(x4, x5)
POP_PAIR(x2, x3)
POP_PAIR(x0, x1)
br lr
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageAdd
.p2align 4
.type XDamageAdd, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageAdd
#endif
XDamageAdd:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXdamage_so_tramp_table+0
ldr ip0, [ip0, #:lo12:_libXdamage_so_tramp_table+0]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 0 & 0xffff
#if 0 > 0xffff
movk ip0, 0 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXdamage_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageCreate
.p2align 4
.type XDamageCreate, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageCreate
#endif
XDamageCreate:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXdamage_so_tramp_table+8
ldr ip0, [ip0, #:lo12:_libXdamage_so_tramp_table+8]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 1 & 0xffff
#if 1 > 0xffff
movk ip0, 1 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXdamage_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageDestroy
.p2align 4
.type XDamageDestroy, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageDestroy
#endif
XDamageDestroy:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXdamage_so_tramp_table+16
ldr ip0, [ip0, #:lo12:_libXdamage_so_tramp_table+16]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 2 & 0xffff
#if 2 > 0xffff
movk ip0, 2 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXdamage_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageFindDisplay
.p2align 4
.type XDamageFindDisplay, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageFindDisplay
#endif
XDamageFindDisplay:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXdamage_so_tramp_table+24
ldr ip0, [ip0, #:lo12:_libXdamage_so_tramp_table+24]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 3 & 0xffff
#if 3 > 0xffff
movk ip0, 3 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXdamage_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageQueryExtension
.p2align 4
.type XDamageQueryExtension, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageQueryExtension
#endif
XDamageQueryExtension:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXdamage_so_tramp_table+32
ldr ip0, [ip0, #:lo12:_libXdamage_so_tramp_table+32]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 4 & 0xffff
#if 4 > 0xffff
movk ip0, 4 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXdamage_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageQueryVersion
.p2align 4
.type XDamageQueryVersion, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageQueryVersion
#endif
XDamageQueryVersion:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXdamage_so_tramp_table+40
ldr ip0, [ip0, #:lo12:_libXdamage_so_tramp_table+40]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 5 & 0xffff
#if 5 > 0xffff
movk ip0, 5 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXdamage_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageSubtract
.p2align 4
.type XDamageSubtract, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageSubtract
#endif
XDamageSubtract:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libXdamage_so_tramp_table+48
ldr ip0, [ip0, #:lo12:_libXdamage_so_tramp_table+48]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 6 & 0xffff
#if 6 > 0xffff
movk ip0, 6 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libXdamage_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
@@ -0,0 +1,376 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXext.so.6: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXext.so.6");
CHECK(lib_handle, "failed to load library 'libXext.so.6' via callback ''");
#else
lib_handle = dlopen("libXext.so.6", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXext.so.6' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"DPMSCapable",
"DPMSDisable",
"DPMSEnable",
"DPMSForceLevel",
"DPMSGetTimeouts",
"DPMSGetVersion",
"DPMSInfo",
"DPMSQueryExtension",
"DPMSSetTimeouts",
"XGEQueryExtension",
"XGEQueryVersion",
"XLbxGetEventBase",
"XLbxQueryExtension",
"XLbxQueryVersion",
"XMITMiscGetBugMode",
"XMITMiscQueryExtension",
"XMITMiscSetBugMode",
"XMissingExtension",
"XSecurityAllocXauth",
"XSecurityFreeXauth",
"XSecurityGenerateAuthorization",
"XSecurityQueryExtension",
"XSecurityRevokeAuthorization",
"XSetExtensionErrorHandler",
"XShapeCombineMask",
"XShapeCombineRectangles",
"XShapeCombineRegion",
"XShapeCombineShape",
"XShapeGetRectangles",
"XShapeInputSelected",
"XShapeOffsetShape",
"XShapeQueryExtension",
"XShapeQueryExtents",
"XShapeQueryVersion",
"XShapeSelectInput",
"XShmAttach",
"XShmCreateImage",
"XShmCreatePixmap",
"XShmDetach",
"XShmGetEventBase",
"XShmGetImage",
"XShmPixmapFormat",
"XShmPutImage",
"XShmQueryExtension",
"XShmQueryVersion",
"XSyncAwait",
"XSyncAwaitFence",
"XSyncChangeAlarm",
"XSyncChangeCounter",
"XSyncCreateAlarm",
"XSyncCreateCounter",
"XSyncCreateFence",
"XSyncDestroyAlarm",
"XSyncDestroyCounter",
"XSyncDestroyFence",
"XSyncFreeSystemCounterList",
"XSyncGetPriority",
"XSyncInitialize",
"XSyncIntToValue",
"XSyncIntsToValue",
"XSyncListSystemCounters",
"XSyncMaxValue",
"XSyncMinValue",
"XSyncQueryAlarm",
"XSyncQueryCounter",
"XSyncQueryExtension",
"XSyncQueryFence",
"XSyncResetFence",
"XSyncSetCounter",
"XSyncSetPriority",
"XSyncTriggerFence",
"XSyncValueAdd",
"XSyncValueEqual",
"XSyncValueGreaterOrEqual",
"XSyncValueGreaterThan",
"XSyncValueHigh32",
"XSyncValueIsNegative",
"XSyncValueIsPositive",
"XSyncValueIsZero",
"XSyncValueLessOrEqual",
"XSyncValueLessThan",
"XSyncValueLow32",
"XSyncValueSubtract",
"XTestFakeInput",
"XTestFlush",
"XTestGetInput",
"XTestMovePointer",
"XTestPressButton",
"XTestPressKey",
"XTestQueryInputSize",
"XTestReset",
"XTestStopInput",
"XagCreateAssociation",
"XagCreateEmbeddedApplicationGroup",
"XagCreateNonembeddedApplicationGroup",
"XagDestroyApplicationGroup",
"XagDestroyAssociation",
"XagGetApplicationGroupAttributes",
"XagQueryApplicationGroup",
"XagQueryVersion",
"XcupGetReservedColormapEntries",
"XcupQueryVersion",
"XcupStoreColors",
"XdbeAllocateBackBufferName",
"XdbeBeginIdiom",
"XdbeDeallocateBackBufferName",
"XdbeEndIdiom",
"XdbeFreeVisualInfo",
"XdbeGetBackBufferAttributes",
"XdbeGetVisualInfo",
"XdbeQueryExtension",
"XdbeSwapBuffers",
"XeviGetVisualInfo",
"XeviQueryExtension",
"XeviQueryVersion",
"XextAddDisplay",
"XextCreateExtension",
"XextDestroyExtension",
"XextFindDisplay",
"XextRemoveDisplay",
"XmbufChangeBufferAttributes",
"XmbufChangeWindowAttributes",
"XmbufClearBufferArea",
"XmbufCreateBuffers",
"XmbufCreateStereoWindow",
"XmbufDestroyBuffers",
"XmbufDisplayBuffers",
"XmbufGetBufferAttributes",
"XmbufGetScreenInfo",
"XmbufGetVersion",
"XmbufGetWindowAttributes",
"XmbufQueryExtension",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXext_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXext_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXext_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXext_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXext_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXext_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXext_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXext_so_tramp_table, 0, SYM_COUNT * sizeof(_libXext_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,282 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXfixes.so.3: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXfixes.so.3");
CHECK(lib_handle, "failed to load library 'libXfixes.so.3' via callback ''");
#else
lib_handle = dlopen("libXfixes.so.3", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXfixes.so.3' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"XFixesChangeCursor",
"XFixesChangeCursorByName",
"XFixesChangeSaveSet",
"XFixesCopyRegion",
"XFixesCreatePointerBarrier",
"XFixesCreateRegion",
"XFixesCreateRegionFromBitmap",
"XFixesCreateRegionFromGC",
"XFixesCreateRegionFromPicture",
"XFixesCreateRegionFromWindow",
"XFixesDestroyPointerBarrier",
"XFixesDestroyRegion",
"XFixesExpandRegion",
"XFixesFetchRegion",
"XFixesFetchRegionAndBounds",
"XFixesFindDisplay",
"XFixesGetClientDisconnectMode",
"XFixesGetCursorImage",
"XFixesGetCursorName",
"XFixesHideCursor",
"XFixesIntersectRegion",
"XFixesInvertRegion",
"XFixesQueryExtension",
"XFixesQueryVersion",
"XFixesRegionExtents",
"XFixesSelectCursorInput",
"XFixesSelectSelectionInput",
"XFixesSetClientDisconnectMode",
"XFixesSetCursorName",
"XFixesSetGCClipRegion",
"XFixesSetPictureClipRegion",
"XFixesSetRegion",
"XFixesSetWindowShapeRegion",
"XFixesShowCursor",
"XFixesSubtractRegion",
"XFixesTranslateRegion",
"XFixesUnionRegion",
"XFixesVersion",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXfixes_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXfixes_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXfixes_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXfixes_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXfixes_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXfixes_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXfixes_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXfixes_so_tramp_table, 0, SYM_COUNT * sizeof(_libXfixes_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,314 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXrandr.so.2: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXrandr.so.2");
CHECK(lib_handle, "failed to load library 'libXrandr.so.2' via callback ''");
#else
lib_handle = dlopen("libXrandr.so.2", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXrandr.so.2' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"XRRAddOutputMode",
"XRRAllocGamma",
"XRRAllocModeInfo",
"XRRAllocateMonitor",
"XRRChangeOutputProperty",
"XRRChangeProviderProperty",
"XRRConfigCurrentConfiguration",
"XRRConfigCurrentRate",
"XRRConfigRates",
"XRRConfigRotations",
"XRRConfigSizes",
"XRRConfigTimes",
"XRRConfigureOutputProperty",
"XRRConfigureProviderProperty",
"XRRCreateMode",
"XRRDeleteMonitor",
"XRRDeleteOutputMode",
"XRRDeleteOutputProperty",
"XRRDeleteProviderProperty",
"XRRDestroyMode",
"XRRFreeCrtcInfo",
"XRRFreeGamma",
"XRRFreeModeInfo",
"XRRFreeMonitors",
"XRRFreeOutputInfo",
"XRRFreePanning",
"XRRFreeProviderInfo",
"XRRFreeProviderResources",
"XRRFreeScreenConfigInfo",
"XRRFreeScreenResources",
"XRRGetCrtcGamma",
"XRRGetCrtcGammaSize",
"XRRGetCrtcInfo",
"XRRGetCrtcTransform",
"XRRGetMonitors",
"XRRGetOutputInfo",
"XRRGetOutputPrimary",
"XRRGetOutputProperty",
"XRRGetPanning",
"XRRGetProviderInfo",
"XRRGetProviderProperty",
"XRRGetProviderResources",
"XRRGetScreenInfo",
"XRRGetScreenResources",
"XRRGetScreenResourcesCurrent",
"XRRGetScreenSizeRange",
"XRRListOutputProperties",
"XRRListProviderProperties",
"XRRQueryExtension",
"XRRQueryOutputProperty",
"XRRQueryProviderProperty",
"XRRQueryVersion",
"XRRRates",
"XRRRootToScreen",
"XRRRotations",
"XRRSelectInput",
"XRRSetCrtcConfig",
"XRRSetCrtcGamma",
"XRRSetCrtcTransform",
"XRRSetMonitor",
"XRRSetOutputPrimary",
"XRRSetPanning",
"XRRSetProviderOffloadSink",
"XRRSetProviderOutputSource",
"XRRSetScreenConfig",
"XRRSetScreenConfigAndRate",
"XRRSetScreenSize",
"XRRSizes",
"XRRTimes",
"XRRUpdateConfiguration",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXrandr_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXrandr_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXrandr_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXrandr_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXrandr_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXrandr_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXrandr_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXrandr_so_tramp_table, 0, SYM_COUNT * sizeof(_libXrandr_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,456 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libdrm.so.2: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libdrm.so.2");
CHECK(lib_handle, "failed to load library 'libdrm.so.2' via callback ''");
#else
lib_handle = dlopen("libdrm.so.2", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libdrm.so.2' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"drmAddBufs",
"drmAddContextPrivateMapping",
"drmAddContextTag",
"drmAddMap",
"drmAgpAcquire",
"drmAgpAlloc",
"drmAgpBase",
"drmAgpBind",
"drmAgpDeviceId",
"drmAgpEnable",
"drmAgpFree",
"drmAgpGetMode",
"drmAgpMemoryAvail",
"drmAgpMemoryUsed",
"drmAgpRelease",
"drmAgpSize",
"drmAgpUnbind",
"drmAgpVendorId",
"drmAgpVersionMajor",
"drmAgpVersionMinor",
"drmAuthMagic",
"drmAvailable",
"drmCheckModesettingSupported",
"drmClose",
"drmCloseBufferHandle",
"drmCloseOnce",
"drmCommandNone",
"drmCommandRead",
"drmCommandWrite",
"drmCommandWriteRead",
"drmCreateContext",
"drmCreateDrawable",
"drmCrtcGetSequence",
"drmCrtcQueueSequence",
"drmCtlInstHandler",
"drmCtlUninstHandler",
"drmDMA",
"drmDelContextTag",
"drmDestroyContext",
"drmDestroyDrawable",
"drmDevicesEqual",
"drmDropMaster",
"drmError",
"drmFinish",
"drmFree",
"drmFreeBufs",
"drmFreeBusid",
"drmFreeDevice",
"drmFreeDevices",
"drmFreeReservedContextList",
"drmFreeVersion",
"drmGetBufInfo",
"drmGetBusid",
"drmGetCap",
"drmGetClient",
"drmGetContextFlags",
"drmGetContextPrivateMapping",
"drmGetContextTag",
"drmGetDevice",
"drmGetDevice2",
"drmGetDeviceFromDevId",
"drmGetDeviceNameFromFd",
"drmGetDeviceNameFromFd2",
"drmGetDevices",
"drmGetDevices2",
"drmGetEntry",
"drmGetFormatModifierName",
"drmGetFormatModifierVendor",
"drmGetFormatName",
"drmGetHashTable",
"drmGetInterruptFromBusID",
"drmGetLibVersion",
"drmGetLock",
"drmGetMagic",
"drmGetMap",
"drmGetNodeTypeFromDevId",
"drmGetNodeTypeFromFd",
"drmGetPrimaryDeviceNameFromFd",
"drmGetRenderDeviceNameFromFd",
"drmGetReservedContextList",
"drmGetStats",
"drmGetVersion",
"drmHandleEvent",
"drmHashCreate",
"drmHashDelete",
"drmHashDestroy",
"drmHashFirst",
"drmHashInsert",
"drmHashLookup",
"drmHashNext",
"drmIoctl",
"drmIsKMS",
"drmIsMaster",
"drmMalloc",
"drmMap",
"drmMapBufs",
"drmMarkBufs",
"drmModeAddFB",
"drmModeAddFB2",
"drmModeAddFB2WithModifiers",
"drmModeAtomicAddProperty",
"drmModeAtomicAlloc",
"drmModeAtomicCommit",
"drmModeAtomicDuplicate",
"drmModeAtomicFree",
"drmModeAtomicGetCursor",
"drmModeAtomicMerge",
"drmModeAtomicSetCursor",
"drmModeAttachMode",
"drmModeCloseFB",
"drmModeConnectorGetPossibleCrtcs",
"drmModeConnectorSetProperty",
"drmModeCreateDumbBuffer",
"drmModeCreateLease",
"drmModeCreatePropertyBlob",
"drmModeCrtcGetGamma",
"drmModeCrtcSetGamma",
"drmModeDestroyDumbBuffer",
"drmModeDestroyPropertyBlob",
"drmModeDetachMode",
"drmModeDirtyFB",
"drmModeFormatModifierBlobIterNext",
"drmModeFreeConnector",
"drmModeFreeCrtc",
"drmModeFreeEncoder",
"drmModeFreeFB",
"drmModeFreeFB2",
"drmModeFreeModeInfo",
"drmModeFreeObjectProperties",
"drmModeFreePlane",
"drmModeFreePlaneResources",
"drmModeFreeProperty",
"drmModeFreePropertyBlob",
"drmModeFreeResources",
"drmModeGetConnector",
"drmModeGetConnectorCurrent",
"drmModeGetConnectorTypeName",
"drmModeGetCrtc",
"drmModeGetEncoder",
"drmModeGetFB",
"drmModeGetFB2",
"drmModeGetLease",
"drmModeGetPlane",
"drmModeGetPlaneResources",
"drmModeGetProperty",
"drmModeGetPropertyBlob",
"drmModeGetResources",
"drmModeListLessees",
"drmModeMapDumbBuffer",
"drmModeMoveCursor",
"drmModeObjectGetProperties",
"drmModeObjectSetProperty",
"drmModePageFlip",
"drmModePageFlipTarget",
"drmModeRevokeLease",
"drmModeRmFB",
"drmModeSetCrtc",
"drmModeSetCursor",
"drmModeSetCursor2",
"drmModeSetPlane",
"drmMsg",
"drmOpen",
"drmOpenControl",
"drmOpenOnce",
"drmOpenOnceWithType",
"drmOpenRender",
"drmOpenWithType",
"drmPrimeFDToHandle",
"drmPrimeHandleToFD",
"drmRandom",
"drmRandomCreate",
"drmRandomDestroy",
"drmRandomDouble",
"drmRmMap",
"drmSLCreate",
"drmSLDelete",
"drmSLDestroy",
"drmSLDump",
"drmSLFirst",
"drmSLInsert",
"drmSLLookup",
"drmSLLookupNeighbors",
"drmSLNext",
"drmScatterGatherAlloc",
"drmScatterGatherFree",
"drmSetBusid",
"drmSetClientCap",
"drmSetContextFlags",
"drmSetInterfaceVersion",
"drmSetMaster",
"drmSetServerInfo",
"drmSwitchToContext",
"drmSyncobjCreate",
"drmSyncobjDestroy",
"drmSyncobjEventfd",
"drmSyncobjExportSyncFile",
"drmSyncobjFDToHandle",
"drmSyncobjHandleToFD",
"drmSyncobjImportSyncFile",
"drmSyncobjQuery",
"drmSyncobjQuery2",
"drmSyncobjReset",
"drmSyncobjSignal",
"drmSyncobjTimelineSignal",
"drmSyncobjTimelineWait",
"drmSyncobjTransfer",
"drmSyncobjWait",
"drmUnlock",
"drmUnmap",
"drmUnmapBufs",
"drmUpdateDrawableInfo",
"drmWaitVBlank",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libdrm_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libdrm_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libdrm_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libdrm_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libdrm_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libdrm_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libdrm_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libdrm_so_tramp_table, 0, SYM_COUNT * sizeof(_libdrm_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,282 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libgbm.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libgbm.so.1");
CHECK(lib_handle, "failed to load library 'libgbm.so.1' via callback ''");
#else
lib_handle = dlopen("libgbm.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libgbm.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"gbm_bo_create",
"gbm_bo_create_with_modifiers",
"gbm_bo_create_with_modifiers2",
"gbm_bo_destroy",
"gbm_bo_get_bpp",
"gbm_bo_get_device",
"gbm_bo_get_fd",
"gbm_bo_get_fd_for_plane",
"gbm_bo_get_format",
"gbm_bo_get_handle",
"gbm_bo_get_handle_for_plane",
"gbm_bo_get_height",
"gbm_bo_get_modifier",
"gbm_bo_get_offset",
"gbm_bo_get_plane_count",
"gbm_bo_get_stride",
"gbm_bo_get_stride_for_plane",
"gbm_bo_get_user_data",
"gbm_bo_get_width",
"gbm_bo_import",
"gbm_bo_map",
"gbm_bo_set_user_data",
"gbm_bo_unmap",
"gbm_bo_write",
"gbm_create_device",
"gbm_device_destroy",
"gbm_device_get_backend_name",
"gbm_device_get_fd",
"gbm_device_get_format_modifier_plane_count",
"gbm_device_is_format_supported",
"gbm_format_get_name",
"gbm_surface_create",
"gbm_surface_create_with_modifiers",
"gbm_surface_create_with_modifiers2",
"gbm_surface_destroy",
"gbm_surface_has_free_buffers",
"gbm_surface_lock_front_buffer",
"gbm_surface_release_buffer",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libgbm_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libgbm_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libgbm_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libgbm_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libgbm_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libgbm_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libgbm_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libgbm_so_tramp_table, 0, SYM_COUNT * sizeof(_libgbm_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,256 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXcomposite.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXcomposite.so.1");
CHECK(lib_handle, "failed to load library 'libXcomposite.so.1' via callback ''");
#else
lib_handle = dlopen("libXcomposite.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXcomposite.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"XCompositeCreateRegionFromBorderClip",
"XCompositeFindDisplay",
"XCompositeGetOverlayWindow",
"XCompositeNameWindowPixmap",
"XCompositeQueryExtension",
"XCompositeQueryVersion",
"XCompositeRedirectSubwindows",
"XCompositeRedirectWindow",
"XCompositeReleaseOverlayWindow",
"XCompositeUnredirectSubwindows",
"XCompositeUnredirectWindow",
"XCompositeVersion",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXcomposite_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXcomposite_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXcomposite_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXcomposite_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXcomposite_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXcomposite_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXcomposite_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXcomposite_so_tramp_table, 0, SYM_COUNT * sizeof(_libXcomposite_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,566 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.section .note.GNU-stack,"",@progbits
.data
.globl _libXcomposite_so_tramp_table
.hidden _libXcomposite_so_tramp_table
.align 8
_libXcomposite_so_tramp_table:
.zero 104
.text
.globl _libXcomposite_so_tramp_resolve
.hidden _libXcomposite_so_tramp_resolve
.globl _libXcomposite_so_save_regs_and_resolve
.hidden _libXcomposite_so_save_regs_and_resolve
.type _libXcomposite_so_save_regs_and_resolve, %function
_libXcomposite_so_save_regs_and_resolve:
.cfi_startproc
#define PUSH_REG(reg) pushq %reg ; .cfi_adjust_cfa_offset 8; .cfi_rel_offset reg, 0
#define POP_REG(reg) popq %reg ; .cfi_adjust_cfa_offset -8; .cfi_restore reg
#define DEC_STACK(d) subq $d, %rsp; .cfi_adjust_cfa_offset d
#define INC_STACK(d) addq $d, %rsp; .cfi_adjust_cfa_offset -d
#define PUSH_MMX_REG(reg) DEC_STACK(8); movq %reg, (%rsp); .cfi_rel_offset reg, 0
#define POP_MMX_REG(reg) movq (%rsp), %reg; .cfi_restore reg; INC_STACK(8)
#define PUSH_XMM_REG(reg) DEC_STACK(16); movdqa %reg, (%rsp); .cfi_rel_offset reg, 0
#define POP_XMM_REG(reg) movdqa (%rsp), %reg; .cfi_restore reg; INC_STACK(16)
// TODO: cfi_offset/cfi_restore
#define PUSH_YMM_REG(reg) DEC_STACK(32); vmovdqu %reg, (%rsp)
#define POP_YMM_REG(reg) vmovdqu (%rsp), %reg; INC_STACK(32)
// TODO: cfi_offset/cfi_restore
#define PUSH_ZMM_REG(reg) DEC_STACK(64); vmovdqu32 %reg, (%rsp)
#define POP_ZMM_REG(reg) vmovdqu32 (%rsp), %reg; INC_STACK(64)
// Slow path which calls dlsym, taken only on first call.
// All registers are stored to handle arbitrary calling conventions
// (except x87 FPU registers which do not have to be preserved).
// For Dwarf directives, read https://www.imperialviolet.org/2017/01/18/cfi.html.
.cfi_def_cfa_offset 8 // Return address
PUSH_REG(rdi) // 16
mov 0x10(%rsp), %rdi
PUSH_REG(rbx)
PUSH_REG(rbx) // 16
PUSH_REG(rcx)
PUSH_REG(rdx) // 16
PUSH_REG(rbp)
PUSH_REG(rsi) // 16
PUSH_REG(r8)
PUSH_REG(r9) // 16
PUSH_REG(r10)
PUSH_REG(r11) // 16
PUSH_REG(r12)
PUSH_REG(r13) // 16
PUSH_REG(r14)
PUSH_REG(r15) // 16
// Maybe use cpuid instead of macro to detect current vector size...
#ifdef __AVX512F__
PUSH_ZMM_REG(zmm0)
PUSH_ZMM_REG(zmm1)
PUSH_ZMM_REG(zmm2)
PUSH_ZMM_REG(zmm3)
PUSH_ZMM_REG(zmm4)
PUSH_ZMM_REG(zmm5)
PUSH_ZMM_REG(zmm6)
PUSH_ZMM_REG(zmm7)
#elif defined __AVX__
PUSH_YMM_REG(ymm0)
PUSH_YMM_REG(ymm1)
PUSH_YMM_REG(ymm2)
PUSH_YMM_REG(ymm3)
PUSH_YMM_REG(ymm4)
PUSH_YMM_REG(ymm5)
PUSH_YMM_REG(ymm6)
PUSH_YMM_REG(ymm7)
#elif defined __SSE__
PUSH_XMM_REG(xmm0)
PUSH_XMM_REG(xmm1)
PUSH_XMM_REG(xmm2)
PUSH_XMM_REG(xmm3)
PUSH_XMM_REG(xmm4)
PUSH_XMM_REG(xmm5)
PUSH_XMM_REG(xmm6)
PUSH_XMM_REG(xmm7)
#endif
// MMX registers are not used to pass arguments so we do not save them
// Stack is just 8-byte aligned but callee will re-align to 16
call _libXcomposite_so_tramp_resolve
#ifdef __AVX512F__
POP_ZMM_REG(zmm7)
POP_ZMM_REG(zmm6)
POP_ZMM_REG(zmm5)
POP_ZMM_REG(zmm4)
POP_ZMM_REG(zmm3)
POP_ZMM_REG(zmm2)
POP_ZMM_REG(zmm1)
POP_ZMM_REG(zmm0) // 16
#elif defined __AVX__
POP_YMM_REG(ymm7)
POP_YMM_REG(ymm6)
POP_YMM_REG(ymm5)
POP_YMM_REG(ymm4)
POP_YMM_REG(ymm3)
POP_YMM_REG(ymm2)
POP_YMM_REG(ymm1)
POP_YMM_REG(ymm0) // 16
#elif defined __SSE__
POP_XMM_REG(xmm7)
POP_XMM_REG(xmm6)
POP_XMM_REG(xmm5)
POP_XMM_REG(xmm4)
POP_XMM_REG(xmm3)
POP_XMM_REG(xmm2)
POP_XMM_REG(xmm1)
POP_XMM_REG(xmm0) // 16
#endif
POP_REG(r15)
POP_REG(r14) // 16
POP_REG(r13)
POP_REG(r12) // 16
POP_REG(r11)
POP_REG(r10) // 16
POP_REG(r9)
POP_REG(r8) // 16
POP_REG(rsi)
POP_REG(rbp) // 16
POP_REG(rdx)
POP_REG(rcx) // 16
POP_REG(rbx)
POP_REG(rbx) // 16
POP_REG(rdi)
ret
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeCreateRegionFromBorderClip
.p2align 4
.type XCompositeCreateRegionFromBorderClip, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeCreateRegionFromBorderClip
#endif
XCompositeCreateRegionFromBorderClip:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+0(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+0(%rip)
2:
pushq $0
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeFindDisplay
.p2align 4
.type XCompositeFindDisplay, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeFindDisplay
#endif
XCompositeFindDisplay:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+8(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+8(%rip)
2:
pushq $1
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeGetOverlayWindow
.p2align 4
.type XCompositeGetOverlayWindow, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeGetOverlayWindow
#endif
XCompositeGetOverlayWindow:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+16(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+16(%rip)
2:
pushq $2
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeNameWindowPixmap
.p2align 4
.type XCompositeNameWindowPixmap, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeNameWindowPixmap
#endif
XCompositeNameWindowPixmap:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+24(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+24(%rip)
2:
pushq $3
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeQueryExtension
.p2align 4
.type XCompositeQueryExtension, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeQueryExtension
#endif
XCompositeQueryExtension:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+32(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+32(%rip)
2:
pushq $4
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeQueryVersion
.p2align 4
.type XCompositeQueryVersion, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeQueryVersion
#endif
XCompositeQueryVersion:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+40(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+40(%rip)
2:
pushq $5
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeRedirectSubwindows
.p2align 4
.type XCompositeRedirectSubwindows, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeRedirectSubwindows
#endif
XCompositeRedirectSubwindows:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+48(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+48(%rip)
2:
pushq $6
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeRedirectWindow
.p2align 4
.type XCompositeRedirectWindow, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeRedirectWindow
#endif
XCompositeRedirectWindow:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+56(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+56(%rip)
2:
pushq $7
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeReleaseOverlayWindow
.p2align 4
.type XCompositeReleaseOverlayWindow, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeReleaseOverlayWindow
#endif
XCompositeReleaseOverlayWindow:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+64(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+64(%rip)
2:
pushq $8
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeUnredirectSubwindows
.p2align 4
.type XCompositeUnredirectSubwindows, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeUnredirectSubwindows
#endif
XCompositeUnredirectSubwindows:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+72(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+72(%rip)
2:
pushq $9
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeUnredirectWindow
.p2align 4
.type XCompositeUnredirectWindow, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeUnredirectWindow
#endif
XCompositeUnredirectWindow:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+80(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+80(%rip)
2:
pushq $10
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XCompositeVersion
.p2align 4
.type XCompositeVersion, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XCompositeVersion
#endif
XCompositeVersion:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXcomposite_so_tramp_table+88(%rip)
je 2f
1:
jmp *_libXcomposite_so_tramp_table+88(%rip)
2:
pushq $11
.cfi_adjust_cfa_offset 8
call _libXcomposite_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
@@ -0,0 +1,251 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXdamage.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXdamage.so.1");
CHECK(lib_handle, "failed to load library 'libXdamage.so.1' via callback ''");
#else
lib_handle = dlopen("libXdamage.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXdamage.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"XDamageAdd",
"XDamageCreate",
"XDamageDestroy",
"XDamageFindDisplay",
"XDamageQueryExtension",
"XDamageQueryVersion",
"XDamageSubtract",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXdamage_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXdamage_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXdamage_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXdamage_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXdamage_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXdamage_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXdamage_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXdamage_so_tramp_table, 0, SYM_COUNT * sizeof(_libXdamage_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,396 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.section .note.GNU-stack,"",@progbits
.data
.globl _libXdamage_so_tramp_table
.hidden _libXdamage_so_tramp_table
.align 8
_libXdamage_so_tramp_table:
.zero 64
.text
.globl _libXdamage_so_tramp_resolve
.hidden _libXdamage_so_tramp_resolve
.globl _libXdamage_so_save_regs_and_resolve
.hidden _libXdamage_so_save_regs_and_resolve
.type _libXdamage_so_save_regs_and_resolve, %function
_libXdamage_so_save_regs_and_resolve:
.cfi_startproc
#define PUSH_REG(reg) pushq %reg ; .cfi_adjust_cfa_offset 8; .cfi_rel_offset reg, 0
#define POP_REG(reg) popq %reg ; .cfi_adjust_cfa_offset -8; .cfi_restore reg
#define DEC_STACK(d) subq $d, %rsp; .cfi_adjust_cfa_offset d
#define INC_STACK(d) addq $d, %rsp; .cfi_adjust_cfa_offset -d
#define PUSH_MMX_REG(reg) DEC_STACK(8); movq %reg, (%rsp); .cfi_rel_offset reg, 0
#define POP_MMX_REG(reg) movq (%rsp), %reg; .cfi_restore reg; INC_STACK(8)
#define PUSH_XMM_REG(reg) DEC_STACK(16); movdqa %reg, (%rsp); .cfi_rel_offset reg, 0
#define POP_XMM_REG(reg) movdqa (%rsp), %reg; .cfi_restore reg; INC_STACK(16)
// TODO: cfi_offset/cfi_restore
#define PUSH_YMM_REG(reg) DEC_STACK(32); vmovdqu %reg, (%rsp)
#define POP_YMM_REG(reg) vmovdqu (%rsp), %reg; INC_STACK(32)
// TODO: cfi_offset/cfi_restore
#define PUSH_ZMM_REG(reg) DEC_STACK(64); vmovdqu32 %reg, (%rsp)
#define POP_ZMM_REG(reg) vmovdqu32 (%rsp), %reg; INC_STACK(64)
// Slow path which calls dlsym, taken only on first call.
// All registers are stored to handle arbitrary calling conventions
// (except x87 FPU registers which do not have to be preserved).
// For Dwarf directives, read https://www.imperialviolet.org/2017/01/18/cfi.html.
.cfi_def_cfa_offset 8 // Return address
PUSH_REG(rdi) // 16
mov 0x10(%rsp), %rdi
PUSH_REG(rbx)
PUSH_REG(rbx) // 16
PUSH_REG(rcx)
PUSH_REG(rdx) // 16
PUSH_REG(rbp)
PUSH_REG(rsi) // 16
PUSH_REG(r8)
PUSH_REG(r9) // 16
PUSH_REG(r10)
PUSH_REG(r11) // 16
PUSH_REG(r12)
PUSH_REG(r13) // 16
PUSH_REG(r14)
PUSH_REG(r15) // 16
// Maybe use cpuid instead of macro to detect current vector size...
#ifdef __AVX512F__
PUSH_ZMM_REG(zmm0)
PUSH_ZMM_REG(zmm1)
PUSH_ZMM_REG(zmm2)
PUSH_ZMM_REG(zmm3)
PUSH_ZMM_REG(zmm4)
PUSH_ZMM_REG(zmm5)
PUSH_ZMM_REG(zmm6)
PUSH_ZMM_REG(zmm7)
#elif defined __AVX__
PUSH_YMM_REG(ymm0)
PUSH_YMM_REG(ymm1)
PUSH_YMM_REG(ymm2)
PUSH_YMM_REG(ymm3)
PUSH_YMM_REG(ymm4)
PUSH_YMM_REG(ymm5)
PUSH_YMM_REG(ymm6)
PUSH_YMM_REG(ymm7)
#elif defined __SSE__
PUSH_XMM_REG(xmm0)
PUSH_XMM_REG(xmm1)
PUSH_XMM_REG(xmm2)
PUSH_XMM_REG(xmm3)
PUSH_XMM_REG(xmm4)
PUSH_XMM_REG(xmm5)
PUSH_XMM_REG(xmm6)
PUSH_XMM_REG(xmm7)
#endif
// MMX registers are not used to pass arguments so we do not save them
// Stack is just 8-byte aligned but callee will re-align to 16
call _libXdamage_so_tramp_resolve
#ifdef __AVX512F__
POP_ZMM_REG(zmm7)
POP_ZMM_REG(zmm6)
POP_ZMM_REG(zmm5)
POP_ZMM_REG(zmm4)
POP_ZMM_REG(zmm3)
POP_ZMM_REG(zmm2)
POP_ZMM_REG(zmm1)
POP_ZMM_REG(zmm0) // 16
#elif defined __AVX__
POP_YMM_REG(ymm7)
POP_YMM_REG(ymm6)
POP_YMM_REG(ymm5)
POP_YMM_REG(ymm4)
POP_YMM_REG(ymm3)
POP_YMM_REG(ymm2)
POP_YMM_REG(ymm1)
POP_YMM_REG(ymm0) // 16
#elif defined __SSE__
POP_XMM_REG(xmm7)
POP_XMM_REG(xmm6)
POP_XMM_REG(xmm5)
POP_XMM_REG(xmm4)
POP_XMM_REG(xmm3)
POP_XMM_REG(xmm2)
POP_XMM_REG(xmm1)
POP_XMM_REG(xmm0) // 16
#endif
POP_REG(r15)
POP_REG(r14) // 16
POP_REG(r13)
POP_REG(r12) // 16
POP_REG(r11)
POP_REG(r10) // 16
POP_REG(r9)
POP_REG(r8) // 16
POP_REG(rsi)
POP_REG(rbp) // 16
POP_REG(rdx)
POP_REG(rcx) // 16
POP_REG(rbx)
POP_REG(rbx) // 16
POP_REG(rdi)
ret
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageAdd
.p2align 4
.type XDamageAdd, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageAdd
#endif
XDamageAdd:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXdamage_so_tramp_table+0(%rip)
je 2f
1:
jmp *_libXdamage_so_tramp_table+0(%rip)
2:
pushq $0
.cfi_adjust_cfa_offset 8
call _libXdamage_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageCreate
.p2align 4
.type XDamageCreate, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageCreate
#endif
XDamageCreate:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXdamage_so_tramp_table+8(%rip)
je 2f
1:
jmp *_libXdamage_so_tramp_table+8(%rip)
2:
pushq $1
.cfi_adjust_cfa_offset 8
call _libXdamage_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageDestroy
.p2align 4
.type XDamageDestroy, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageDestroy
#endif
XDamageDestroy:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXdamage_so_tramp_table+16(%rip)
je 2f
1:
jmp *_libXdamage_so_tramp_table+16(%rip)
2:
pushq $2
.cfi_adjust_cfa_offset 8
call _libXdamage_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageFindDisplay
.p2align 4
.type XDamageFindDisplay, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageFindDisplay
#endif
XDamageFindDisplay:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXdamage_so_tramp_table+24(%rip)
je 2f
1:
jmp *_libXdamage_so_tramp_table+24(%rip)
2:
pushq $3
.cfi_adjust_cfa_offset 8
call _libXdamage_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageQueryExtension
.p2align 4
.type XDamageQueryExtension, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageQueryExtension
#endif
XDamageQueryExtension:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXdamage_so_tramp_table+32(%rip)
je 2f
1:
jmp *_libXdamage_so_tramp_table+32(%rip)
2:
pushq $4
.cfi_adjust_cfa_offset 8
call _libXdamage_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageQueryVersion
.p2align 4
.type XDamageQueryVersion, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageQueryVersion
#endif
XDamageQueryVersion:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXdamage_so_tramp_table+40(%rip)
je 2f
1:
jmp *_libXdamage_so_tramp_table+40(%rip)
2:
pushq $5
.cfi_adjust_cfa_offset 8
call _libXdamage_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl XDamageSubtract
.p2align 4
.type XDamageSubtract, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden XDamageSubtract
#endif
XDamageSubtract:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libXdamage_so_tramp_table+48(%rip)
je 2f
1:
jmp *_libXdamage_so_tramp_table+48(%rip)
2:
pushq $6
.cfi_adjust_cfa_offset 8
call _libXdamage_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
@@ -0,0 +1,376 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXext.so.6: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXext.so.6");
CHECK(lib_handle, "failed to load library 'libXext.so.6' via callback ''");
#else
lib_handle = dlopen("libXext.so.6", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXext.so.6' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"DPMSCapable",
"DPMSDisable",
"DPMSEnable",
"DPMSForceLevel",
"DPMSGetTimeouts",
"DPMSGetVersion",
"DPMSInfo",
"DPMSQueryExtension",
"DPMSSetTimeouts",
"XGEQueryExtension",
"XGEQueryVersion",
"XLbxGetEventBase",
"XLbxQueryExtension",
"XLbxQueryVersion",
"XMITMiscGetBugMode",
"XMITMiscQueryExtension",
"XMITMiscSetBugMode",
"XMissingExtension",
"XSecurityAllocXauth",
"XSecurityFreeXauth",
"XSecurityGenerateAuthorization",
"XSecurityQueryExtension",
"XSecurityRevokeAuthorization",
"XSetExtensionErrorHandler",
"XShapeCombineMask",
"XShapeCombineRectangles",
"XShapeCombineRegion",
"XShapeCombineShape",
"XShapeGetRectangles",
"XShapeInputSelected",
"XShapeOffsetShape",
"XShapeQueryExtension",
"XShapeQueryExtents",
"XShapeQueryVersion",
"XShapeSelectInput",
"XShmAttach",
"XShmCreateImage",
"XShmCreatePixmap",
"XShmDetach",
"XShmGetEventBase",
"XShmGetImage",
"XShmPixmapFormat",
"XShmPutImage",
"XShmQueryExtension",
"XShmQueryVersion",
"XSyncAwait",
"XSyncAwaitFence",
"XSyncChangeAlarm",
"XSyncChangeCounter",
"XSyncCreateAlarm",
"XSyncCreateCounter",
"XSyncCreateFence",
"XSyncDestroyAlarm",
"XSyncDestroyCounter",
"XSyncDestroyFence",
"XSyncFreeSystemCounterList",
"XSyncGetPriority",
"XSyncInitialize",
"XSyncIntToValue",
"XSyncIntsToValue",
"XSyncListSystemCounters",
"XSyncMaxValue",
"XSyncMinValue",
"XSyncQueryAlarm",
"XSyncQueryCounter",
"XSyncQueryExtension",
"XSyncQueryFence",
"XSyncResetFence",
"XSyncSetCounter",
"XSyncSetPriority",
"XSyncTriggerFence",
"XSyncValueAdd",
"XSyncValueEqual",
"XSyncValueGreaterOrEqual",
"XSyncValueGreaterThan",
"XSyncValueHigh32",
"XSyncValueIsNegative",
"XSyncValueIsPositive",
"XSyncValueIsZero",
"XSyncValueLessOrEqual",
"XSyncValueLessThan",
"XSyncValueLow32",
"XSyncValueSubtract",
"XTestFakeInput",
"XTestFlush",
"XTestGetInput",
"XTestMovePointer",
"XTestPressButton",
"XTestPressKey",
"XTestQueryInputSize",
"XTestReset",
"XTestStopInput",
"XagCreateAssociation",
"XagCreateEmbeddedApplicationGroup",
"XagCreateNonembeddedApplicationGroup",
"XagDestroyApplicationGroup",
"XagDestroyAssociation",
"XagGetApplicationGroupAttributes",
"XagQueryApplicationGroup",
"XagQueryVersion",
"XcupGetReservedColormapEntries",
"XcupQueryVersion",
"XcupStoreColors",
"XdbeAllocateBackBufferName",
"XdbeBeginIdiom",
"XdbeDeallocateBackBufferName",
"XdbeEndIdiom",
"XdbeFreeVisualInfo",
"XdbeGetBackBufferAttributes",
"XdbeGetVisualInfo",
"XdbeQueryExtension",
"XdbeSwapBuffers",
"XeviGetVisualInfo",
"XeviQueryExtension",
"XeviQueryVersion",
"XextAddDisplay",
"XextCreateExtension",
"XextDestroyExtension",
"XextFindDisplay",
"XextRemoveDisplay",
"XmbufChangeBufferAttributes",
"XmbufChangeWindowAttributes",
"XmbufClearBufferArea",
"XmbufCreateBuffers",
"XmbufCreateStereoWindow",
"XmbufDestroyBuffers",
"XmbufDisplayBuffers",
"XmbufGetBufferAttributes",
"XmbufGetScreenInfo",
"XmbufGetVersion",
"XmbufGetWindowAttributes",
"XmbufQueryExtension",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXext_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXext_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXext_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXext_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXext_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXext_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXext_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXext_so_tramp_table, 0, SYM_COUNT * sizeof(_libXext_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,282 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXfixes.so.3: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXfixes.so.3");
CHECK(lib_handle, "failed to load library 'libXfixes.so.3' via callback ''");
#else
lib_handle = dlopen("libXfixes.so.3", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXfixes.so.3' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"XFixesChangeCursor",
"XFixesChangeCursorByName",
"XFixesChangeSaveSet",
"XFixesCopyRegion",
"XFixesCreatePointerBarrier",
"XFixesCreateRegion",
"XFixesCreateRegionFromBitmap",
"XFixesCreateRegionFromGC",
"XFixesCreateRegionFromPicture",
"XFixesCreateRegionFromWindow",
"XFixesDestroyPointerBarrier",
"XFixesDestroyRegion",
"XFixesExpandRegion",
"XFixesFetchRegion",
"XFixesFetchRegionAndBounds",
"XFixesFindDisplay",
"XFixesGetClientDisconnectMode",
"XFixesGetCursorImage",
"XFixesGetCursorName",
"XFixesHideCursor",
"XFixesIntersectRegion",
"XFixesInvertRegion",
"XFixesQueryExtension",
"XFixesQueryVersion",
"XFixesRegionExtents",
"XFixesSelectCursorInput",
"XFixesSelectSelectionInput",
"XFixesSetClientDisconnectMode",
"XFixesSetCursorName",
"XFixesSetGCClipRegion",
"XFixesSetPictureClipRegion",
"XFixesSetRegion",
"XFixesSetWindowShapeRegion",
"XFixesShowCursor",
"XFixesSubtractRegion",
"XFixesTranslateRegion",
"XFixesUnionRegion",
"XFixesVersion",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXfixes_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXfixes_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXfixes_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXfixes_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXfixes_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXfixes_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXfixes_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXfixes_so_tramp_table, 0, SYM_COUNT * sizeof(_libXfixes_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,314 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libXrandr.so.2: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libXrandr.so.2");
CHECK(lib_handle, "failed to load library 'libXrandr.so.2' via callback ''");
#else
lib_handle = dlopen("libXrandr.so.2", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libXrandr.so.2' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"XRRAddOutputMode",
"XRRAllocGamma",
"XRRAllocModeInfo",
"XRRAllocateMonitor",
"XRRChangeOutputProperty",
"XRRChangeProviderProperty",
"XRRConfigCurrentConfiguration",
"XRRConfigCurrentRate",
"XRRConfigRates",
"XRRConfigRotations",
"XRRConfigSizes",
"XRRConfigTimes",
"XRRConfigureOutputProperty",
"XRRConfigureProviderProperty",
"XRRCreateMode",
"XRRDeleteMonitor",
"XRRDeleteOutputMode",
"XRRDeleteOutputProperty",
"XRRDeleteProviderProperty",
"XRRDestroyMode",
"XRRFreeCrtcInfo",
"XRRFreeGamma",
"XRRFreeModeInfo",
"XRRFreeMonitors",
"XRRFreeOutputInfo",
"XRRFreePanning",
"XRRFreeProviderInfo",
"XRRFreeProviderResources",
"XRRFreeScreenConfigInfo",
"XRRFreeScreenResources",
"XRRGetCrtcGamma",
"XRRGetCrtcGammaSize",
"XRRGetCrtcInfo",
"XRRGetCrtcTransform",
"XRRGetMonitors",
"XRRGetOutputInfo",
"XRRGetOutputPrimary",
"XRRGetOutputProperty",
"XRRGetPanning",
"XRRGetProviderInfo",
"XRRGetProviderProperty",
"XRRGetProviderResources",
"XRRGetScreenInfo",
"XRRGetScreenResources",
"XRRGetScreenResourcesCurrent",
"XRRGetScreenSizeRange",
"XRRListOutputProperties",
"XRRListProviderProperties",
"XRRQueryExtension",
"XRRQueryOutputProperty",
"XRRQueryProviderProperty",
"XRRQueryVersion",
"XRRRates",
"XRRRootToScreen",
"XRRRotations",
"XRRSelectInput",
"XRRSetCrtcConfig",
"XRRSetCrtcGamma",
"XRRSetCrtcTransform",
"XRRSetMonitor",
"XRRSetOutputPrimary",
"XRRSetPanning",
"XRRSetProviderOffloadSink",
"XRRSetProviderOutputSource",
"XRRSetScreenConfig",
"XRRSetScreenConfigAndRate",
"XRRSetScreenSize",
"XRRSizes",
"XRRTimes",
"XRRUpdateConfiguration",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libXrandr_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libXrandr_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libXrandr_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libXrandr_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libXrandr_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libXrandr_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libXrandr_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libXrandr_so_tramp_table, 0, SYM_COUNT * sizeof(_libXrandr_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,456 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libdrm.so.2: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libdrm.so.2");
CHECK(lib_handle, "failed to load library 'libdrm.so.2' via callback ''");
#else
lib_handle = dlopen("libdrm.so.2", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libdrm.so.2' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"drmAddBufs",
"drmAddContextPrivateMapping",
"drmAddContextTag",
"drmAddMap",
"drmAgpAcquire",
"drmAgpAlloc",
"drmAgpBase",
"drmAgpBind",
"drmAgpDeviceId",
"drmAgpEnable",
"drmAgpFree",
"drmAgpGetMode",
"drmAgpMemoryAvail",
"drmAgpMemoryUsed",
"drmAgpRelease",
"drmAgpSize",
"drmAgpUnbind",
"drmAgpVendorId",
"drmAgpVersionMajor",
"drmAgpVersionMinor",
"drmAuthMagic",
"drmAvailable",
"drmCheckModesettingSupported",
"drmClose",
"drmCloseBufferHandle",
"drmCloseOnce",
"drmCommandNone",
"drmCommandRead",
"drmCommandWrite",
"drmCommandWriteRead",
"drmCreateContext",
"drmCreateDrawable",
"drmCrtcGetSequence",
"drmCrtcQueueSequence",
"drmCtlInstHandler",
"drmCtlUninstHandler",
"drmDMA",
"drmDelContextTag",
"drmDestroyContext",
"drmDestroyDrawable",
"drmDevicesEqual",
"drmDropMaster",
"drmError",
"drmFinish",
"drmFree",
"drmFreeBufs",
"drmFreeBusid",
"drmFreeDevice",
"drmFreeDevices",
"drmFreeReservedContextList",
"drmFreeVersion",
"drmGetBufInfo",
"drmGetBusid",
"drmGetCap",
"drmGetClient",
"drmGetContextFlags",
"drmGetContextPrivateMapping",
"drmGetContextTag",
"drmGetDevice",
"drmGetDevice2",
"drmGetDeviceFromDevId",
"drmGetDeviceNameFromFd",
"drmGetDeviceNameFromFd2",
"drmGetDevices",
"drmGetDevices2",
"drmGetEntry",
"drmGetFormatModifierName",
"drmGetFormatModifierVendor",
"drmGetFormatName",
"drmGetHashTable",
"drmGetInterruptFromBusID",
"drmGetLibVersion",
"drmGetLock",
"drmGetMagic",
"drmGetMap",
"drmGetNodeTypeFromDevId",
"drmGetNodeTypeFromFd",
"drmGetPrimaryDeviceNameFromFd",
"drmGetRenderDeviceNameFromFd",
"drmGetReservedContextList",
"drmGetStats",
"drmGetVersion",
"drmHandleEvent",
"drmHashCreate",
"drmHashDelete",
"drmHashDestroy",
"drmHashFirst",
"drmHashInsert",
"drmHashLookup",
"drmHashNext",
"drmIoctl",
"drmIsKMS",
"drmIsMaster",
"drmMalloc",
"drmMap",
"drmMapBufs",
"drmMarkBufs",
"drmModeAddFB",
"drmModeAddFB2",
"drmModeAddFB2WithModifiers",
"drmModeAtomicAddProperty",
"drmModeAtomicAlloc",
"drmModeAtomicCommit",
"drmModeAtomicDuplicate",
"drmModeAtomicFree",
"drmModeAtomicGetCursor",
"drmModeAtomicMerge",
"drmModeAtomicSetCursor",
"drmModeAttachMode",
"drmModeCloseFB",
"drmModeConnectorGetPossibleCrtcs",
"drmModeConnectorSetProperty",
"drmModeCreateDumbBuffer",
"drmModeCreateLease",
"drmModeCreatePropertyBlob",
"drmModeCrtcGetGamma",
"drmModeCrtcSetGamma",
"drmModeDestroyDumbBuffer",
"drmModeDestroyPropertyBlob",
"drmModeDetachMode",
"drmModeDirtyFB",
"drmModeFormatModifierBlobIterNext",
"drmModeFreeConnector",
"drmModeFreeCrtc",
"drmModeFreeEncoder",
"drmModeFreeFB",
"drmModeFreeFB2",
"drmModeFreeModeInfo",
"drmModeFreeObjectProperties",
"drmModeFreePlane",
"drmModeFreePlaneResources",
"drmModeFreeProperty",
"drmModeFreePropertyBlob",
"drmModeFreeResources",
"drmModeGetConnector",
"drmModeGetConnectorCurrent",
"drmModeGetConnectorTypeName",
"drmModeGetCrtc",
"drmModeGetEncoder",
"drmModeGetFB",
"drmModeGetFB2",
"drmModeGetLease",
"drmModeGetPlane",
"drmModeGetPlaneResources",
"drmModeGetProperty",
"drmModeGetPropertyBlob",
"drmModeGetResources",
"drmModeListLessees",
"drmModeMapDumbBuffer",
"drmModeMoveCursor",
"drmModeObjectGetProperties",
"drmModeObjectSetProperty",
"drmModePageFlip",
"drmModePageFlipTarget",
"drmModeRevokeLease",
"drmModeRmFB",
"drmModeSetCrtc",
"drmModeSetCursor",
"drmModeSetCursor2",
"drmModeSetPlane",
"drmMsg",
"drmOpen",
"drmOpenControl",
"drmOpenOnce",
"drmOpenOnceWithType",
"drmOpenRender",
"drmOpenWithType",
"drmPrimeFDToHandle",
"drmPrimeHandleToFD",
"drmRandom",
"drmRandomCreate",
"drmRandomDestroy",
"drmRandomDouble",
"drmRmMap",
"drmSLCreate",
"drmSLDelete",
"drmSLDestroy",
"drmSLDump",
"drmSLFirst",
"drmSLInsert",
"drmSLLookup",
"drmSLLookupNeighbors",
"drmSLNext",
"drmScatterGatherAlloc",
"drmScatterGatherFree",
"drmSetBusid",
"drmSetClientCap",
"drmSetContextFlags",
"drmSetInterfaceVersion",
"drmSetMaster",
"drmSetServerInfo",
"drmSwitchToContext",
"drmSyncobjCreate",
"drmSyncobjDestroy",
"drmSyncobjEventfd",
"drmSyncobjExportSyncFile",
"drmSyncobjFDToHandle",
"drmSyncobjHandleToFD",
"drmSyncobjImportSyncFile",
"drmSyncobjQuery",
"drmSyncobjQuery2",
"drmSyncobjReset",
"drmSyncobjSignal",
"drmSyncobjTimelineSignal",
"drmSyncobjTimelineWait",
"drmSyncobjTransfer",
"drmSyncobjWait",
"drmUnlock",
"drmUnmap",
"drmUnmapBufs",
"drmUpdateDrawableInfo",
"drmWaitVBlank",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libdrm_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libdrm_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libdrm_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libdrm_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libdrm_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libdrm_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libdrm_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libdrm_so_tramp_table, 0, SYM_COUNT * sizeof(_libdrm_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,282 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libgbm.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libgbm.so.1");
CHECK(lib_handle, "failed to load library 'libgbm.so.1' via callback ''");
#else
lib_handle = dlopen("libgbm.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libgbm.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"gbm_bo_create",
"gbm_bo_create_with_modifiers",
"gbm_bo_create_with_modifiers2",
"gbm_bo_destroy",
"gbm_bo_get_bpp",
"gbm_bo_get_device",
"gbm_bo_get_fd",
"gbm_bo_get_fd_for_plane",
"gbm_bo_get_format",
"gbm_bo_get_handle",
"gbm_bo_get_handle_for_plane",
"gbm_bo_get_height",
"gbm_bo_get_modifier",
"gbm_bo_get_offset",
"gbm_bo_get_plane_count",
"gbm_bo_get_stride",
"gbm_bo_get_stride_for_plane",
"gbm_bo_get_user_data",
"gbm_bo_get_width",
"gbm_bo_import",
"gbm_bo_map",
"gbm_bo_set_user_data",
"gbm_bo_unmap",
"gbm_bo_write",
"gbm_create_device",
"gbm_device_destroy",
"gbm_device_get_backend_name",
"gbm_device_get_fd",
"gbm_device_get_format_modifier_plane_count",
"gbm_device_is_format_supported",
"gbm_format_get_name",
"gbm_surface_create",
"gbm_surface_create_with_modifiers",
"gbm_surface_create_with_modifiers2",
"gbm_surface_destroy",
"gbm_surface_has_free_buffers",
"gbm_surface_lock_front_buffer",
"gbm_surface_release_buffer",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libgbm_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libgbm_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libgbm_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libgbm_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libgbm_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libgbm_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libgbm_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libgbm_so_tramp_table, 0, SYM_COUNT * sizeof(_libgbm_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,53 @@
#!/bin/bash
# Copyright 2023 LiveKit, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ ! -e "$(pwd)/Implib.so" ]
then
git clone --depth 1 https://github.com/yugr/Implib.so.git
fi
generate_implib() {
category=$1
libname=$2
arch=$3
echo "Generating implib for category: ${category} libname: ${libname} - ${arch}, output to ${category}/${arch}/"
mkdir -p ${category}/${arch}/
python3 $(pwd)/Implib.so/implib-gen.py /lib/x86_64-linux-gnu/${libname}.so --target ${arch} --outdir ${category}/${arch}/
}
desktop_capturer_deps=("libdrm" "libgbm" "libXfixes" "libXdamage" "libXcomposite" "libXrandr" "libXext" "libX11")
for dep in "${desktop_capturer_deps[@]}"
do
generate_implib "desktop_capturer" ${dep} "x86_64-linux-gnu"
generate_implib "desktop_capturer" ${dep} "aarch64-linux-gnu"
done
nvidia_deps=("libcuda" "libnvcuvid")
for dep in "${nvidia_deps[@]}"
do
generate_implib "nvidia" ${dep} "x86_64-linux-gnu"
generate_implib "nvidia" ${dep} "aarch64-linux-gnu"
done
vaapi_deps=("libva" "libva-drm")
for dep in "${vaapi_deps[@]}"
do
generate_implib "vaapi" ${dep} "x86_64-linux-gnu"
generate_implib "vaapi" ${dep} "aarch64-linux-gnu"
done
@@ -0,0 +1,903 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libcuda.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libcuda.so.1");
CHECK(lib_handle, "failed to load library 'libcuda.so.1' via callback ''");
#else
lib_handle = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libcuda.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"cuArray3DCreate",
"cuArray3DCreate_v2",
"cuArray3DGetDescriptor",
"cuArray3DGetDescriptor_v2",
"cuArrayCreate",
"cuArrayCreate_v2",
"cuArrayDestroy",
"cuArrayGetDescriptor",
"cuArrayGetDescriptor_v2",
"cuArrayGetMemoryRequirements",
"cuArrayGetPlane",
"cuArrayGetSparseProperties",
"cuCheckpointProcessCheckpoint",
"cuCheckpointProcessGetRestoreThreadId",
"cuCheckpointProcessGetState",
"cuCheckpointProcessLock",
"cuCheckpointProcessRestore",
"cuCheckpointProcessUnlock",
"cuCoredumpGetAttribute",
"cuCoredumpGetAttributeGlobal",
"cuCoredumpSetAttribute",
"cuCoredumpSetAttributeGlobal",
"cuCtxAttach",
"cuCtxCreate",
"cuCtxCreate_v2",
"cuCtxCreate_v3",
"cuCtxCreate_v4",
"cuCtxDestroy",
"cuCtxDestroy_v2",
"cuCtxDetach",
"cuCtxDisablePeerAccess",
"cuCtxEnablePeerAccess",
"cuCtxFromGreenCtx",
"cuCtxGetApiVersion",
"cuCtxGetCacheConfig",
"cuCtxGetCurrent",
"cuCtxGetDevResource",
"cuCtxGetDevice",
"cuCtxGetExecAffinity",
"cuCtxGetFlags",
"cuCtxGetId",
"cuCtxGetLimit",
"cuCtxGetSharedMemConfig",
"cuCtxGetStreamPriorityRange",
"cuCtxPopCurrent",
"cuCtxPopCurrent_v2",
"cuCtxPushCurrent",
"cuCtxPushCurrent_v2",
"cuCtxRecordEvent",
"cuCtxResetPersistingL2Cache",
"cuCtxSetCacheConfig",
"cuCtxSetCurrent",
"cuCtxSetFlags",
"cuCtxSetLimit",
"cuCtxSetSharedMemConfig",
"cuCtxSynchronize",
"cuCtxWaitEvent",
"cuDestroyExternalMemory",
"cuDestroyExternalSemaphore",
"cuDevResourceGenerateDesc",
"cuDevSmResourceSplitByCount",
"cuDeviceCanAccessPeer",
"cuDeviceComputeCapability",
"cuDeviceGet",
"cuDeviceGetAttribute",
"cuDeviceGetByPCIBusId",
"cuDeviceGetCount",
"cuDeviceGetDefaultMemPool",
"cuDeviceGetDevResource",
"cuDeviceGetExecAffinitySupport",
"cuDeviceGetGraphMemAttribute",
"cuDeviceGetLuid",
"cuDeviceGetMemPool",
"cuDeviceGetName",
"cuDeviceGetNvSciSyncAttributes",
"cuDeviceGetP2PAttribute",
"cuDeviceGetPCIBusId",
"cuDeviceGetProperties",
"cuDeviceGetTexture1DLinearMaxWidth",
"cuDeviceGetUuid",
"cuDeviceGetUuid_v2",
"cuDeviceGraphMemTrim",
"cuDevicePrimaryCtxGetState",
"cuDevicePrimaryCtxRelease",
"cuDevicePrimaryCtxRelease_v2",
"cuDevicePrimaryCtxReset",
"cuDevicePrimaryCtxReset_v2",
"cuDevicePrimaryCtxRetain",
"cuDevicePrimaryCtxSetFlags",
"cuDevicePrimaryCtxSetFlags_v2",
"cuDeviceRegisterAsyncNotification",
"cuDeviceSetGraphMemAttribute",
"cuDeviceSetMemPool",
"cuDeviceTotalMem",
"cuDeviceTotalMem_v2",
"cuDeviceUnregisterAsyncNotification",
"cuDriverGetVersion",
"cuEGLApiInit",
"cuEGLStreamConsumerAcquireFrame",
"cuEGLStreamConsumerConnect",
"cuEGLStreamConsumerConnectWithFlags",
"cuEGLStreamConsumerDisconnect",
"cuEGLStreamConsumerReleaseFrame",
"cuEGLStreamProducerConnect",
"cuEGLStreamProducerDisconnect",
"cuEGLStreamProducerPresentFrame",
"cuEGLStreamProducerReturnFrame",
"cuEventCreate",
"cuEventDestroy",
"cuEventDestroy_v2",
"cuEventElapsedTime",
"cuEventElapsedTime_v2",
"cuEventQuery",
"cuEventRecord",
"cuEventRecordWithFlags",
"cuEventRecordWithFlags_ptsz",
"cuEventRecord_ptsz",
"cuEventSynchronize",
"cuExternalMemoryGetMappedBuffer",
"cuExternalMemoryGetMappedMipmappedArray",
"cuFlushGPUDirectRDMAWrites",
"cuFuncGetAttribute",
"cuFuncGetModule",
"cuFuncGetName",
"cuFuncGetParamInfo",
"cuFuncIsLoaded",
"cuFuncLoad",
"cuFuncSetAttribute",
"cuFuncSetBlockShape",
"cuFuncSetCacheConfig",
"cuFuncSetSharedMemConfig",
"cuFuncSetSharedSize",
"cuGLCtxCreate",
"cuGLCtxCreate_v2",
"cuGLGetDevices",
"cuGLGetDevices_v2",
"cuGLInit",
"cuGLMapBufferObject",
"cuGLMapBufferObjectAsync",
"cuGLMapBufferObjectAsync_v2",
"cuGLMapBufferObjectAsync_v2_ptsz",
"cuGLMapBufferObject_v2",
"cuGLMapBufferObject_v2_ptds",
"cuGLRegisterBufferObject",
"cuGLSetBufferObjectMapFlags",
"cuGLUnmapBufferObject",
"cuGLUnmapBufferObjectAsync",
"cuGLUnregisterBufferObject",
"cuGetErrorName",
"cuGetErrorString",
"cuGetExportTable",
"cuGetProcAddress",
"cuGetProcAddress_v2",
"cuGraphAddBatchMemOpNode",
"cuGraphAddChildGraphNode",
"cuGraphAddDependencies",
"cuGraphAddDependencies_v2",
"cuGraphAddEmptyNode",
"cuGraphAddEventRecordNode",
"cuGraphAddEventWaitNode",
"cuGraphAddExternalSemaphoresSignalNode",
"cuGraphAddExternalSemaphoresWaitNode",
"cuGraphAddHostNode",
"cuGraphAddKernelNode",
"cuGraphAddKernelNode_v2",
"cuGraphAddMemAllocNode",
"cuGraphAddMemFreeNode",
"cuGraphAddMemcpyNode",
"cuGraphAddMemsetNode",
"cuGraphAddNode",
"cuGraphAddNode_v2",
"cuGraphBatchMemOpNodeGetParams",
"cuGraphBatchMemOpNodeSetParams",
"cuGraphChildGraphNodeGetGraph",
"cuGraphClone",
"cuGraphConditionalHandleCreate",
"cuGraphCreate",
"cuGraphDebugDotPrint",
"cuGraphDestroy",
"cuGraphDestroyNode",
"cuGraphEventRecordNodeGetEvent",
"cuGraphEventRecordNodeSetEvent",
"cuGraphEventWaitNodeGetEvent",
"cuGraphEventWaitNodeSetEvent",
"cuGraphExecBatchMemOpNodeSetParams",
"cuGraphExecChildGraphNodeSetParams",
"cuGraphExecDestroy",
"cuGraphExecEventRecordNodeSetEvent",
"cuGraphExecEventWaitNodeSetEvent",
"cuGraphExecExternalSemaphoresSignalNodeSetParams",
"cuGraphExecExternalSemaphoresWaitNodeSetParams",
"cuGraphExecGetFlags",
"cuGraphExecHostNodeSetParams",
"cuGraphExecKernelNodeSetParams",
"cuGraphExecKernelNodeSetParams_v2",
"cuGraphExecMemcpyNodeSetParams",
"cuGraphExecMemsetNodeSetParams",
"cuGraphExecNodeSetParams",
"cuGraphExecUpdate",
"cuGraphExecUpdate_v2",
"cuGraphExternalSemaphoresSignalNodeGetParams",
"cuGraphExternalSemaphoresSignalNodeSetParams",
"cuGraphExternalSemaphoresWaitNodeGetParams",
"cuGraphExternalSemaphoresWaitNodeSetParams",
"cuGraphGetEdges",
"cuGraphGetEdges_v2",
"cuGraphGetNodes",
"cuGraphGetRootNodes",
"cuGraphHostNodeGetParams",
"cuGraphHostNodeSetParams",
"cuGraphInstantiate",
"cuGraphInstantiateWithFlags",
"cuGraphInstantiateWithParams",
"cuGraphInstantiateWithParams_ptsz",
"cuGraphInstantiate_v2",
"cuGraphKernelNodeCopyAttributes",
"cuGraphKernelNodeGetAttribute",
"cuGraphKernelNodeGetParams",
"cuGraphKernelNodeGetParams_v2",
"cuGraphKernelNodeSetAttribute",
"cuGraphKernelNodeSetParams",
"cuGraphKernelNodeSetParams_v2",
"cuGraphLaunch",
"cuGraphLaunch_ptsz",
"cuGraphMemAllocNodeGetParams",
"cuGraphMemFreeNodeGetParams",
"cuGraphMemcpyNodeGetParams",
"cuGraphMemcpyNodeSetParams",
"cuGraphMemsetNodeGetParams",
"cuGraphMemsetNodeSetParams",
"cuGraphNodeFindInClone",
"cuGraphNodeGetDependencies",
"cuGraphNodeGetDependencies_v2",
"cuGraphNodeGetDependentNodes",
"cuGraphNodeGetDependentNodes_v2",
"cuGraphNodeGetEnabled",
"cuGraphNodeGetType",
"cuGraphNodeSetEnabled",
"cuGraphNodeSetParams",
"cuGraphReleaseUserObject",
"cuGraphRemoveDependencies",
"cuGraphRemoveDependencies_v2",
"cuGraphRetainUserObject",
"cuGraphUpload",
"cuGraphUpload_ptsz",
"cuGraphicsEGLRegisterImage",
"cuGraphicsGLRegisterBuffer",
"cuGraphicsGLRegisterImage",
"cuGraphicsMapResources",
"cuGraphicsMapResources_ptsz",
"cuGraphicsResourceGetMappedEglFrame",
"cuGraphicsResourceGetMappedMipmappedArray",
"cuGraphicsResourceGetMappedPointer",
"cuGraphicsResourceGetMappedPointer_v2",
"cuGraphicsResourceSetMapFlags",
"cuGraphicsResourceSetMapFlags_v2",
"cuGraphicsSubResourceGetMappedArray",
"cuGraphicsUnmapResources",
"cuGraphicsUnmapResources_ptsz",
"cuGraphicsUnregisterResource",
"cuGraphicsVDPAURegisterOutputSurface",
"cuGraphicsVDPAURegisterVideoSurface",
"cuGreenCtxCreate",
"cuGreenCtxDestroy",
"cuGreenCtxGetDevResource",
"cuGreenCtxRecordEvent",
"cuGreenCtxStreamCreate",
"cuGreenCtxWaitEvent",
"cuImportExternalMemory",
"cuImportExternalSemaphore",
"cuInit",
"cuIpcCloseMemHandle",
"cuIpcGetEventHandle",
"cuIpcGetMemHandle",
"cuIpcOpenEventHandle",
"cuIpcOpenMemHandle",
"cuIpcOpenMemHandle_v2",
"cuKernelGetAttribute",
"cuKernelGetFunction",
"cuKernelGetLibrary",
"cuKernelGetName",
"cuKernelGetParamInfo",
"cuKernelSetAttribute",
"cuKernelSetCacheConfig",
"cuLaunch",
"cuLaunchCooperativeKernel",
"cuLaunchCooperativeKernelMultiDevice",
"cuLaunchCooperativeKernel_ptsz",
"cuLaunchGrid",
"cuLaunchGridAsync",
"cuLaunchHostFunc",
"cuLaunchHostFunc_ptsz",
"cuLaunchKernel",
"cuLaunchKernelEx",
"cuLaunchKernelEx_ptsz",
"cuLaunchKernel_ptsz",
"cuLibraryEnumerateKernels",
"cuLibraryGetGlobal",
"cuLibraryGetKernel",
"cuLibraryGetKernelCount",
"cuLibraryGetManaged",
"cuLibraryGetModule",
"cuLibraryGetUnifiedFunction",
"cuLibraryLoadData",
"cuLibraryLoadFromFile",
"cuLibraryUnload",
"cuLinkAddData",
"cuLinkAddData_v2",
"cuLinkAddFile",
"cuLinkAddFile_v2",
"cuLinkComplete",
"cuLinkCreate",
"cuLinkCreate_v2",
"cuLinkDestroy",
"cuMemAddressFree",
"cuMemAddressReserve",
"cuMemAdvise",
"cuMemAdvise_v2",
"cuMemAlloc",
"cuMemAllocAsync",
"cuMemAllocAsync_ptsz",
"cuMemAllocFromPoolAsync",
"cuMemAllocFromPoolAsync_ptsz",
"cuMemAllocHost",
"cuMemAllocHost_v2",
"cuMemAllocManaged",
"cuMemAllocPitch",
"cuMemAllocPitch_v2",
"cuMemAlloc_v2",
"cuMemBatchDecompressAsync",
"cuMemBatchDecompressAsync_ptsz",
"cuMemCreate",
"cuMemExportToShareableHandle",
"cuMemFree",
"cuMemFreeAsync",
"cuMemFreeAsync_ptsz",
"cuMemFreeHost",
"cuMemFree_v2",
"cuMemGetAccess",
"cuMemGetAddressRange",
"cuMemGetAddressRange_v2",
"cuMemGetAllocationGranularity",
"cuMemGetAllocationPropertiesFromHandle",
"cuMemGetAttribute",
"cuMemGetAttribute_v2",
"cuMemGetHandleForAddressRange",
"cuMemGetInfo",
"cuMemGetInfo_v2",
"cuMemHostAlloc",
"cuMemHostGetDevicePointer",
"cuMemHostGetDevicePointer_v2",
"cuMemHostGetFlags",
"cuMemHostRegister",
"cuMemHostRegister_v2",
"cuMemHostUnregister",
"cuMemImportFromShareableHandle",
"cuMemMap",
"cuMemMapArrayAsync",
"cuMemMapArrayAsync_ptsz",
"cuMemPoolCreate",
"cuMemPoolDestroy",
"cuMemPoolExportPointer",
"cuMemPoolExportToShareableHandle",
"cuMemPoolGetAccess",
"cuMemPoolGetAttribute",
"cuMemPoolImportFromShareableHandle",
"cuMemPoolImportPointer",
"cuMemPoolSetAccess",
"cuMemPoolSetAttribute",
"cuMemPoolTrimTo",
"cuMemPrefetchAsync",
"cuMemPrefetchAsync_ptsz",
"cuMemPrefetchAsync_v2",
"cuMemPrefetchAsync_v2_ptsz",
"cuMemRangeGetAttribute",
"cuMemRangeGetAttributes",
"cuMemRelease",
"cuMemRetainAllocationHandle",
"cuMemSetAccess",
"cuMemUnmap",
"cuMemcpy",
"cuMemcpy2D",
"cuMemcpy2DAsync",
"cuMemcpy2DAsync_v2",
"cuMemcpy2DAsync_v2_ptsz",
"cuMemcpy2DUnaligned",
"cuMemcpy2DUnaligned_v2",
"cuMemcpy2DUnaligned_v2_ptds",
"cuMemcpy2D_v2",
"cuMemcpy2D_v2_ptds",
"cuMemcpy3D",
"cuMemcpy3DAsync",
"cuMemcpy3DAsync_v2",
"cuMemcpy3DAsync_v2_ptsz",
"cuMemcpy3DBatchAsync",
"cuMemcpy3DBatchAsync_ptsz",
"cuMemcpy3DPeer",
"cuMemcpy3DPeerAsync",
"cuMemcpy3DPeerAsync_ptsz",
"cuMemcpy3DPeer_ptds",
"cuMemcpy3D_v2",
"cuMemcpy3D_v2_ptds",
"cuMemcpyAsync",
"cuMemcpyAsync_ptsz",
"cuMemcpyAtoA",
"cuMemcpyAtoA_v2",
"cuMemcpyAtoA_v2_ptds",
"cuMemcpyAtoD",
"cuMemcpyAtoD_v2",
"cuMemcpyAtoD_v2_ptds",
"cuMemcpyAtoH",
"cuMemcpyAtoHAsync",
"cuMemcpyAtoHAsync_v2",
"cuMemcpyAtoHAsync_v2_ptsz",
"cuMemcpyAtoH_v2",
"cuMemcpyAtoH_v2_ptds",
"cuMemcpyBatchAsync",
"cuMemcpyBatchAsync_ptsz",
"cuMemcpyDtoA",
"cuMemcpyDtoA_v2",
"cuMemcpyDtoA_v2_ptds",
"cuMemcpyDtoD",
"cuMemcpyDtoDAsync",
"cuMemcpyDtoDAsync_v2",
"cuMemcpyDtoDAsync_v2_ptsz",
"cuMemcpyDtoD_v2",
"cuMemcpyDtoD_v2_ptds",
"cuMemcpyDtoH",
"cuMemcpyDtoHAsync",
"cuMemcpyDtoHAsync_v2",
"cuMemcpyDtoHAsync_v2_ptsz",
"cuMemcpyDtoH_v2",
"cuMemcpyDtoH_v2_ptds",
"cuMemcpyHtoA",
"cuMemcpyHtoAAsync",
"cuMemcpyHtoAAsync_v2",
"cuMemcpyHtoAAsync_v2_ptsz",
"cuMemcpyHtoA_v2",
"cuMemcpyHtoA_v2_ptds",
"cuMemcpyHtoD",
"cuMemcpyHtoDAsync",
"cuMemcpyHtoDAsync_v2",
"cuMemcpyHtoDAsync_v2_ptsz",
"cuMemcpyHtoD_v2",
"cuMemcpyHtoD_v2_ptds",
"cuMemcpyPeer",
"cuMemcpyPeerAsync",
"cuMemcpyPeerAsync_ptsz",
"cuMemcpyPeer_ptds",
"cuMemcpy_ptds",
"cuMemsetD16",
"cuMemsetD16Async",
"cuMemsetD16Async_ptsz",
"cuMemsetD16_v2",
"cuMemsetD16_v2_ptds",
"cuMemsetD2D16",
"cuMemsetD2D16Async",
"cuMemsetD2D16Async_ptsz",
"cuMemsetD2D16_v2",
"cuMemsetD2D16_v2_ptds",
"cuMemsetD2D32",
"cuMemsetD2D32Async",
"cuMemsetD2D32Async_ptsz",
"cuMemsetD2D32_v2",
"cuMemsetD2D32_v2_ptds",
"cuMemsetD2D8",
"cuMemsetD2D8Async",
"cuMemsetD2D8Async_ptsz",
"cuMemsetD2D8_v2",
"cuMemsetD2D8_v2_ptds",
"cuMemsetD32",
"cuMemsetD32Async",
"cuMemsetD32Async_ptsz",
"cuMemsetD32_v2",
"cuMemsetD32_v2_ptds",
"cuMemsetD8",
"cuMemsetD8Async",
"cuMemsetD8Async_ptsz",
"cuMemsetD8_v2",
"cuMemsetD8_v2_ptds",
"cuMipmappedArrayCreate",
"cuMipmappedArrayDestroy",
"cuMipmappedArrayGetLevel",
"cuMipmappedArrayGetMemoryRequirements",
"cuMipmappedArrayGetSparseProperties",
"cuModuleEnumerateFunctions",
"cuModuleGetFunction",
"cuModuleGetFunctionCount",
"cuModuleGetGlobal",
"cuModuleGetGlobal_v2",
"cuModuleGetLoadingMode",
"cuModuleGetSurfRef",
"cuModuleGetTexRef",
"cuModuleLoad",
"cuModuleLoadData",
"cuModuleLoadDataEx",
"cuModuleLoadFatBinary",
"cuModuleUnload",
"cuMulticastAddDevice",
"cuMulticastBindAddr",
"cuMulticastBindMem",
"cuMulticastCreate",
"cuMulticastGetGranularity",
"cuMulticastUnbind",
"cuOccupancyAvailableDynamicSMemPerBlock",
"cuOccupancyMaxActiveBlocksPerMultiprocessor",
"cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags",
"cuOccupancyMaxActiveClusters",
"cuOccupancyMaxPotentialBlockSize",
"cuOccupancyMaxPotentialBlockSizeWithFlags",
"cuOccupancyMaxPotentialClusterSize",
"cuParamSetSize",
"cuParamSetTexRef",
"cuParamSetf",
"cuParamSeti",
"cuParamSetv",
"cuPointerGetAttribute",
"cuPointerGetAttributes",
"cuPointerSetAttribute",
"cuProfilerInitialize",
"cuProfilerStart",
"cuProfilerStop",
"cuSignalExternalSemaphoresAsync",
"cuSignalExternalSemaphoresAsync_ptsz",
"cuStreamAddCallback",
"cuStreamAddCallback_ptsz",
"cuStreamAttachMemAsync",
"cuStreamAttachMemAsync_ptsz",
"cuStreamBatchMemOp",
"cuStreamBatchMemOp_ptsz",
"cuStreamBatchMemOp_v2",
"cuStreamBatchMemOp_v2_ptsz",
"cuStreamBeginCapture",
"cuStreamBeginCaptureToGraph",
"cuStreamBeginCaptureToGraph_ptsz",
"cuStreamBeginCapture_ptsz",
"cuStreamBeginCapture_v2",
"cuStreamBeginCapture_v2_ptsz",
"cuStreamCopyAttributes",
"cuStreamCopyAttributes_ptsz",
"cuStreamCreate",
"cuStreamCreateWithPriority",
"cuStreamDestroy",
"cuStreamDestroy_v2",
"cuStreamEndCapture",
"cuStreamEndCapture_ptsz",
"cuStreamGetAttribute",
"cuStreamGetAttribute_ptsz",
"cuStreamGetCaptureInfo",
"cuStreamGetCaptureInfo_ptsz",
"cuStreamGetCaptureInfo_v2",
"cuStreamGetCaptureInfo_v2_ptsz",
"cuStreamGetCaptureInfo_v3",
"cuStreamGetCaptureInfo_v3_ptsz",
"cuStreamGetCtx",
"cuStreamGetCtx_ptsz",
"cuStreamGetCtx_v2",
"cuStreamGetCtx_v2_ptsz",
"cuStreamGetDevice",
"cuStreamGetDevice_ptsz",
"cuStreamGetFlags",
"cuStreamGetFlags_ptsz",
"cuStreamGetGreenCtx",
"cuStreamGetId",
"cuStreamGetId_ptsz",
"cuStreamGetPriority",
"cuStreamGetPriority_ptsz",
"cuStreamIsCapturing",
"cuStreamIsCapturing_ptsz",
"cuStreamQuery",
"cuStreamQuery_ptsz",
"cuStreamSetAttribute",
"cuStreamSetAttribute_ptsz",
"cuStreamSynchronize",
"cuStreamSynchronize_ptsz",
"cuStreamUpdateCaptureDependencies",
"cuStreamUpdateCaptureDependencies_ptsz",
"cuStreamUpdateCaptureDependencies_v2",
"cuStreamUpdateCaptureDependencies_v2_ptsz",
"cuStreamWaitEvent",
"cuStreamWaitEvent_ptsz",
"cuStreamWaitValue32",
"cuStreamWaitValue32_ptsz",
"cuStreamWaitValue32_v2",
"cuStreamWaitValue32_v2_ptsz",
"cuStreamWaitValue64",
"cuStreamWaitValue64_ptsz",
"cuStreamWaitValue64_v2",
"cuStreamWaitValue64_v2_ptsz",
"cuStreamWriteValue32",
"cuStreamWriteValue32_ptsz",
"cuStreamWriteValue32_v2",
"cuStreamWriteValue32_v2_ptsz",
"cuStreamWriteValue64",
"cuStreamWriteValue64_ptsz",
"cuStreamWriteValue64_v2",
"cuStreamWriteValue64_v2_ptsz",
"cuSurfObjectCreate",
"cuSurfObjectDestroy",
"cuSurfObjectGetResourceDesc",
"cuSurfRefGetArray",
"cuSurfRefSetArray",
"cuTensorMapEncodeIm2col",
"cuTensorMapEncodeIm2colWide",
"cuTensorMapEncodeTiled",
"cuTensorMapReplaceAddress",
"cuTexObjectCreate",
"cuTexObjectDestroy",
"cuTexObjectGetResourceDesc",
"cuTexObjectGetResourceViewDesc",
"cuTexObjectGetTextureDesc",
"cuTexRefCreate",
"cuTexRefDestroy",
"cuTexRefGetAddress",
"cuTexRefGetAddressMode",
"cuTexRefGetAddress_v2",
"cuTexRefGetArray",
"cuTexRefGetBorderColor",
"cuTexRefGetFilterMode",
"cuTexRefGetFlags",
"cuTexRefGetFormat",
"cuTexRefGetMaxAnisotropy",
"cuTexRefGetMipmapFilterMode",
"cuTexRefGetMipmapLevelBias",
"cuTexRefGetMipmapLevelClamp",
"cuTexRefGetMipmappedArray",
"cuTexRefSetAddress",
"cuTexRefSetAddress2D",
"cuTexRefSetAddress2D_v2",
"cuTexRefSetAddress2D_v3",
"cuTexRefSetAddressMode",
"cuTexRefSetAddress_v2",
"cuTexRefSetArray",
"cuTexRefSetBorderColor",
"cuTexRefSetFilterMode",
"cuTexRefSetFlags",
"cuTexRefSetFormat",
"cuTexRefSetMaxAnisotropy",
"cuTexRefSetMipmapFilterMode",
"cuTexRefSetMipmapLevelBias",
"cuTexRefSetMipmapLevelClamp",
"cuTexRefSetMipmappedArray",
"cuThreadExchangeStreamCaptureMode",
"cuUserObjectCreate",
"cuUserObjectRelease",
"cuUserObjectRetain",
"cuVDPAUCtxCreate",
"cuVDPAUCtxCreate_v2",
"cuVDPAUGetDevice",
"cuWaitExternalSemaphoresAsync",
"cuWaitExternalSemaphoresAsync_ptsz",
"cudbgApiAttach",
"cudbgApiDetach",
"cudbgApiInit",
"cudbgGetAPI",
"cudbgGetAPIVersion",
"cudbgMain",
"cudbgReportDriverApiError",
"cudbgReportDriverInternalError",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libcuda_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libcuda_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libcuda_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libcuda_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libcuda_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libcuda_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libcuda_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libcuda_so_tramp_table, 0, SYM_COUNT * sizeof(_libcuda_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,282 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libnvcuvid.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libnvcuvid.so.1");
CHECK(lib_handle, "failed to load library 'libnvcuvid.so.1' via callback ''");
#else
lib_handle = dlopen("libnvcuvid.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libnvcuvid.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"NvToolCreateInterface",
"NvToolDestroyInterface",
"NvToolGetApiFunctionCount",
"NvToolGetApiID",
"NvToolGetApiNames",
"NvToolGetInterface",
"NvToolSetApiID",
"NvToolSetInterface",
"__std_1U4S4U_X02",
"__std_2U4S4U_X08",
"__std_4U4S4U_X04",
"cuvidConvertYUVToRGB",
"cuvidConvertYUVToRGBArray",
"cuvidCreateDecoder",
"cuvidCreateVideoParser",
"cuvidCreateVideoSource",
"cuvidCreateVideoSourceW",
"cuvidCtxLock",
"cuvidCtxLockCreate",
"cuvidCtxLockDestroy",
"cuvidCtxUnlock",
"cuvidDecodePicture",
"cuvidDestroyDecoder",
"cuvidDestroyVideoParser",
"cuvidDestroyVideoSource",
"cuvidGetDecodeStatus",
"cuvidGetDecoderCaps",
"cuvidGetSourceAudioFormat",
"cuvidGetSourceVideoFormat",
"cuvidGetVideoSourceState",
"cuvidMapVideoFrame",
"cuvidMapVideoFrame64",
"cuvidParseVideoData",
"cuvidPrivateOp",
"cuvidReconfigureDecoder",
"cuvidSetVideoSourceState",
"cuvidUnmapVideoFrame",
"cuvidUnmapVideoFrame64",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libnvcuvid_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libnvcuvid_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libnvcuvid_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libnvcuvid_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libnvcuvid_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libnvcuvid_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libnvcuvid_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libnvcuvid_so_tramp_table, 0, SYM_COUNT * sizeof(_libnvcuvid_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,903 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libcuda.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libcuda.so.1");
CHECK(lib_handle, "failed to load library 'libcuda.so.1' via callback ''");
#else
lib_handle = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libcuda.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"cuArray3DCreate",
"cuArray3DCreate_v2",
"cuArray3DGetDescriptor",
"cuArray3DGetDescriptor_v2",
"cuArrayCreate",
"cuArrayCreate_v2",
"cuArrayDestroy",
"cuArrayGetDescriptor",
"cuArrayGetDescriptor_v2",
"cuArrayGetMemoryRequirements",
"cuArrayGetPlane",
"cuArrayGetSparseProperties",
"cuCheckpointProcessCheckpoint",
"cuCheckpointProcessGetRestoreThreadId",
"cuCheckpointProcessGetState",
"cuCheckpointProcessLock",
"cuCheckpointProcessRestore",
"cuCheckpointProcessUnlock",
"cuCoredumpGetAttribute",
"cuCoredumpGetAttributeGlobal",
"cuCoredumpSetAttribute",
"cuCoredumpSetAttributeGlobal",
"cuCtxAttach",
"cuCtxCreate",
"cuCtxCreate_v2",
"cuCtxCreate_v3",
"cuCtxCreate_v4",
"cuCtxDestroy",
"cuCtxDestroy_v2",
"cuCtxDetach",
"cuCtxDisablePeerAccess",
"cuCtxEnablePeerAccess",
"cuCtxFromGreenCtx",
"cuCtxGetApiVersion",
"cuCtxGetCacheConfig",
"cuCtxGetCurrent",
"cuCtxGetDevResource",
"cuCtxGetDevice",
"cuCtxGetExecAffinity",
"cuCtxGetFlags",
"cuCtxGetId",
"cuCtxGetLimit",
"cuCtxGetSharedMemConfig",
"cuCtxGetStreamPriorityRange",
"cuCtxPopCurrent",
"cuCtxPopCurrent_v2",
"cuCtxPushCurrent",
"cuCtxPushCurrent_v2",
"cuCtxRecordEvent",
"cuCtxResetPersistingL2Cache",
"cuCtxSetCacheConfig",
"cuCtxSetCurrent",
"cuCtxSetFlags",
"cuCtxSetLimit",
"cuCtxSetSharedMemConfig",
"cuCtxSynchronize",
"cuCtxWaitEvent",
"cuDestroyExternalMemory",
"cuDestroyExternalSemaphore",
"cuDevResourceGenerateDesc",
"cuDevSmResourceSplitByCount",
"cuDeviceCanAccessPeer",
"cuDeviceComputeCapability",
"cuDeviceGet",
"cuDeviceGetAttribute",
"cuDeviceGetByPCIBusId",
"cuDeviceGetCount",
"cuDeviceGetDefaultMemPool",
"cuDeviceGetDevResource",
"cuDeviceGetExecAffinitySupport",
"cuDeviceGetGraphMemAttribute",
"cuDeviceGetLuid",
"cuDeviceGetMemPool",
"cuDeviceGetName",
"cuDeviceGetNvSciSyncAttributes",
"cuDeviceGetP2PAttribute",
"cuDeviceGetPCIBusId",
"cuDeviceGetProperties",
"cuDeviceGetTexture1DLinearMaxWidth",
"cuDeviceGetUuid",
"cuDeviceGetUuid_v2",
"cuDeviceGraphMemTrim",
"cuDevicePrimaryCtxGetState",
"cuDevicePrimaryCtxRelease",
"cuDevicePrimaryCtxRelease_v2",
"cuDevicePrimaryCtxReset",
"cuDevicePrimaryCtxReset_v2",
"cuDevicePrimaryCtxRetain",
"cuDevicePrimaryCtxSetFlags",
"cuDevicePrimaryCtxSetFlags_v2",
"cuDeviceRegisterAsyncNotification",
"cuDeviceSetGraphMemAttribute",
"cuDeviceSetMemPool",
"cuDeviceTotalMem",
"cuDeviceTotalMem_v2",
"cuDeviceUnregisterAsyncNotification",
"cuDriverGetVersion",
"cuEGLApiInit",
"cuEGLStreamConsumerAcquireFrame",
"cuEGLStreamConsumerConnect",
"cuEGLStreamConsumerConnectWithFlags",
"cuEGLStreamConsumerDisconnect",
"cuEGLStreamConsumerReleaseFrame",
"cuEGLStreamProducerConnect",
"cuEGLStreamProducerDisconnect",
"cuEGLStreamProducerPresentFrame",
"cuEGLStreamProducerReturnFrame",
"cuEventCreate",
"cuEventDestroy",
"cuEventDestroy_v2",
"cuEventElapsedTime",
"cuEventElapsedTime_v2",
"cuEventQuery",
"cuEventRecord",
"cuEventRecordWithFlags",
"cuEventRecordWithFlags_ptsz",
"cuEventRecord_ptsz",
"cuEventSynchronize",
"cuExternalMemoryGetMappedBuffer",
"cuExternalMemoryGetMappedMipmappedArray",
"cuFlushGPUDirectRDMAWrites",
"cuFuncGetAttribute",
"cuFuncGetModule",
"cuFuncGetName",
"cuFuncGetParamInfo",
"cuFuncIsLoaded",
"cuFuncLoad",
"cuFuncSetAttribute",
"cuFuncSetBlockShape",
"cuFuncSetCacheConfig",
"cuFuncSetSharedMemConfig",
"cuFuncSetSharedSize",
"cuGLCtxCreate",
"cuGLCtxCreate_v2",
"cuGLGetDevices",
"cuGLGetDevices_v2",
"cuGLInit",
"cuGLMapBufferObject",
"cuGLMapBufferObjectAsync",
"cuGLMapBufferObjectAsync_v2",
"cuGLMapBufferObjectAsync_v2_ptsz",
"cuGLMapBufferObject_v2",
"cuGLMapBufferObject_v2_ptds",
"cuGLRegisterBufferObject",
"cuGLSetBufferObjectMapFlags",
"cuGLUnmapBufferObject",
"cuGLUnmapBufferObjectAsync",
"cuGLUnregisterBufferObject",
"cuGetErrorName",
"cuGetErrorString",
"cuGetExportTable",
"cuGetProcAddress",
"cuGetProcAddress_v2",
"cuGraphAddBatchMemOpNode",
"cuGraphAddChildGraphNode",
"cuGraphAddDependencies",
"cuGraphAddDependencies_v2",
"cuGraphAddEmptyNode",
"cuGraphAddEventRecordNode",
"cuGraphAddEventWaitNode",
"cuGraphAddExternalSemaphoresSignalNode",
"cuGraphAddExternalSemaphoresWaitNode",
"cuGraphAddHostNode",
"cuGraphAddKernelNode",
"cuGraphAddKernelNode_v2",
"cuGraphAddMemAllocNode",
"cuGraphAddMemFreeNode",
"cuGraphAddMemcpyNode",
"cuGraphAddMemsetNode",
"cuGraphAddNode",
"cuGraphAddNode_v2",
"cuGraphBatchMemOpNodeGetParams",
"cuGraphBatchMemOpNodeSetParams",
"cuGraphChildGraphNodeGetGraph",
"cuGraphClone",
"cuGraphConditionalHandleCreate",
"cuGraphCreate",
"cuGraphDebugDotPrint",
"cuGraphDestroy",
"cuGraphDestroyNode",
"cuGraphEventRecordNodeGetEvent",
"cuGraphEventRecordNodeSetEvent",
"cuGraphEventWaitNodeGetEvent",
"cuGraphEventWaitNodeSetEvent",
"cuGraphExecBatchMemOpNodeSetParams",
"cuGraphExecChildGraphNodeSetParams",
"cuGraphExecDestroy",
"cuGraphExecEventRecordNodeSetEvent",
"cuGraphExecEventWaitNodeSetEvent",
"cuGraphExecExternalSemaphoresSignalNodeSetParams",
"cuGraphExecExternalSemaphoresWaitNodeSetParams",
"cuGraphExecGetFlags",
"cuGraphExecHostNodeSetParams",
"cuGraphExecKernelNodeSetParams",
"cuGraphExecKernelNodeSetParams_v2",
"cuGraphExecMemcpyNodeSetParams",
"cuGraphExecMemsetNodeSetParams",
"cuGraphExecNodeSetParams",
"cuGraphExecUpdate",
"cuGraphExecUpdate_v2",
"cuGraphExternalSemaphoresSignalNodeGetParams",
"cuGraphExternalSemaphoresSignalNodeSetParams",
"cuGraphExternalSemaphoresWaitNodeGetParams",
"cuGraphExternalSemaphoresWaitNodeSetParams",
"cuGraphGetEdges",
"cuGraphGetEdges_v2",
"cuGraphGetNodes",
"cuGraphGetRootNodes",
"cuGraphHostNodeGetParams",
"cuGraphHostNodeSetParams",
"cuGraphInstantiate",
"cuGraphInstantiateWithFlags",
"cuGraphInstantiateWithParams",
"cuGraphInstantiateWithParams_ptsz",
"cuGraphInstantiate_v2",
"cuGraphKernelNodeCopyAttributes",
"cuGraphKernelNodeGetAttribute",
"cuGraphKernelNodeGetParams",
"cuGraphKernelNodeGetParams_v2",
"cuGraphKernelNodeSetAttribute",
"cuGraphKernelNodeSetParams",
"cuGraphKernelNodeSetParams_v2",
"cuGraphLaunch",
"cuGraphLaunch_ptsz",
"cuGraphMemAllocNodeGetParams",
"cuGraphMemFreeNodeGetParams",
"cuGraphMemcpyNodeGetParams",
"cuGraphMemcpyNodeSetParams",
"cuGraphMemsetNodeGetParams",
"cuGraphMemsetNodeSetParams",
"cuGraphNodeFindInClone",
"cuGraphNodeGetDependencies",
"cuGraphNodeGetDependencies_v2",
"cuGraphNodeGetDependentNodes",
"cuGraphNodeGetDependentNodes_v2",
"cuGraphNodeGetEnabled",
"cuGraphNodeGetType",
"cuGraphNodeSetEnabled",
"cuGraphNodeSetParams",
"cuGraphReleaseUserObject",
"cuGraphRemoveDependencies",
"cuGraphRemoveDependencies_v2",
"cuGraphRetainUserObject",
"cuGraphUpload",
"cuGraphUpload_ptsz",
"cuGraphicsEGLRegisterImage",
"cuGraphicsGLRegisterBuffer",
"cuGraphicsGLRegisterImage",
"cuGraphicsMapResources",
"cuGraphicsMapResources_ptsz",
"cuGraphicsResourceGetMappedEglFrame",
"cuGraphicsResourceGetMappedMipmappedArray",
"cuGraphicsResourceGetMappedPointer",
"cuGraphicsResourceGetMappedPointer_v2",
"cuGraphicsResourceSetMapFlags",
"cuGraphicsResourceSetMapFlags_v2",
"cuGraphicsSubResourceGetMappedArray",
"cuGraphicsUnmapResources",
"cuGraphicsUnmapResources_ptsz",
"cuGraphicsUnregisterResource",
"cuGraphicsVDPAURegisterOutputSurface",
"cuGraphicsVDPAURegisterVideoSurface",
"cuGreenCtxCreate",
"cuGreenCtxDestroy",
"cuGreenCtxGetDevResource",
"cuGreenCtxRecordEvent",
"cuGreenCtxStreamCreate",
"cuGreenCtxWaitEvent",
"cuImportExternalMemory",
"cuImportExternalSemaphore",
"cuInit",
"cuIpcCloseMemHandle",
"cuIpcGetEventHandle",
"cuIpcGetMemHandle",
"cuIpcOpenEventHandle",
"cuIpcOpenMemHandle",
"cuIpcOpenMemHandle_v2",
"cuKernelGetAttribute",
"cuKernelGetFunction",
"cuKernelGetLibrary",
"cuKernelGetName",
"cuKernelGetParamInfo",
"cuKernelSetAttribute",
"cuKernelSetCacheConfig",
"cuLaunch",
"cuLaunchCooperativeKernel",
"cuLaunchCooperativeKernelMultiDevice",
"cuLaunchCooperativeKernel_ptsz",
"cuLaunchGrid",
"cuLaunchGridAsync",
"cuLaunchHostFunc",
"cuLaunchHostFunc_ptsz",
"cuLaunchKernel",
"cuLaunchKernelEx",
"cuLaunchKernelEx_ptsz",
"cuLaunchKernel_ptsz",
"cuLibraryEnumerateKernels",
"cuLibraryGetGlobal",
"cuLibraryGetKernel",
"cuLibraryGetKernelCount",
"cuLibraryGetManaged",
"cuLibraryGetModule",
"cuLibraryGetUnifiedFunction",
"cuLibraryLoadData",
"cuLibraryLoadFromFile",
"cuLibraryUnload",
"cuLinkAddData",
"cuLinkAddData_v2",
"cuLinkAddFile",
"cuLinkAddFile_v2",
"cuLinkComplete",
"cuLinkCreate",
"cuLinkCreate_v2",
"cuLinkDestroy",
"cuMemAddressFree",
"cuMemAddressReserve",
"cuMemAdvise",
"cuMemAdvise_v2",
"cuMemAlloc",
"cuMemAllocAsync",
"cuMemAllocAsync_ptsz",
"cuMemAllocFromPoolAsync",
"cuMemAllocFromPoolAsync_ptsz",
"cuMemAllocHost",
"cuMemAllocHost_v2",
"cuMemAllocManaged",
"cuMemAllocPitch",
"cuMemAllocPitch_v2",
"cuMemAlloc_v2",
"cuMemBatchDecompressAsync",
"cuMemBatchDecompressAsync_ptsz",
"cuMemCreate",
"cuMemExportToShareableHandle",
"cuMemFree",
"cuMemFreeAsync",
"cuMemFreeAsync_ptsz",
"cuMemFreeHost",
"cuMemFree_v2",
"cuMemGetAccess",
"cuMemGetAddressRange",
"cuMemGetAddressRange_v2",
"cuMemGetAllocationGranularity",
"cuMemGetAllocationPropertiesFromHandle",
"cuMemGetAttribute",
"cuMemGetAttribute_v2",
"cuMemGetHandleForAddressRange",
"cuMemGetInfo",
"cuMemGetInfo_v2",
"cuMemHostAlloc",
"cuMemHostGetDevicePointer",
"cuMemHostGetDevicePointer_v2",
"cuMemHostGetFlags",
"cuMemHostRegister",
"cuMemHostRegister_v2",
"cuMemHostUnregister",
"cuMemImportFromShareableHandle",
"cuMemMap",
"cuMemMapArrayAsync",
"cuMemMapArrayAsync_ptsz",
"cuMemPoolCreate",
"cuMemPoolDestroy",
"cuMemPoolExportPointer",
"cuMemPoolExportToShareableHandle",
"cuMemPoolGetAccess",
"cuMemPoolGetAttribute",
"cuMemPoolImportFromShareableHandle",
"cuMemPoolImportPointer",
"cuMemPoolSetAccess",
"cuMemPoolSetAttribute",
"cuMemPoolTrimTo",
"cuMemPrefetchAsync",
"cuMemPrefetchAsync_ptsz",
"cuMemPrefetchAsync_v2",
"cuMemPrefetchAsync_v2_ptsz",
"cuMemRangeGetAttribute",
"cuMemRangeGetAttributes",
"cuMemRelease",
"cuMemRetainAllocationHandle",
"cuMemSetAccess",
"cuMemUnmap",
"cuMemcpy",
"cuMemcpy2D",
"cuMemcpy2DAsync",
"cuMemcpy2DAsync_v2",
"cuMemcpy2DAsync_v2_ptsz",
"cuMemcpy2DUnaligned",
"cuMemcpy2DUnaligned_v2",
"cuMemcpy2DUnaligned_v2_ptds",
"cuMemcpy2D_v2",
"cuMemcpy2D_v2_ptds",
"cuMemcpy3D",
"cuMemcpy3DAsync",
"cuMemcpy3DAsync_v2",
"cuMemcpy3DAsync_v2_ptsz",
"cuMemcpy3DBatchAsync",
"cuMemcpy3DBatchAsync_ptsz",
"cuMemcpy3DPeer",
"cuMemcpy3DPeerAsync",
"cuMemcpy3DPeerAsync_ptsz",
"cuMemcpy3DPeer_ptds",
"cuMemcpy3D_v2",
"cuMemcpy3D_v2_ptds",
"cuMemcpyAsync",
"cuMemcpyAsync_ptsz",
"cuMemcpyAtoA",
"cuMemcpyAtoA_v2",
"cuMemcpyAtoA_v2_ptds",
"cuMemcpyAtoD",
"cuMemcpyAtoD_v2",
"cuMemcpyAtoD_v2_ptds",
"cuMemcpyAtoH",
"cuMemcpyAtoHAsync",
"cuMemcpyAtoHAsync_v2",
"cuMemcpyAtoHAsync_v2_ptsz",
"cuMemcpyAtoH_v2",
"cuMemcpyAtoH_v2_ptds",
"cuMemcpyBatchAsync",
"cuMemcpyBatchAsync_ptsz",
"cuMemcpyDtoA",
"cuMemcpyDtoA_v2",
"cuMemcpyDtoA_v2_ptds",
"cuMemcpyDtoD",
"cuMemcpyDtoDAsync",
"cuMemcpyDtoDAsync_v2",
"cuMemcpyDtoDAsync_v2_ptsz",
"cuMemcpyDtoD_v2",
"cuMemcpyDtoD_v2_ptds",
"cuMemcpyDtoH",
"cuMemcpyDtoHAsync",
"cuMemcpyDtoHAsync_v2",
"cuMemcpyDtoHAsync_v2_ptsz",
"cuMemcpyDtoH_v2",
"cuMemcpyDtoH_v2_ptds",
"cuMemcpyHtoA",
"cuMemcpyHtoAAsync",
"cuMemcpyHtoAAsync_v2",
"cuMemcpyHtoAAsync_v2_ptsz",
"cuMemcpyHtoA_v2",
"cuMemcpyHtoA_v2_ptds",
"cuMemcpyHtoD",
"cuMemcpyHtoDAsync",
"cuMemcpyHtoDAsync_v2",
"cuMemcpyHtoDAsync_v2_ptsz",
"cuMemcpyHtoD_v2",
"cuMemcpyHtoD_v2_ptds",
"cuMemcpyPeer",
"cuMemcpyPeerAsync",
"cuMemcpyPeerAsync_ptsz",
"cuMemcpyPeer_ptds",
"cuMemcpy_ptds",
"cuMemsetD16",
"cuMemsetD16Async",
"cuMemsetD16Async_ptsz",
"cuMemsetD16_v2",
"cuMemsetD16_v2_ptds",
"cuMemsetD2D16",
"cuMemsetD2D16Async",
"cuMemsetD2D16Async_ptsz",
"cuMemsetD2D16_v2",
"cuMemsetD2D16_v2_ptds",
"cuMemsetD2D32",
"cuMemsetD2D32Async",
"cuMemsetD2D32Async_ptsz",
"cuMemsetD2D32_v2",
"cuMemsetD2D32_v2_ptds",
"cuMemsetD2D8",
"cuMemsetD2D8Async",
"cuMemsetD2D8Async_ptsz",
"cuMemsetD2D8_v2",
"cuMemsetD2D8_v2_ptds",
"cuMemsetD32",
"cuMemsetD32Async",
"cuMemsetD32Async_ptsz",
"cuMemsetD32_v2",
"cuMemsetD32_v2_ptds",
"cuMemsetD8",
"cuMemsetD8Async",
"cuMemsetD8Async_ptsz",
"cuMemsetD8_v2",
"cuMemsetD8_v2_ptds",
"cuMipmappedArrayCreate",
"cuMipmappedArrayDestroy",
"cuMipmappedArrayGetLevel",
"cuMipmappedArrayGetMemoryRequirements",
"cuMipmappedArrayGetSparseProperties",
"cuModuleEnumerateFunctions",
"cuModuleGetFunction",
"cuModuleGetFunctionCount",
"cuModuleGetGlobal",
"cuModuleGetGlobal_v2",
"cuModuleGetLoadingMode",
"cuModuleGetSurfRef",
"cuModuleGetTexRef",
"cuModuleLoad",
"cuModuleLoadData",
"cuModuleLoadDataEx",
"cuModuleLoadFatBinary",
"cuModuleUnload",
"cuMulticastAddDevice",
"cuMulticastBindAddr",
"cuMulticastBindMem",
"cuMulticastCreate",
"cuMulticastGetGranularity",
"cuMulticastUnbind",
"cuOccupancyAvailableDynamicSMemPerBlock",
"cuOccupancyMaxActiveBlocksPerMultiprocessor",
"cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags",
"cuOccupancyMaxActiveClusters",
"cuOccupancyMaxPotentialBlockSize",
"cuOccupancyMaxPotentialBlockSizeWithFlags",
"cuOccupancyMaxPotentialClusterSize",
"cuParamSetSize",
"cuParamSetTexRef",
"cuParamSetf",
"cuParamSeti",
"cuParamSetv",
"cuPointerGetAttribute",
"cuPointerGetAttributes",
"cuPointerSetAttribute",
"cuProfilerInitialize",
"cuProfilerStart",
"cuProfilerStop",
"cuSignalExternalSemaphoresAsync",
"cuSignalExternalSemaphoresAsync_ptsz",
"cuStreamAddCallback",
"cuStreamAddCallback_ptsz",
"cuStreamAttachMemAsync",
"cuStreamAttachMemAsync_ptsz",
"cuStreamBatchMemOp",
"cuStreamBatchMemOp_ptsz",
"cuStreamBatchMemOp_v2",
"cuStreamBatchMemOp_v2_ptsz",
"cuStreamBeginCapture",
"cuStreamBeginCaptureToGraph",
"cuStreamBeginCaptureToGraph_ptsz",
"cuStreamBeginCapture_ptsz",
"cuStreamBeginCapture_v2",
"cuStreamBeginCapture_v2_ptsz",
"cuStreamCopyAttributes",
"cuStreamCopyAttributes_ptsz",
"cuStreamCreate",
"cuStreamCreateWithPriority",
"cuStreamDestroy",
"cuStreamDestroy_v2",
"cuStreamEndCapture",
"cuStreamEndCapture_ptsz",
"cuStreamGetAttribute",
"cuStreamGetAttribute_ptsz",
"cuStreamGetCaptureInfo",
"cuStreamGetCaptureInfo_ptsz",
"cuStreamGetCaptureInfo_v2",
"cuStreamGetCaptureInfo_v2_ptsz",
"cuStreamGetCaptureInfo_v3",
"cuStreamGetCaptureInfo_v3_ptsz",
"cuStreamGetCtx",
"cuStreamGetCtx_ptsz",
"cuStreamGetCtx_v2",
"cuStreamGetCtx_v2_ptsz",
"cuStreamGetDevice",
"cuStreamGetDevice_ptsz",
"cuStreamGetFlags",
"cuStreamGetFlags_ptsz",
"cuStreamGetGreenCtx",
"cuStreamGetId",
"cuStreamGetId_ptsz",
"cuStreamGetPriority",
"cuStreamGetPriority_ptsz",
"cuStreamIsCapturing",
"cuStreamIsCapturing_ptsz",
"cuStreamQuery",
"cuStreamQuery_ptsz",
"cuStreamSetAttribute",
"cuStreamSetAttribute_ptsz",
"cuStreamSynchronize",
"cuStreamSynchronize_ptsz",
"cuStreamUpdateCaptureDependencies",
"cuStreamUpdateCaptureDependencies_ptsz",
"cuStreamUpdateCaptureDependencies_v2",
"cuStreamUpdateCaptureDependencies_v2_ptsz",
"cuStreamWaitEvent",
"cuStreamWaitEvent_ptsz",
"cuStreamWaitValue32",
"cuStreamWaitValue32_ptsz",
"cuStreamWaitValue32_v2",
"cuStreamWaitValue32_v2_ptsz",
"cuStreamWaitValue64",
"cuStreamWaitValue64_ptsz",
"cuStreamWaitValue64_v2",
"cuStreamWaitValue64_v2_ptsz",
"cuStreamWriteValue32",
"cuStreamWriteValue32_ptsz",
"cuStreamWriteValue32_v2",
"cuStreamWriteValue32_v2_ptsz",
"cuStreamWriteValue64",
"cuStreamWriteValue64_ptsz",
"cuStreamWriteValue64_v2",
"cuStreamWriteValue64_v2_ptsz",
"cuSurfObjectCreate",
"cuSurfObjectDestroy",
"cuSurfObjectGetResourceDesc",
"cuSurfRefGetArray",
"cuSurfRefSetArray",
"cuTensorMapEncodeIm2col",
"cuTensorMapEncodeIm2colWide",
"cuTensorMapEncodeTiled",
"cuTensorMapReplaceAddress",
"cuTexObjectCreate",
"cuTexObjectDestroy",
"cuTexObjectGetResourceDesc",
"cuTexObjectGetResourceViewDesc",
"cuTexObjectGetTextureDesc",
"cuTexRefCreate",
"cuTexRefDestroy",
"cuTexRefGetAddress",
"cuTexRefGetAddressMode",
"cuTexRefGetAddress_v2",
"cuTexRefGetArray",
"cuTexRefGetBorderColor",
"cuTexRefGetFilterMode",
"cuTexRefGetFlags",
"cuTexRefGetFormat",
"cuTexRefGetMaxAnisotropy",
"cuTexRefGetMipmapFilterMode",
"cuTexRefGetMipmapLevelBias",
"cuTexRefGetMipmapLevelClamp",
"cuTexRefGetMipmappedArray",
"cuTexRefSetAddress",
"cuTexRefSetAddress2D",
"cuTexRefSetAddress2D_v2",
"cuTexRefSetAddress2D_v3",
"cuTexRefSetAddressMode",
"cuTexRefSetAddress_v2",
"cuTexRefSetArray",
"cuTexRefSetBorderColor",
"cuTexRefSetFilterMode",
"cuTexRefSetFlags",
"cuTexRefSetFormat",
"cuTexRefSetMaxAnisotropy",
"cuTexRefSetMipmapFilterMode",
"cuTexRefSetMipmapLevelBias",
"cuTexRefSetMipmapLevelClamp",
"cuTexRefSetMipmappedArray",
"cuThreadExchangeStreamCaptureMode",
"cuUserObjectCreate",
"cuUserObjectRelease",
"cuUserObjectRetain",
"cuVDPAUCtxCreate",
"cuVDPAUCtxCreate_v2",
"cuVDPAUGetDevice",
"cuWaitExternalSemaphoresAsync",
"cuWaitExternalSemaphoresAsync_ptsz",
"cudbgApiAttach",
"cudbgApiDetach",
"cudbgApiInit",
"cudbgGetAPI",
"cudbgGetAPIVersion",
"cudbgMain",
"cudbgReportDriverApiError",
"cudbgReportDriverInternalError",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libcuda_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libcuda_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libcuda_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libcuda_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libcuda_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libcuda_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libcuda_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libcuda_so_tramp_table, 0, SYM_COUNT * sizeof(_libcuda_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,282 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libnvcuvid.so.1: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libnvcuvid.so.1");
CHECK(lib_handle, "failed to load library 'libnvcuvid.so.1' via callback ''");
#else
lib_handle = dlopen("libnvcuvid.so.1", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libnvcuvid.so.1' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"NvToolCreateInterface",
"NvToolDestroyInterface",
"NvToolGetApiFunctionCount",
"NvToolGetApiID",
"NvToolGetApiNames",
"NvToolGetInterface",
"NvToolSetApiID",
"NvToolSetInterface",
"__std_1U4S4U_X02",
"__std_2U4S4U_X08",
"__std_4U4S4U_X04",
"cuvidConvertYUVToRGB",
"cuvidConvertYUVToRGBArray",
"cuvidCreateDecoder",
"cuvidCreateVideoParser",
"cuvidCreateVideoSource",
"cuvidCreateVideoSourceW",
"cuvidCtxLock",
"cuvidCtxLockCreate",
"cuvidCtxLockDestroy",
"cuvidCtxUnlock",
"cuvidDecodePicture",
"cuvidDestroyDecoder",
"cuvidDestroyVideoParser",
"cuvidDestroyVideoSource",
"cuvidGetDecodeStatus",
"cuvidGetDecoderCaps",
"cuvidGetSourceAudioFormat",
"cuvidGetSourceVideoFormat",
"cuvidGetVideoSourceState",
"cuvidMapVideoFrame",
"cuvidMapVideoFrame64",
"cuvidParseVideoData",
"cuvidPrivateOp",
"cuvidReconfigureDecoder",
"cuvidSetVideoSourceState",
"cuvidUnmapVideoFrame",
"cuvidUnmapVideoFrame64",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libnvcuvid_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libnvcuvid_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libnvcuvid_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libnvcuvid_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libnvcuvid_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libnvcuvid_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libnvcuvid_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libnvcuvid_so_tramp_table, 0, SYM_COUNT * sizeof(_libnvcuvid_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,245 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libva-drm.so.2: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libva-drm.so.2");
CHECK(lib_handle, "failed to load library 'libva-drm.so.2' via callback ''");
#else
lib_handle = dlopen("libva-drm.so.2", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libva-drm.so.2' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"vaGetDisplayDRM",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libva_drm_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libva_drm_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libva_drm_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libva_drm_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libva_drm_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libva_drm_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libva_drm_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libva_drm_so_tramp_table, 0, SYM_COUNT * sizeof(_libva_drm_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,122 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#define lr x30
#define ip0 x16
.section .note.GNU-stack,"",@progbits
.data
.globl _libva_drm_so_tramp_table
.hidden _libva_drm_so_tramp_table
.align 8
_libva_drm_so_tramp_table:
.zero 16
.text
.globl _libva_drm_so_tramp_resolve
.hidden _libva_drm_so_tramp_resolve
.globl _libva_drm_so_save_regs_and_resolve
.hidden _libva_drm_so_save_regs_and_resolve
.type _libva_drm_so_save_regs_and_resolve, %function
_libva_drm_so_save_regs_and_resolve:
.cfi_startproc
// Slow path which calls dlsym, taken only on first call.
// Registers are saved according to "Procedure Call Standard for the Arm® 64-bit Architecture".
// For DWARF directives, read https://www.imperialviolet.org/2017/01/18/cfi.html.
// Stack is aligned at 16 bytes
#define PUSH_PAIR(reg1, reg2) stp reg1, reg2, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset reg1, 0; .cfi_rel_offset reg2, 8
#define POP_PAIR(reg1, reg2) ldp reg1, reg2, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore reg2; .cfi_restore reg1
#define PUSH_WIDE_PAIR(reg1, reg2) stp reg1, reg2, [sp, #-32]!; .cfi_adjust_cfa_offset 32; .cfi_rel_offset reg1, 0; .cfi_rel_offset reg2, 16
#define POP_WIDE_PAIR(reg1, reg2) ldp reg1, reg2, [sp], #32; .cfi_adjust_cfa_offset -32; .cfi_restore reg2; .cfi_restore reg1
// Save only arguments (and lr)
PUSH_PAIR(x0, x1)
PUSH_PAIR(x2, x3)
PUSH_PAIR(x4, x5)
PUSH_PAIR(x6, x7)
PUSH_PAIR(x8, lr)
ldr x0, [sp, #80] // 16*5
PUSH_WIDE_PAIR(q0, q1)
PUSH_WIDE_PAIR(q2, q3)
PUSH_WIDE_PAIR(q4, q5)
PUSH_WIDE_PAIR(q6, q7)
// Stack is aligned at 16 bytes
bl _libva_drm_so_tramp_resolve
mov ip0, x0
// TODO: pop pc?
POP_WIDE_PAIR(q6, q7)
POP_WIDE_PAIR(q4, q5)
POP_WIDE_PAIR(q2, q3)
POP_WIDE_PAIR(q0, q1)
POP_PAIR(x8, lr)
POP_PAIR(x6, x7)
POP_PAIR(x4, x5)
POP_PAIR(x2, x3)
POP_PAIR(x0, x1)
br lr
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl vaGetDisplayDRM
.p2align 4
.type vaGetDisplayDRM, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden vaGetDisplayDRM
#endif
vaGetDisplayDRM:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _libva_drm_so_tramp_table+0
ldr ip0, [ip0, #:lo12:_libva_drm_so_tramp_table+0]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, 0 & 0xffff
#if 0 > 0xffff
movk ip0, 0 >> 16, lsl #16
#endif
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset lr, 8
bl _libva_drm_so_save_regs_and_resolve
ldp xzr, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr
br ip0
.cfi_endproc
@@ -0,0 +1,332 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libva.so.2: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libva.so.2");
CHECK(lib_handle, "failed to load library 'libva.so.2' via callback ''");
#else
lib_handle = dlopen("libva.so.2", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libva.so.2' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"vaAcquireBufferHandle",
"vaAssociateSubpicture",
"vaAttachProtectedSession",
"vaBeginPicture",
"vaBufferInfo",
"vaBufferSetNumElements",
"vaBufferTypeStr",
"vaConfigAttribTypeStr",
"vaCopy",
"vaCreateBuffer",
"vaCreateBuffer2",
"vaCreateConfig",
"vaCreateContext",
"vaCreateImage",
"vaCreateMFContext",
"vaCreateProtectedSession",
"vaCreateSubpicture",
"vaCreateSurfaces",
"vaDeassociateSubpicture",
"vaDeriveImage",
"vaDestroyBuffer",
"vaDestroyConfig",
"vaDestroyContext",
"vaDestroyImage",
"vaDestroyProtectedSession",
"vaDestroySubpicture",
"vaDestroySurfaces",
"vaDetachProtectedSession",
"vaDisplayIsValid",
"vaEndPicture",
"vaEntrypointStr",
"vaErrorStr",
"vaExportSurfaceHandle",
"vaGetConfigAttributes",
"vaGetDisplayAttributes",
"vaGetImage",
"vaGetLibFunc",
"vaInitialize",
"vaLockSurface",
"vaMFAddContext",
"vaMFReleaseContext",
"vaMFSubmit",
"vaMapBuffer",
"vaMapBuffer2",
"vaMaxNumConfigAttributes",
"vaMaxNumDisplayAttributes",
"vaMaxNumEntrypoints",
"vaMaxNumImageFormats",
"vaMaxNumProfiles",
"vaMaxNumSubpictureFormats",
"vaProfileStr",
"vaProtectedSessionExecute",
"vaPutImage",
"vaQueryConfigAttributes",
"vaQueryConfigEntrypoints",
"vaQueryConfigProfiles",
"vaQueryDisplayAttributes",
"vaQueryImageFormats",
"vaQueryProcessingRate",
"vaQuerySubpictureFormats",
"vaQuerySurfaceAttributes",
"vaQuerySurfaceError",
"vaQuerySurfaceStatus",
"vaQueryVendorString",
"vaQueryVideoProcFilterCaps",
"vaQueryVideoProcFilters",
"vaQueryVideoProcPipelineCaps",
"vaReleaseBufferHandle",
"vaRenderPicture",
"vaSetDisplayAttributes",
"vaSetDriverName",
"vaSetErrorCallback",
"vaSetImagePalette",
"vaSetInfoCallback",
"vaSetSubpictureChromakey",
"vaSetSubpictureGlobalAlpha",
"vaSetSubpictureImage",
"vaStatusStr",
"vaSyncBuffer",
"vaSyncSurface",
"vaSyncSurface2",
"vaTerminate",
"vaUnlockSurface",
"vaUnmapBuffer",
"va_TracePutSurface",
"va_TraceStatus",
"va_newDisplayContext",
"va_newDriverContext",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libva_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libva_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libva_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libva_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libva_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libva_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libva_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libva_so_tramp_table, 0, SYM_COUNT * sizeof(_libva_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,245 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libva-drm.so.2: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libva-drm.so.2");
CHECK(lib_handle, "failed to load library 'libva-drm.so.2' via callback ''");
#else
lib_handle = dlopen("libva-drm.so.2", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libva-drm.so.2' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"vaGetDisplayDRM",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libva_drm_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libva_drm_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libva_drm_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libva_drm_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libva_drm_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libva_drm_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libva_drm_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libva_drm_so_tramp_table, 0, SYM_COUNT * sizeof(_libva_drm_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,192 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.section .note.GNU-stack,"",@progbits
.data
.globl _libva_drm_so_tramp_table
.hidden _libva_drm_so_tramp_table
.align 8
_libva_drm_so_tramp_table:
.zero 16
.text
.globl _libva_drm_so_tramp_resolve
.hidden _libva_drm_so_tramp_resolve
.globl _libva_drm_so_save_regs_and_resolve
.hidden _libva_drm_so_save_regs_and_resolve
.type _libva_drm_so_save_regs_and_resolve, %function
_libva_drm_so_save_regs_and_resolve:
.cfi_startproc
#define PUSH_REG(reg) pushq %reg ; .cfi_adjust_cfa_offset 8; .cfi_rel_offset reg, 0
#define POP_REG(reg) popq %reg ; .cfi_adjust_cfa_offset -8; .cfi_restore reg
#define DEC_STACK(d) subq $d, %rsp; .cfi_adjust_cfa_offset d
#define INC_STACK(d) addq $d, %rsp; .cfi_adjust_cfa_offset -d
#define PUSH_MMX_REG(reg) DEC_STACK(8); movq %reg, (%rsp); .cfi_rel_offset reg, 0
#define POP_MMX_REG(reg) movq (%rsp), %reg; .cfi_restore reg; INC_STACK(8)
#define PUSH_XMM_REG(reg) DEC_STACK(16); movdqa %reg, (%rsp); .cfi_rel_offset reg, 0
#define POP_XMM_REG(reg) movdqa (%rsp), %reg; .cfi_restore reg; INC_STACK(16)
// TODO: cfi_offset/cfi_restore
#define PUSH_YMM_REG(reg) DEC_STACK(32); vmovdqu %reg, (%rsp)
#define POP_YMM_REG(reg) vmovdqu (%rsp), %reg; INC_STACK(32)
// TODO: cfi_offset/cfi_restore
#define PUSH_ZMM_REG(reg) DEC_STACK(64); vmovdqu32 %reg, (%rsp)
#define POP_ZMM_REG(reg) vmovdqu32 (%rsp), %reg; INC_STACK(64)
// Slow path which calls dlsym, taken only on first call.
// All registers are stored to handle arbitrary calling conventions
// (except x87 FPU registers which do not have to be preserved).
// For Dwarf directives, read https://www.imperialviolet.org/2017/01/18/cfi.html.
.cfi_def_cfa_offset 8 // Return address
PUSH_REG(rdi) // 16
mov 0x10(%rsp), %rdi
PUSH_REG(rbx)
PUSH_REG(rbx) // 16
PUSH_REG(rcx)
PUSH_REG(rdx) // 16
PUSH_REG(rbp)
PUSH_REG(rsi) // 16
PUSH_REG(r8)
PUSH_REG(r9) // 16
PUSH_REG(r10)
PUSH_REG(r11) // 16
PUSH_REG(r12)
PUSH_REG(r13) // 16
PUSH_REG(r14)
PUSH_REG(r15) // 16
// Maybe use cpuid instead of macro to detect current vector size...
#ifdef __AVX512F__
PUSH_ZMM_REG(zmm0)
PUSH_ZMM_REG(zmm1)
PUSH_ZMM_REG(zmm2)
PUSH_ZMM_REG(zmm3)
PUSH_ZMM_REG(zmm4)
PUSH_ZMM_REG(zmm5)
PUSH_ZMM_REG(zmm6)
PUSH_ZMM_REG(zmm7)
#elif defined __AVX__
PUSH_YMM_REG(ymm0)
PUSH_YMM_REG(ymm1)
PUSH_YMM_REG(ymm2)
PUSH_YMM_REG(ymm3)
PUSH_YMM_REG(ymm4)
PUSH_YMM_REG(ymm5)
PUSH_YMM_REG(ymm6)
PUSH_YMM_REG(ymm7)
#elif defined __SSE__
PUSH_XMM_REG(xmm0)
PUSH_XMM_REG(xmm1)
PUSH_XMM_REG(xmm2)
PUSH_XMM_REG(xmm3)
PUSH_XMM_REG(xmm4)
PUSH_XMM_REG(xmm5)
PUSH_XMM_REG(xmm6)
PUSH_XMM_REG(xmm7)
#endif
// MMX registers are not used to pass arguments so we do not save them
// Stack is just 8-byte aligned but callee will re-align to 16
call _libva_drm_so_tramp_resolve
#ifdef __AVX512F__
POP_ZMM_REG(zmm7)
POP_ZMM_REG(zmm6)
POP_ZMM_REG(zmm5)
POP_ZMM_REG(zmm4)
POP_ZMM_REG(zmm3)
POP_ZMM_REG(zmm2)
POP_ZMM_REG(zmm1)
POP_ZMM_REG(zmm0) // 16
#elif defined __AVX__
POP_YMM_REG(ymm7)
POP_YMM_REG(ymm6)
POP_YMM_REG(ymm5)
POP_YMM_REG(ymm4)
POP_YMM_REG(ymm3)
POP_YMM_REG(ymm2)
POP_YMM_REG(ymm1)
POP_YMM_REG(ymm0) // 16
#elif defined __SSE__
POP_XMM_REG(xmm7)
POP_XMM_REG(xmm6)
POP_XMM_REG(xmm5)
POP_XMM_REG(xmm4)
POP_XMM_REG(xmm3)
POP_XMM_REG(xmm2)
POP_XMM_REG(xmm1)
POP_XMM_REG(xmm0) // 16
#endif
POP_REG(r15)
POP_REG(r14) // 16
POP_REG(r13)
POP_REG(r12) // 16
POP_REG(r11)
POP_REG(r10) // 16
POP_REG(r9)
POP_REG(r8) // 16
POP_REG(rsi)
POP_REG(rbp) // 16
POP_REG(rdx)
POP_REG(rcx) // 16
POP_REG(rbx)
POP_REG(rbx) // 16
POP_REG(rdi)
ret
.cfi_endproc
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl vaGetDisplayDRM
.p2align 4
.type vaGetDisplayDRM, %function
#ifndef IMPLIB_EXPORT_SHIMS
.hidden vaGetDisplayDRM
#endif
vaGetDisplayDRM:
.cfi_startproc
.cfi_def_cfa_offset 8 // Return address
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $0, _libva_drm_so_tramp_table+0(%rip)
je 2f
1:
jmp *_libva_drm_so_tramp_table+0(%rip)
2:
pushq $0
.cfi_adjust_cfa_offset 8
call _libva_drm_so_save_regs_and_resolve
addq $8, %rsp
.cfi_adjust_cfa_offset -8
jmp *%rax
.cfi_endproc
@@ -0,0 +1,333 @@
/*
* Copyright 2018-2025 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // For RTLD_DEFAULT
#endif
#define HAS_DLOPEN_CALLBACK 0
#define HAS_DLSYM_CALLBACK 0
#define NO_DLOPEN 0
#define LAZY_LOAD 1
#define THREAD_SAFE 1
#include <dlfcn.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <assert.h>
#if THREAD_SAFE
#include <pthread.h>
#endif
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: libva.so.2: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
abort(); \
} \
} while(0)
static void *lib_handle;
static int dlopened;
#if ! NO_DLOPEN
#if THREAD_SAFE
// We need to consider two cases:
// - different threads calling intercepted APIs in parallel
// - same thread calling 2 intercepted APIs recursively
// due to dlopen calling library constructors
// (usually happens only under IMPLIB_EXPORT_SHIMS)
// Current recursive mutex approach will deadlock
// if library constructor starts and joins a new thread
// which (directly or indirectly) calls another library function.
// Such situations should be very rare (although chances
// are higher when -DIMLIB_EXPORT_SHIMS are enabled).
//
// Similar issue is present in Glibc so hopefully it's
// not a big deal: // http://sourceware.org/bugzilla/show_bug.cgi?id=15686
// (also google for "dlopen deadlock).
static pthread_mutex_t mtx;
static int rec_count;
static void init_lock(void) {
// We need recursive lock because dlopen will call library constructors
// which may call other intercepted APIs that will call load_library again.
// PTHREAD_RECURSIVE_MUTEX_INITIALIZER is not portable
// so we do it hard way.
pthread_mutexattr_t attr;
CHECK(0 == pthread_mutexattr_init(&attr), "failed to init mutex");
CHECK(0 == pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE), "failed to init mutex");
CHECK(0 == pthread_mutex_init(&mtx, &attr), "failed to init mutex");
}
static int lock(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
CHECK(0 == pthread_once(&once, init_lock), "failed to init lock");
CHECK(0 == pthread_mutex_lock(&mtx), "failed to lock mutex");
return 0 == __sync_fetch_and_add(&rec_count, 1);
}
static void unlock(void) {
__sync_fetch_and_add(&rec_count, -1);
CHECK(0 == pthread_mutex_unlock(&mtx), "failed to unlock mutex");
}
#else
static int lock(void) {
return 1;
}
static void unlock(void) {}
#endif
static int load_library(void) {
int publish = lock();
if (lib_handle) {
unlock();
return publish;
}
#if HAS_DLOPEN_CALLBACK
extern void *(const char *lib_name);
lib_handle = ("libva.so.2");
CHECK(lib_handle, "failed to load library 'libva.so.2' via callback ''");
#else
lib_handle = dlopen("libva.so.2", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library 'libva.so.2' via dlopen: %s", dlerror());
#endif
// With (non-default) IMPLIB_EXPORT_SHIMS we may call dlopen more than once
// so dlclose it if we are not the first ones
if (__sync_val_compare_and_swap(&dlopened, 0, 1)) {
dlclose(lib_handle);
}
unlock();
return publish;
}
// Run dtor as late as possible in case library functions are
// called in other global dtors
// FIXME: this may crash if one thread is calling into library
// while some other thread executes exit(). It's no clear
// how to fix this besides simply NOT dlclosing library at all.
static void __attribute__((destructor(101))) unload_lib(void) {
if (dlopened) {
dlclose(lib_handle);
lib_handle = 0;
dlopened = 0;
}
}
#endif
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor(101))) load_lib(void) {
load_library();
}
#endif
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
"disabled_va_TraceInit",
"vaAcquireBufferHandle",
"vaAssociateSubpicture",
"vaAttachProtectedSession",
"vaBeginPicture",
"vaBufferInfo",
"vaBufferSetNumElements",
"vaBufferTypeStr",
"vaConfigAttribTypeStr",
"vaCopy",
"vaCreateBuffer",
"vaCreateBuffer2",
"vaCreateConfig",
"vaCreateContext",
"vaCreateImage",
"vaCreateMFContext",
"vaCreateProtectedSession",
"vaCreateSubpicture",
"vaCreateSurfaces",
"vaDeassociateSubpicture",
"vaDeriveImage",
"vaDestroyBuffer",
"vaDestroyConfig",
"vaDestroyContext",
"vaDestroyImage",
"vaDestroyProtectedSession",
"vaDestroySubpicture",
"vaDestroySurfaces",
"vaDetachProtectedSession",
"vaDisplayIsValid",
"vaEndPicture",
"vaEntrypointStr",
"vaErrorStr",
"vaExportSurfaceHandle",
"vaGetConfigAttributes",
"vaGetDisplayAttributes",
"vaGetImage",
"vaGetLibFunc",
"vaInitialize",
"vaLockSurface",
"vaMFAddContext",
"vaMFReleaseContext",
"vaMFSubmit",
"vaMapBuffer",
"vaMapBuffer2",
"vaMaxNumConfigAttributes",
"vaMaxNumDisplayAttributes",
"vaMaxNumEntrypoints",
"vaMaxNumImageFormats",
"vaMaxNumProfiles",
"vaMaxNumSubpictureFormats",
"vaProfileStr",
"vaProtectedSessionExecute",
"vaPutImage",
"vaQueryConfigAttributes",
"vaQueryConfigEntrypoints",
"vaQueryConfigProfiles",
"vaQueryDisplayAttributes",
"vaQueryImageFormats",
"vaQueryProcessingRate",
"vaQuerySubpictureFormats",
"vaQuerySurfaceAttributes",
"vaQuerySurfaceError",
"vaQuerySurfaceStatus",
"vaQueryVendorString",
"vaQueryVideoProcFilterCaps",
"vaQueryVideoProcFilters",
"vaQueryVideoProcPipelineCaps",
"vaReleaseBufferHandle",
"vaRenderPicture",
"vaSetDisplayAttributes",
"vaSetDriverName",
"vaSetErrorCallback",
"vaSetImagePalette",
"vaSetInfoCallback",
"vaSetSubpictureChromakey",
"vaSetSubpictureGlobalAlpha",
"vaSetSubpictureImage",
"vaStatusStr",
"vaSyncBuffer",
"vaSyncSurface",
"vaSyncSurface2",
"vaTerminate",
"vaUnlockSurface",
"vaUnmapBuffer",
"va_TracePutSurface",
"va_TraceStatus",
"va_newDisplayContext",
"va_newDriverContext",
0
};
#define SYM_COUNT (sizeof(sym_names)/sizeof(sym_names[0]) - 1)
extern void *_libva_so_tramp_table[];
// Can be sped up by manually parsing library symtab...
void *_libva_so_tramp_resolve(size_t i) {
assert(i < SYM_COUNT);
int publish = 1;
void *h = 0;
#if NO_DLOPEN
// Library with implementations must have already been loaded.
if (lib_handle) {
// User has specified loaded library
h = lib_handle;
} else {
// User hasn't provided us the loaded library so search the global namespace.
# ifndef IMPLIB_EXPORT_SHIMS
// If shim symbols are hidden we should search
// for first available definition of symbol in library list
h = RTLD_DEFAULT;
# else
// Otherwise look for next available definition
h = RTLD_NEXT;
# endif
}
#else
publish = load_library();
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
void *addr;
#if HAS_DLSYM_CALLBACK
extern void *(void *handle, const char *sym_name);
addr = (h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via callback ", sym_names[i]);
#else
// Dlsym is thread-safe so don't need to protect it.
addr = dlsym(h, sym_names[i]);
CHECK(addr, "failed to resolve symbol '%s' via dlsym: %s", sym_names[i], dlerror());
#endif
if (publish) {
// Use atomic to please Tsan and ensure that preceeding writes
// in library ctors have been delivered before publishing address
(void)__sync_val_compare_and_swap(&_libva_so_tramp_table[i], 0, addr);
}
return addr;
}
// Below APIs are not thread-safe
// and it's not clear how make them such
// (we can not know if some other thread is
// currently executing library code).
// Helper for user to resolve all symbols
void _libva_so_tramp_resolve_all(void) {
size_t i;
for(i = 0; i < SYM_COUNT; ++i)
_libva_so_tramp_resolve(i);
}
// Allows user to specify manually loaded implementation library.
void _libva_so_tramp_set_handle(void *handle) {
// TODO: call unload_lib ?
lib_handle = handle;
dlopened = 0;
}
// Resets all resolved symbols. This is needed in case
// client code wants to reload interposed library multiple times.
void _libva_so_tramp_reset(void) {
// TODO: call unload_lib ?
memset(_libva_so_tramp_table, 0, SYM_COUNT * sizeof(_libva_so_tramp_table[0]));
lib_handle = 0;
dlopened = 0;
}
#ifdef __cplusplus
} // extern "C"
#endif
@@ -0,0 +1,62 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(target_os = "android")]
pub mod android;
pub mod apm;
pub mod audio_device_controller;
pub mod audio_mixer;
pub mod audio_resampler;
pub mod audio_track;
pub mod candidate;
pub mod data_channel;
#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
pub mod desktop_capturer;
pub mod frame_cryptor;
pub mod helper;
pub mod jsep;
pub mod media_stream;
pub mod media_stream_track;
pub mod packet_trailer;
pub mod peer_connection;
pub mod peer_connection_factory;
pub mod prohibit_libsrtp_initialization;
pub mod recorded_audio_tap;
pub mod rtc_error;
pub mod rtp_parameters;
pub mod rtp_receiver;
pub mod rtp_sender;
pub mod rtp_transceiver;
pub mod video_frame;
pub mod video_frame_buffer;
pub mod video_track;
pub mod webrtc;
pub mod yuv_helper;
pub const MEDIA_TYPE_VIDEO: &str = "video";
pub const MEDIA_TYPE_AUDIO: &str = "audio";
pub const MEDIA_TYPE_DATA: &str = "data";
macro_rules! impl_thread_safety {
($obj:ty, Send) => {
unsafe impl Send for $obj {}
};
($obj:ty, Send + Sync) => {
unsafe impl Send for $obj {}
unsafe impl Sync for $obj {}
};
}
pub(crate) use impl_thread_safety;
@@ -0,0 +1,101 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "livekit/media_stream.h"
#include <algorithm>
#include <iostream>
#include <memory>
#include "api/media_stream_interface.h"
#include "api/video/video_frame.h"
#include "api/video/video_rotation.h"
#include "audio/remix_resample.h"
#include "common_audio/include/audio_util.h"
#include "rtc_base/logging.h"
#include "rtc_base/ref_counted_object.h"
#include "rtc_base/time_utils.h"
namespace livekit_ffi {
MediaStream::MediaStream(
std::shared_ptr<RtcRuntime> rtc_runtime,
webrtc::scoped_refptr<webrtc::MediaStreamInterface> stream)
: rtc_runtime_(rtc_runtime), media_stream_(std::move(stream)) {}
rust::String MediaStream::id() const {
return media_stream_->id();
}
rust::Vec<VideoTrackPtr> MediaStream::get_video_tracks() const {
rust::Vec<VideoTrackPtr> rust;
for (auto video : media_stream_->GetVideoTracks())
rust.push_back(
VideoTrackPtr{rtc_runtime_->get_or_create_video_track(video)});
return rust;
}
rust::Vec<AudioTrackPtr> MediaStream::get_audio_tracks() const {
rust::Vec<AudioTrackPtr> rust;
for (auto audio : media_stream_->GetAudioTracks())
rust.push_back(
AudioTrackPtr{rtc_runtime_->get_or_create_audio_track(audio)});
return rust;
}
std::shared_ptr<AudioTrack> MediaStream::find_audio_track(
rust::String track_id) const {
return rtc_runtime_->get_or_create_audio_track(
media_stream_->FindAudioTrack(track_id.c_str()));
}
std::shared_ptr<VideoTrack> MediaStream::find_video_track(
rust::String track_id) const {
return rtc_runtime_->get_or_create_video_track(
media_stream_->FindVideoTrack(track_id.c_str()));
}
bool MediaStream::add_track(std::shared_ptr<MediaStreamTrack> track) const {
if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
return media_stream_->AddTrack(
webrtc::scoped_refptr<webrtc::VideoTrackInterface>(
static_cast<webrtc::VideoTrackInterface*>(
track->rtc_track().get())));
} else {
return media_stream_->AddTrack(
webrtc::scoped_refptr<webrtc::AudioTrackInterface>(
static_cast<webrtc::AudioTrackInterface*>(
track->rtc_track().get())));
}
}
bool MediaStream::remove_track(std::shared_ptr<MediaStreamTrack> track) const {
if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
return media_stream_->RemoveTrack(
webrtc::scoped_refptr<webrtc::VideoTrackInterface>(
static_cast<webrtc::VideoTrackInterface*>(
track->rtc_track().get())));
} else {
return media_stream_->RemoveTrack(
webrtc::scoped_refptr<webrtc::AudioTrackInterface>(
static_cast<webrtc::AudioTrackInterface*>(
track->rtc_track().get())));
}
}
} // namespace livekit_ffi
@@ -0,0 +1,49 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
extern "C++" {
include!("livekit/helper.h");
include!("livekit/media_stream_track.h");
include!("livekit/audio_track.h");
include!("livekit/video_track.h");
type MediaStreamTrack = crate::media_stream_track::ffi::MediaStreamTrack;
type AudioTrack = crate::audio_track::ffi::AudioTrack;
type VideoTrack = crate::video_track::ffi::VideoTrack;
type VideoTrackPtr = crate::helper::ffi::VideoTrackPtr;
type AudioTrackPtr = crate::helper::ffi::AudioTrackPtr;
}
unsafe extern "C++" {
include!("livekit/media_stream.h");
type MediaStream;
fn id(self: &MediaStream) -> String;
fn get_audio_tracks(self: &MediaStream) -> Vec<AudioTrackPtr>;
fn get_video_tracks(self: &MediaStream) -> Vec<VideoTrackPtr>;
fn find_audio_track(self: &MediaStream, track_id: String) -> SharedPtr<AudioTrack>;
fn find_video_track(self: &MediaStream, track_id: String) -> SharedPtr<VideoTrack>;
fn add_track(self: &MediaStream, audio_track: SharedPtr<MediaStreamTrack>) -> bool;
fn remove_track(self: &MediaStream, audio_track: SharedPtr<MediaStreamTrack>) -> bool;
fn _shared_media_stream() -> SharedPtr<MediaStream>;
}
}
impl_thread_safety!(ffi::MediaStream, Send + Sync);
@@ -0,0 +1,58 @@
/*
* Copyright 2025 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <algorithm>
#include <iostream>
#include <memory>
#include "api/media_stream_interface.h"
#include "api/video/video_frame.h"
#include "api/video/video_rotation.h"
#include "audio/remix_resample.h"
#include "common_audio/include/audio_util.h"
#include "livekit/media_stream.h"
#include "rtc_base/logging.h"
#include "rtc_base/ref_counted_object.h"
#include "rtc_base/time_utils.h"
namespace livekit_ffi {
MediaStreamTrack::MediaStreamTrack(
std::shared_ptr<RtcRuntime> rtc_runtime,
webrtc::scoped_refptr<webrtc::MediaStreamTrackInterface> track)
: rtc_runtime_(rtc_runtime), track_(std::move(track)) {}
rust::String MediaStreamTrack::kind() const {
return track_->kind();
}
rust::String MediaStreamTrack::id() const {
return track_->id();
}
bool MediaStreamTrack::enabled() const {
return track_->enabled();
}
bool MediaStreamTrack::set_enabled(bool enable) const {
return track_->set_enabled(enable);
}
TrackState MediaStreamTrack::state() const {
return static_cast<TrackState>(track_->state());
}
} // namespace livekit_ffi
@@ -0,0 +1,40 @@
// Copyright 2025 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::impl_thread_safety;
#[cxx::bridge(namespace = "livekit_ffi")]
pub mod ffi {
#[repr(i32)]
pub enum TrackState {
Live,
Ended,
}
unsafe extern "C++" {
include!("livekit/media_stream_track.h");
type MediaStreamTrack;
fn kind(self: &MediaStreamTrack) -> String;
fn id(self: &MediaStreamTrack) -> String;
fn enabled(self: &MediaStreamTrack) -> bool;
fn set_enabled(self: &MediaStreamTrack, enable: bool) -> bool;
fn state(self: &MediaStreamTrack) -> TrackState;
fn _shared_media_stream_track() -> SharedPtr<MediaStreamTrack>;
}
}
impl_thread_safety!(ffi::MediaStreamTrack, Send + Sync);
@@ -0,0 +1,850 @@
/*
* Copyright 2017-2022 NVIDIA Corporation. All rights reserved.
*
* Please refer to the NVIDIA end user license agreement (EULA) associated
* with this source code for terms and conditions that govern your use of
* this software. Any use, reproduction, disclosure, or distribution of
* this software and related documentation outside the terms of the EULA
* is strictly prohibited.
*
*/
#include <iostream>
#include <algorithm>
#include <chrono>
#include <cmath>
#include "nvcuvid.h"
#include "NvDecoder.h"
#include "Utils/Logger.h"
simplelogger::Logger* logger = simplelogger::LoggerFactory::CreateConsoleLogger();
#define START_TIMER auto start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(print_message) int64_t elapsedTime = std::chrono::duration_cast<std::chrono::milliseconds>( \
std::chrono::high_resolution_clock::now() - start).count(); \
std::cout << print_message << \
elapsedTime \
<< " ms " << std::endl;
#define CUDA_DRVAPI_CALL( call ) \
do \
{ \
CUresult err__ = call; \
if (err__ != CUDA_SUCCESS) \
{ \
const char *szErrName = NULL; \
cuGetErrorName(err__, &szErrName); \
std::ostringstream errorLog; \
errorLog << "CUDA driver API error " << szErrName ; \
throw NVDECException::makeNVDECException(errorLog.str(), err__, __FUNCTION__, __FILE__, __LINE__); \
} \
} \
while (0)
static const char * GetVideoCodecString(cudaVideoCodec eCodec) {
static struct {
cudaVideoCodec eCodec;
const char *name;
} aCodecName [] = {
{ cudaVideoCodec_MPEG1, "MPEG-1" },
{ cudaVideoCodec_MPEG2, "MPEG-2" },
{ cudaVideoCodec_MPEG4, "MPEG-4 (ASP)" },
{ cudaVideoCodec_VC1, "VC-1/WMV" },
{ cudaVideoCodec_H264, "AVC/H.264" },
{ cudaVideoCodec_JPEG, "M-JPEG" },
{ cudaVideoCodec_H264_SVC, "H.264/SVC" },
{ cudaVideoCodec_H264_MVC, "H.264/MVC" },
{ cudaVideoCodec_HEVC, "H.265/HEVC" },
{ cudaVideoCodec_VP8, "VP8" },
{ cudaVideoCodec_VP9, "VP9" },
{ cudaVideoCodec_AV1, "AV1" },
{ cudaVideoCodec_NumCodecs, "Invalid" },
{ cudaVideoCodec_YUV420, "YUV 4:2:0" },
{ cudaVideoCodec_YV12, "YV12 4:2:0" },
{ cudaVideoCodec_NV12, "NV12 4:2:0" },
{ cudaVideoCodec_YUYV, "YUYV 4:2:2" },
{ cudaVideoCodec_UYVY, "UYVY 4:2:2" },
};
if (eCodec >= 0 && eCodec <= cudaVideoCodec_NumCodecs) {
return aCodecName[eCodec].name;
}
for (int i = cudaVideoCodec_NumCodecs + 1; i < sizeof(aCodecName) / sizeof(aCodecName[0]); i++) {
if (eCodec == aCodecName[i].eCodec) {
return aCodecName[eCodec].name;
}
}
return "Unknown";
}
static const char * GetVideoChromaFormatString(cudaVideoChromaFormat eChromaFormat) {
static struct {
cudaVideoChromaFormat eChromaFormat;
const char *name;
} aChromaFormatName[] = {
{ cudaVideoChromaFormat_Monochrome, "YUV 400 (Monochrome)" },
{ cudaVideoChromaFormat_420, "YUV 420" },
{ cudaVideoChromaFormat_422, "YUV 422" },
{ cudaVideoChromaFormat_444, "YUV 444" },
};
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wtautological-constant-out-of-range-compare"
if (eChromaFormat >= 0 && eChromaFormat < sizeof(aChromaFormatName) / sizeof(aChromaFormatName[0])) {
return aChromaFormatName[eChromaFormat].name;
}
#pragma clang diagnostic pop
return "Unknown";
}
static float GetChromaHeightFactor(cudaVideoSurfaceFormat eSurfaceFormat)
{
float factor = 0.5;
switch (eSurfaceFormat)
{
case cudaVideoSurfaceFormat_NV12:
case cudaVideoSurfaceFormat_P016:
factor = 0.5;
break;
case cudaVideoSurfaceFormat_YUV444:
case cudaVideoSurfaceFormat_YUV444_16Bit:
factor = 1.0;
break;
}
return factor;
}
static int GetChromaPlaneCount(cudaVideoSurfaceFormat eSurfaceFormat)
{
int numPlane = 1;
switch (eSurfaceFormat)
{
case cudaVideoSurfaceFormat_NV12:
case cudaVideoSurfaceFormat_P016:
numPlane = 1;
break;
case cudaVideoSurfaceFormat_YUV444:
case cudaVideoSurfaceFormat_YUV444_16Bit:
numPlane = 2;
break;
}
return numPlane;
}
std::map<int, int64_t> NvDecoder::sessionOverHead = { {0,0}, {1,0} };
/**
* @brief This function is used to get codec string from codec id
*/
const char *NvDecoder::GetCodecString(cudaVideoCodec eCodec)
{
return GetVideoCodecString(eCodec);
}
/* Called when the parser encounters sequence header for AV1 SVC content
* return value interpretation:
* < 0 : fail, >=0: succeeded (bit 0-9: currOperatingPoint, bit 10-10: bDispAllLayer, bit 11-30: reserved, must be set 0)
*/
int NvDecoder::GetOperatingPoint(CUVIDOPERATINGPOINTINFO *pOPInfo)
{
if (pOPInfo->codec == cudaVideoCodec_AV1)
{
if (pOPInfo->av1.operating_points_cnt > 1)
{
// clip has SVC enabled
if (m_nOperatingPoint >= pOPInfo->av1.operating_points_cnt)
m_nOperatingPoint = 0;
printf("AV1 SVC clip: operating point count %d ", pOPInfo->av1.operating_points_cnt);
printf("Selected operating point: %d, IDC 0x%x bOutputAllLayers %d\n", m_nOperatingPoint, pOPInfo->av1.operating_points_idc[m_nOperatingPoint], m_bDispAllLayers);
return (m_nOperatingPoint | (m_bDispAllLayers << 10));
}
}
return -1;
}
/* Return value from HandleVideoSequence() are interpreted as :
* 0: fail, 1: succeeded, > 1: override dpb size of parser (set by CUVIDPARSERPARAMS::ulMaxNumDecodeSurfaces while creating parser)
*/
int NvDecoder::HandleVideoSequence(CUVIDEOFORMAT *pVideoFormat)
{
START_TIMER
m_videoInfo.str("");
m_videoInfo.clear();
m_videoInfo << "Video Input Information" << std::endl
<< "\tCodec : " << GetVideoCodecString(pVideoFormat->codec) << std::endl
<< "\tFrame rate : " << pVideoFormat->frame_rate.numerator << "/" << pVideoFormat->frame_rate.denominator
<< " = " << 1.0 * pVideoFormat->frame_rate.numerator / pVideoFormat->frame_rate.denominator << " fps" << std::endl
<< "\tSequence : " << (pVideoFormat->progressive_sequence ? "Progressive" : "Interlaced") << std::endl
<< "\tCoded size : [" << pVideoFormat->coded_width << ", " << pVideoFormat->coded_height << "]" << std::endl
<< "\tDisplay area : [" << pVideoFormat->display_area.left << ", " << pVideoFormat->display_area.top << ", "
<< pVideoFormat->display_area.right << ", " << pVideoFormat->display_area.bottom << "]" << std::endl
<< "\tChroma : " << GetVideoChromaFormatString(pVideoFormat->chroma_format) << std::endl
<< "\tBit depth : " << pVideoFormat->bit_depth_luma_minus8 + 8
;
m_videoInfo << std::endl;
int nDecodeSurface = pVideoFormat->min_num_decode_surfaces;
CUVIDDECODECAPS decodecaps;
memset(&decodecaps, 0, sizeof(decodecaps));
decodecaps.eCodecType = pVideoFormat->codec;
decodecaps.eChromaFormat = pVideoFormat->chroma_format;
decodecaps.nBitDepthMinus8 = pVideoFormat->bit_depth_luma_minus8;
CUDA_DRVAPI_CALL(cuCtxPushCurrent(m_cuContext));
NVDEC_API_CALL(cuvidGetDecoderCaps(&decodecaps));
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
if(!decodecaps.bIsSupported){
NVDEC_THROW_ERROR("Codec not supported on this GPU", CUDA_ERROR_NOT_SUPPORTED);
return nDecodeSurface;
}
if ((pVideoFormat->coded_width > decodecaps.nMaxWidth) ||
(pVideoFormat->coded_height > decodecaps.nMaxHeight)){
std::ostringstream errorString;
errorString << std::endl
<< "Resolution : " << pVideoFormat->coded_width << "x" << pVideoFormat->coded_height << std::endl
<< "Max Supported (wxh) : " << decodecaps.nMaxWidth << "x" << decodecaps.nMaxHeight << std::endl
<< "Resolution not supported on this GPU";
const std::string cErr = errorString.str();
NVDEC_THROW_ERROR(cErr, CUDA_ERROR_NOT_SUPPORTED);
return nDecodeSurface;
}
if ((pVideoFormat->coded_width>>4)*(pVideoFormat->coded_height>>4) > decodecaps.nMaxMBCount){
std::ostringstream errorString;
errorString << std::endl
<< "MBCount : " << (pVideoFormat->coded_width >> 4)*(pVideoFormat->coded_height >> 4) << std::endl
<< "Max Supported mbcnt : " << decodecaps.nMaxMBCount << std::endl
<< "MBCount not supported on this GPU";
const std::string cErr = errorString.str();
NVDEC_THROW_ERROR(cErr, CUDA_ERROR_NOT_SUPPORTED);
return nDecodeSurface;
}
if (m_nWidth && m_nLumaHeight && m_nChromaHeight) {
// cuvidCreateDecoder() has been called before, and now there's possible config change
return ReconfigureDecoder(pVideoFormat);
}
// eCodec has been set in the constructor (for parser). Here it's set again for potential correction
m_eCodec = pVideoFormat->codec;
m_eChromaFormat = pVideoFormat->chroma_format;
m_nBitDepthMinus8 = pVideoFormat->bit_depth_luma_minus8;
m_nBPP = m_nBitDepthMinus8 > 0 ? 2 : 1;
// Set the output surface format same as chroma format
if (m_eChromaFormat == cudaVideoChromaFormat_420 || cudaVideoChromaFormat_Monochrome)
m_eOutputFormat = pVideoFormat->bit_depth_luma_minus8 ? cudaVideoSurfaceFormat_P016 : cudaVideoSurfaceFormat_NV12;
else if (m_eChromaFormat == cudaVideoChromaFormat_444)
m_eOutputFormat = pVideoFormat->bit_depth_luma_minus8 ? cudaVideoSurfaceFormat_YUV444_16Bit : cudaVideoSurfaceFormat_YUV444;
else if (m_eChromaFormat == cudaVideoChromaFormat_422)
m_eOutputFormat = cudaVideoSurfaceFormat_NV12; // no 4:2:2 output format supported yet so make 420 default
// Check if output format supported. If not, check falback options
if (!(decodecaps.nOutputFormatMask & (1 << m_eOutputFormat)))
{
if (decodecaps.nOutputFormatMask & (1 << cudaVideoSurfaceFormat_NV12))
m_eOutputFormat = cudaVideoSurfaceFormat_NV12;
else if (decodecaps.nOutputFormatMask & (1 << cudaVideoSurfaceFormat_P016))
m_eOutputFormat = cudaVideoSurfaceFormat_P016;
else if (decodecaps.nOutputFormatMask & (1 << cudaVideoSurfaceFormat_YUV444))
m_eOutputFormat = cudaVideoSurfaceFormat_YUV444;
else if (decodecaps.nOutputFormatMask & (1 << cudaVideoSurfaceFormat_YUV444_16Bit))
m_eOutputFormat = cudaVideoSurfaceFormat_YUV444_16Bit;
else
NVDEC_THROW_ERROR("No supported output format found", CUDA_ERROR_NOT_SUPPORTED);
}
m_videoFormat = *pVideoFormat;
CUVIDDECODECREATEINFO videoDecodeCreateInfo = { 0 };
videoDecodeCreateInfo.CodecType = pVideoFormat->codec;
videoDecodeCreateInfo.ChromaFormat = pVideoFormat->chroma_format;
videoDecodeCreateInfo.OutputFormat = m_eOutputFormat;
videoDecodeCreateInfo.bitDepthMinus8 = pVideoFormat->bit_depth_luma_minus8;
if (pVideoFormat->progressive_sequence)
videoDecodeCreateInfo.DeinterlaceMode = cudaVideoDeinterlaceMode_Weave;
else
videoDecodeCreateInfo.DeinterlaceMode = cudaVideoDeinterlaceMode_Adaptive;
videoDecodeCreateInfo.ulNumOutputSurfaces = 2;
// With PreferCUVID, JPEG is still decoded by CUDA while video is decoded by NVDEC hardware
videoDecodeCreateInfo.ulCreationFlags = cudaVideoCreate_PreferCUVID;
videoDecodeCreateInfo.ulNumDecodeSurfaces = nDecodeSurface;
videoDecodeCreateInfo.vidLock = m_ctxLock;
videoDecodeCreateInfo.ulWidth = pVideoFormat->coded_width;
videoDecodeCreateInfo.ulHeight = pVideoFormat->coded_height;
// AV1 has max width/height of sequence in sequence header
if (pVideoFormat->codec == cudaVideoCodec_AV1 && pVideoFormat->seqhdr_data_length > 0)
{
// dont overwrite if it is already set from cmdline or reconfig.txt
if (!(m_nMaxWidth > pVideoFormat->coded_width || m_nMaxHeight > pVideoFormat->coded_height))
{
CUVIDEOFORMATEX *vidFormatEx = (CUVIDEOFORMATEX *)pVideoFormat;
m_nMaxWidth = vidFormatEx->av1.max_width;
m_nMaxHeight = vidFormatEx->av1.max_height;
}
}
if (m_nMaxWidth < (int)pVideoFormat->coded_width)
m_nMaxWidth = pVideoFormat->coded_width;
if (m_nMaxHeight < (int)pVideoFormat->coded_height)
m_nMaxHeight = pVideoFormat->coded_height;
videoDecodeCreateInfo.ulMaxWidth = m_nMaxWidth;
videoDecodeCreateInfo.ulMaxHeight = m_nMaxHeight;
if (!(m_cropRect.r && m_cropRect.b) && !(m_resizeDim.w && m_resizeDim.h)) {
m_nWidth = pVideoFormat->display_area.right - pVideoFormat->display_area.left;
m_nLumaHeight = pVideoFormat->display_area.bottom - pVideoFormat->display_area.top;
videoDecodeCreateInfo.ulTargetWidth = pVideoFormat->coded_width;
videoDecodeCreateInfo.ulTargetHeight = pVideoFormat->coded_height;
} else {
if (m_resizeDim.w && m_resizeDim.h) {
videoDecodeCreateInfo.display_area.left = pVideoFormat->display_area.left;
videoDecodeCreateInfo.display_area.top = pVideoFormat->display_area.top;
videoDecodeCreateInfo.display_area.right = pVideoFormat->display_area.right;
videoDecodeCreateInfo.display_area.bottom = pVideoFormat->display_area.bottom;
m_nWidth = m_resizeDim.w;
m_nLumaHeight = m_resizeDim.h;
}
if (m_cropRect.r && m_cropRect.b) {
videoDecodeCreateInfo.display_area.left = m_cropRect.l;
videoDecodeCreateInfo.display_area.top = m_cropRect.t;
videoDecodeCreateInfo.display_area.right = m_cropRect.r;
videoDecodeCreateInfo.display_area.bottom = m_cropRect.b;
m_nWidth = m_cropRect.r - m_cropRect.l;
m_nLumaHeight = m_cropRect.b - m_cropRect.t;
}
videoDecodeCreateInfo.ulTargetWidth = m_nWidth;
videoDecodeCreateInfo.ulTargetHeight = m_nLumaHeight;
}
m_nChromaHeight = (int)(ceil(m_nLumaHeight * GetChromaHeightFactor(m_eOutputFormat)));
m_nNumChromaPlanes = GetChromaPlaneCount(m_eOutputFormat);
m_nSurfaceHeight = videoDecodeCreateInfo.ulTargetHeight;
m_nSurfaceWidth = videoDecodeCreateInfo.ulTargetWidth;
m_displayRect.b = videoDecodeCreateInfo.display_area.bottom;
m_displayRect.t = videoDecodeCreateInfo.display_area.top;
m_displayRect.l = videoDecodeCreateInfo.display_area.left;
m_displayRect.r = videoDecodeCreateInfo.display_area.right;
m_videoInfo << "Video Decoding Params:" << std::endl
<< "\tNum Surfaces : " << videoDecodeCreateInfo.ulNumDecodeSurfaces << std::endl
<< "\tCrop : [" << videoDecodeCreateInfo.display_area.left << ", " << videoDecodeCreateInfo.display_area.top << ", "
<< videoDecodeCreateInfo.display_area.right << ", " << videoDecodeCreateInfo.display_area.bottom << "]" << std::endl
<< "\tResize : " << videoDecodeCreateInfo.ulTargetWidth << "x" << videoDecodeCreateInfo.ulTargetHeight << std::endl
<< "\tDeinterlace : " << std::vector<const char *>{"Weave", "Bob", "Adaptive"}[videoDecodeCreateInfo.DeinterlaceMode]
;
m_videoInfo << std::endl;
CUDA_DRVAPI_CALL(cuCtxPushCurrent(m_cuContext));
NVDEC_API_CALL(cuvidCreateDecoder(&m_hDecoder, &videoDecodeCreateInfo));
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
STOP_TIMER("Session Initialization Time: ");
NvDecoder::addDecoderSessionOverHead(getDecoderSessionID(), elapsedTime);
return nDecodeSurface;
}
int NvDecoder::ReconfigureDecoder(CUVIDEOFORMAT *pVideoFormat)
{
if (pVideoFormat->bit_depth_luma_minus8 != m_videoFormat.bit_depth_luma_minus8 || pVideoFormat->bit_depth_chroma_minus8 != m_videoFormat.bit_depth_chroma_minus8){
NVDEC_THROW_ERROR("Reconfigure Not supported for bit depth change", CUDA_ERROR_NOT_SUPPORTED);
}
if (pVideoFormat->chroma_format != m_videoFormat.chroma_format) {
NVDEC_THROW_ERROR("Reconfigure Not supported for chroma format change", CUDA_ERROR_NOT_SUPPORTED);
}
bool bDecodeResChange = !(pVideoFormat->coded_width == m_videoFormat.coded_width && pVideoFormat->coded_height == m_videoFormat.coded_height);
bool bDisplayRectChange = !(pVideoFormat->display_area.bottom == m_videoFormat.display_area.bottom && pVideoFormat->display_area.top == m_videoFormat.display_area.top \
&& pVideoFormat->display_area.left == m_videoFormat.display_area.left && pVideoFormat->display_area.right == m_videoFormat.display_area.right);
int nDecodeSurface = pVideoFormat->min_num_decode_surfaces;
if ((pVideoFormat->coded_width > m_nMaxWidth) || (pVideoFormat->coded_height > m_nMaxHeight)) {
// For VP9, let driver handle the change if new width/height > maxwidth/maxheight
if ((m_eCodec != cudaVideoCodec_VP9) || m_bReconfigExternal)
{
NVDEC_THROW_ERROR("Reconfigure Not supported when width/height > maxwidth/maxheight", CUDA_ERROR_NOT_SUPPORTED);
}
return 1;
}
if (!bDecodeResChange && !m_bReconfigExtPPChange) {
// if the coded_width/coded_height hasn't changed but display resolution has changed, then need to update width/height for
// correct output without cropping. Example : 1920x1080 vs 1920x1088
if (bDisplayRectChange)
{
m_nWidth = pVideoFormat->display_area.right - pVideoFormat->display_area.left;
m_nLumaHeight = pVideoFormat->display_area.bottom - pVideoFormat->display_area.top;
m_nChromaHeight = (int)ceil(m_nLumaHeight * GetChromaHeightFactor(m_eOutputFormat));
m_nNumChromaPlanes = GetChromaPlaneCount(m_eOutputFormat);
}
// no need for reconfigureDecoder(). Just return
return 1;
}
CUVIDRECONFIGUREDECODERINFO reconfigParams = { 0 };
reconfigParams.ulWidth = m_videoFormat.coded_width = pVideoFormat->coded_width;
reconfigParams.ulHeight = m_videoFormat.coded_height = pVideoFormat->coded_height;
// Dont change display rect and get scaled output from decoder. This will help display app to present apps smoothly
reconfigParams.display_area.bottom = m_displayRect.b;
reconfigParams.display_area.top = m_displayRect.t;
reconfigParams.display_area.left = m_displayRect.l;
reconfigParams.display_area.right = m_displayRect.r;
reconfigParams.ulTargetWidth = m_nSurfaceWidth;
reconfigParams.ulTargetHeight = m_nSurfaceHeight;
// If external reconfigure is called along with resolution change even if post processing params is not changed,
// do full reconfigure params update
if ((m_bReconfigExternal && bDecodeResChange) || m_bReconfigExtPPChange) {
// update display rect and target resolution if requested explicitely
m_bReconfigExternal = false;
m_bReconfigExtPPChange = false;
m_videoFormat = *pVideoFormat;
if (!(m_cropRect.r && m_cropRect.b) && !(m_resizeDim.w && m_resizeDim.h)) {
m_nWidth = pVideoFormat->display_area.right - pVideoFormat->display_area.left;
m_nLumaHeight = pVideoFormat->display_area.bottom - pVideoFormat->display_area.top;
reconfigParams.ulTargetWidth = pVideoFormat->coded_width;
reconfigParams.ulTargetHeight = pVideoFormat->coded_height;
}
else {
if (m_resizeDim.w && m_resizeDim.h) {
reconfigParams.display_area.left = pVideoFormat->display_area.left;
reconfigParams.display_area.top = pVideoFormat->display_area.top;
reconfigParams.display_area.right = pVideoFormat->display_area.right;
reconfigParams.display_area.bottom = pVideoFormat->display_area.bottom;
m_nWidth = m_resizeDim.w;
m_nLumaHeight = m_resizeDim.h;
}
if (m_cropRect.r && m_cropRect.b) {
reconfigParams.display_area.left = m_cropRect.l;
reconfigParams.display_area.top = m_cropRect.t;
reconfigParams.display_area.right = m_cropRect.r;
reconfigParams.display_area.bottom = m_cropRect.b;
m_nWidth = m_cropRect.r - m_cropRect.l;
m_nLumaHeight = m_cropRect.b - m_cropRect.t;
}
reconfigParams.ulTargetWidth = m_nWidth;
reconfigParams.ulTargetHeight = m_nLumaHeight;
}
m_nChromaHeight = (int)ceil(m_nLumaHeight * GetChromaHeightFactor(m_eOutputFormat));
m_nNumChromaPlanes = GetChromaPlaneCount(m_eOutputFormat);
m_nSurfaceHeight = reconfigParams.ulTargetHeight;
m_nSurfaceWidth = reconfigParams.ulTargetWidth;
m_displayRect.b = reconfigParams.display_area.bottom;
m_displayRect.t = reconfigParams.display_area.top;
m_displayRect.l = reconfigParams.display_area.left;
m_displayRect.r = reconfigParams.display_area.right;
}
reconfigParams.ulNumDecodeSurfaces = nDecodeSurface;
START_TIMER
CUDA_DRVAPI_CALL(cuCtxPushCurrent(m_cuContext));
NVDEC_API_CALL(cuvidReconfigureDecoder(m_hDecoder, &reconfigParams));
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
STOP_TIMER("Session Reconfigure Time: ");
return nDecodeSurface;
}
int NvDecoder::setReconfigParams(const Rect *pCropRect, const Dim *pResizeDim)
{
m_bReconfigExternal = true;
m_bReconfigExtPPChange = false;
if (pCropRect)
{
if (!((pCropRect->t == m_cropRect.t) && (pCropRect->l == m_cropRect.l) &&
(pCropRect->b == m_cropRect.b) && (pCropRect->r == m_cropRect.r)))
{
m_bReconfigExtPPChange = true;
m_cropRect = *pCropRect;
}
}
if (pResizeDim)
{
if (!((pResizeDim->w == m_resizeDim.w) && (pResizeDim->h == m_resizeDim.h)))
{
m_bReconfigExtPPChange = true;
m_resizeDim = *pResizeDim;
}
}
// Clear existing output buffers of different size
uint8_t *pFrame = NULL;
while (!m_vpFrame.empty())
{
pFrame = m_vpFrame.back();
m_vpFrame.pop_back();
if (m_bUseDeviceFrame)
{
CUDA_DRVAPI_CALL(cuCtxPushCurrent(m_cuContext));
CUDA_DRVAPI_CALL(cuMemFree((CUdeviceptr)pFrame));
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
}
else
{
delete pFrame;
}
}
return 1;
}
/* Return value from HandlePictureDecode() are interpreted as:
* 0: fail, >=1: succeeded
*/
int NvDecoder::HandlePictureDecode(CUVIDPICPARAMS *pPicParams) {
if (!m_hDecoder)
{
NVDEC_THROW_ERROR("Decoder not initialized.", CUDA_ERROR_NOT_INITIALIZED);
return false;
}
m_nPicNumInDecodeOrder[pPicParams->CurrPicIdx] = m_nDecodePicCnt++;
CUDA_DRVAPI_CALL(cuCtxPushCurrent(m_cuContext));
cuvidDecodePicture(m_hDecoder, pPicParams);
if (m_bForce_zero_latency && ((!pPicParams->field_pic_flag) || (pPicParams->second_field)))
{
CUVIDPARSERDISPINFO dispInfo;
memset(&dispInfo, 0, sizeof(dispInfo));
dispInfo.picture_index = pPicParams->CurrPicIdx;
dispInfo.progressive_frame = !pPicParams->field_pic_flag;
dispInfo.top_field_first = pPicParams->bottom_field_flag ^ 1;
HandlePictureDisplay(&dispInfo);
}
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
return 1;
}
/* Return value from HandlePictureDisplay() are interpreted as:
* 0: fail, >=1: succeeded
*/
int NvDecoder::HandlePictureDisplay(CUVIDPARSERDISPINFO *pDispInfo) {
CUVIDPROCPARAMS videoProcessingParameters = {};
videoProcessingParameters.progressive_frame = pDispInfo->progressive_frame;
videoProcessingParameters.second_field = pDispInfo->repeat_first_field + 1;
videoProcessingParameters.top_field_first = pDispInfo->top_field_first;
videoProcessingParameters.unpaired_field = pDispInfo->repeat_first_field < 0;
videoProcessingParameters.output_stream = m_cuvidStream;
if (m_bExtractSEIMessage)
{
if (m_SEIMessagesDisplayOrder[pDispInfo->picture_index].pSEIData)
{
// Write SEI Message
uint8_t *seiBuffer = (uint8_t *)(m_SEIMessagesDisplayOrder[pDispInfo->picture_index].pSEIData);
uint32_t seiNumMessages = m_SEIMessagesDisplayOrder[pDispInfo->picture_index].sei_message_count;
CUSEIMESSAGE *seiMessagesInfo = m_SEIMessagesDisplayOrder[pDispInfo->picture_index].pSEIMessage;
if (m_fpSEI)
{
for (uint32_t i = 0; i < seiNumMessages; i++)
{
if (m_eCodec == cudaVideoCodec_H264 || cudaVideoCodec_H264_SVC || cudaVideoCodec_H264_MVC || cudaVideoCodec_HEVC)
{
switch (seiMessagesInfo[i].sei_message_type)
{
case SEI_TYPE_TIME_CODE:
{
HEVCSEITIMECODE *timecode = (HEVCSEITIMECODE *)seiBuffer;
fwrite(timecode, sizeof(HEVCSEITIMECODE), 1, m_fpSEI);
}
break;
case SEI_TYPE_USER_DATA_UNREGISTERED:
{
fwrite(seiBuffer, seiMessagesInfo[i].sei_message_size, 1, m_fpSEI);
}
break;
}
}
if (m_eCodec == cudaVideoCodec_AV1)
{
fwrite(seiBuffer, seiMessagesInfo[i].sei_message_size, 1, m_fpSEI);
}
seiBuffer += seiMessagesInfo[i].sei_message_size;
}
}
free(m_SEIMessagesDisplayOrder[pDispInfo->picture_index].pSEIData);
free(m_SEIMessagesDisplayOrder[pDispInfo->picture_index].pSEIMessage);
}
}
CUdeviceptr dpSrcFrame = 0;
unsigned int nSrcPitch = 0;
CUDA_DRVAPI_CALL(cuCtxPushCurrent(m_cuContext));
NVDEC_API_CALL(cuvidMapVideoFrame(m_hDecoder, pDispInfo->picture_index, &dpSrcFrame,
&nSrcPitch, &videoProcessingParameters));
CUVIDGETDECODESTATUS DecodeStatus;
memset(&DecodeStatus, 0, sizeof(DecodeStatus));
CUresult result = cuvidGetDecodeStatus(m_hDecoder, pDispInfo->picture_index, &DecodeStatus);
if (result == CUDA_SUCCESS && (DecodeStatus.decodeStatus == cuvidDecodeStatus_Error || DecodeStatus.decodeStatus == cuvidDecodeStatus_Error_Concealed))
{
printf("Decode Error occurred for picture %d\n", m_nPicNumInDecodeOrder[pDispInfo->picture_index]);
}
uint8_t *pDecodedFrame = nullptr;
{
std::lock_guard<std::mutex> lock(m_mtxVPFrame);
if ((unsigned)++m_nDecodedFrame > m_vpFrame.size())
{
// Not enough frames in stock
m_nFrameAlloc++;
uint8_t *pFrame = NULL;
if (m_bUseDeviceFrame)
{
if (m_bDeviceFramePitched)
{
CUDA_DRVAPI_CALL(cuMemAllocPitch((CUdeviceptr *)&pFrame, &m_nDeviceFramePitch, GetWidth() * m_nBPP, m_nLumaHeight + (m_nChromaHeight * m_nNumChromaPlanes), 16));
}
else
{
CUDA_DRVAPI_CALL(cuMemAlloc((CUdeviceptr *)&pFrame, GetFrameSize()));
}
}
else
{
pFrame = new uint8_t[GetFrameSize()];
}
m_vpFrame.push_back(pFrame);
}
pDecodedFrame = m_vpFrame[m_nDecodedFrame - 1];
}
// Copy luma plane
CUDA_MEMCPY2D m = { 0 };
m.srcMemoryType = CU_MEMORYTYPE_DEVICE;
m.srcDevice = dpSrcFrame;
m.srcPitch = nSrcPitch;
m.dstMemoryType = m_bUseDeviceFrame ? CU_MEMORYTYPE_DEVICE : CU_MEMORYTYPE_HOST;
m.dstDevice = (CUdeviceptr)(m.dstHost = pDecodedFrame);
m.dstPitch = m_nDeviceFramePitch ? m_nDeviceFramePitch : GetWidth() * m_nBPP;
m.WidthInBytes = GetWidth() * m_nBPP;
m.Height = m_nLumaHeight;
CUDA_DRVAPI_CALL(cuMemcpy2DAsync(&m, m_cuvidStream));
// Copy chroma plane
// NVDEC output has luma height aligned by 2. Adjust chroma offset by aligning height
m.srcDevice = (CUdeviceptr)((uint8_t *)dpSrcFrame + m.srcPitch * ((m_nSurfaceHeight + 1) & ~1));
m.dstDevice = (CUdeviceptr)(m.dstHost = pDecodedFrame + m.dstPitch * m_nLumaHeight);
m.Height = m_nChromaHeight;
CUDA_DRVAPI_CALL(cuMemcpy2DAsync(&m, m_cuvidStream));
if (m_nNumChromaPlanes == 2)
{
m.srcDevice = (CUdeviceptr)((uint8_t *)dpSrcFrame + m.srcPitch * ((m_nSurfaceHeight + 1) & ~1) * 2);
m.dstDevice = (CUdeviceptr)(m.dstHost = pDecodedFrame + m.dstPitch * m_nLumaHeight * 2);
m.Height = m_nChromaHeight;
CUDA_DRVAPI_CALL(cuMemcpy2DAsync(&m, m_cuvidStream));
}
CUDA_DRVAPI_CALL(cuStreamSynchronize(m_cuvidStream));
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
if ((int)m_vTimestamp.size() < m_nDecodedFrame) {
m_vTimestamp.resize(m_vpFrame.size());
}
m_vTimestamp[m_nDecodedFrame - 1] = pDispInfo->timestamp;
NVDEC_API_CALL(cuvidUnmapVideoFrame(m_hDecoder, dpSrcFrame));
return 1;
}
int NvDecoder::GetSEIMessage(CUVIDSEIMESSAGEINFO *pSEIMessageInfo)
{
uint32_t seiNumMessages = pSEIMessageInfo->sei_message_count;
CUSEIMESSAGE *seiMessagesInfo = pSEIMessageInfo->pSEIMessage;
size_t totalSEIBufferSize = 0;
if ((pSEIMessageInfo->picIdx < 0) || (pSEIMessageInfo->picIdx >= MAX_FRM_CNT))
{
printf("Invalid picture index (%d)\n", pSEIMessageInfo->picIdx);
return 0;
}
for (uint32_t i = 0; i < seiNumMessages; i++)
{
totalSEIBufferSize += seiMessagesInfo[i].sei_message_size;
}
if (!m_pCurrSEIMessage)
{
printf("Out of Memory, Allocation failed for m_pCurrSEIMessage\n");
return 0;
}
m_pCurrSEIMessage->pSEIData = malloc(totalSEIBufferSize);
if (!m_pCurrSEIMessage->pSEIData)
{
printf("Out of Memory, Allocation failed for SEI Buffer\n");
return 0;
}
memcpy(m_pCurrSEIMessage->pSEIData, pSEIMessageInfo->pSEIData, totalSEIBufferSize);
m_pCurrSEIMessage->pSEIMessage = (CUSEIMESSAGE *)malloc(sizeof(CUSEIMESSAGE) * seiNumMessages);
if (!m_pCurrSEIMessage->pSEIMessage)
{
free(m_pCurrSEIMessage->pSEIData);
m_pCurrSEIMessage->pSEIData = NULL;
return 0;
}
memcpy(m_pCurrSEIMessage->pSEIMessage, pSEIMessageInfo->pSEIMessage, sizeof(CUSEIMESSAGE) * seiNumMessages);
m_pCurrSEIMessage->sei_message_count = pSEIMessageInfo->sei_message_count;
m_SEIMessagesDisplayOrder[pSEIMessageInfo->picIdx] = *m_pCurrSEIMessage;
return 1;
}
NvDecoder::NvDecoder(CUcontext cuContext, bool bUseDeviceFrame, cudaVideoCodec eCodec, bool bLowLatency,
bool bDeviceFramePitched, const Rect *pCropRect, const Dim *pResizeDim, bool extract_user_SEI_Message,
int maxWidth, int maxHeight, unsigned int clkRate, bool force_zero_latency) :
m_cuContext(cuContext), m_bUseDeviceFrame(bUseDeviceFrame), m_eCodec(eCodec), m_bDeviceFramePitched(bDeviceFramePitched),
m_bExtractSEIMessage(extract_user_SEI_Message), m_nMaxWidth (maxWidth), m_nMaxHeight(maxHeight),
m_bForce_zero_latency(force_zero_latency)
{
if (pCropRect) m_cropRect = *pCropRect;
if (pResizeDim) m_resizeDim = *pResizeDim;
NVDEC_API_CALL(cuvidCtxLockCreate(&m_ctxLock, cuContext));
ck(cuStreamCreate(&m_cuvidStream, CU_STREAM_DEFAULT));
decoderSessionID = 0;
if (m_bExtractSEIMessage)
{
m_fpSEI = fopen("sei_message.txt", "wb");
m_pCurrSEIMessage = new CUVIDSEIMESSAGEINFO;
memset(&m_SEIMessagesDisplayOrder, 0, sizeof(m_SEIMessagesDisplayOrder));
}
CUVIDPARSERPARAMS videoParserParameters = {};
videoParserParameters.CodecType = eCodec;
videoParserParameters.ulMaxNumDecodeSurfaces = 1;
videoParserParameters.ulClockRate = clkRate;
videoParserParameters.ulMaxDisplayDelay = bLowLatency ? 0 : 1;
videoParserParameters.pUserData = this;
videoParserParameters.pfnSequenceCallback = HandleVideoSequenceProc;
videoParserParameters.pfnDecodePicture = HandlePictureDecodeProc;
videoParserParameters.pfnDisplayPicture = m_bForce_zero_latency ? NULL : HandlePictureDisplayProc;
videoParserParameters.pfnGetOperatingPoint = HandleOperatingPointProc;
videoParserParameters.pfnGetSEIMsg = m_bExtractSEIMessage ? HandleSEIMessagesProc : NULL;
NVDEC_API_CALL(cuvidCreateVideoParser(&m_hParser, &videoParserParameters));
}
NvDecoder::~NvDecoder() {
START_TIMER
if (m_pCurrSEIMessage) {
delete m_pCurrSEIMessage;
m_pCurrSEIMessage = NULL;
}
if (m_fpSEI) {
fclose(m_fpSEI);
m_fpSEI = NULL;
}
if (m_hParser) {
cuvidDestroyVideoParser(m_hParser);
}
cuCtxPushCurrent(m_cuContext);
if (m_hDecoder) {
cuvidDestroyDecoder(m_hDecoder);
}
std::lock_guard<std::mutex> lock(m_mtxVPFrame);
for (uint8_t *pFrame : m_vpFrame)
{
if (m_bUseDeviceFrame)
{
cuMemFree((CUdeviceptr)pFrame);
}
else
{
delete[] pFrame;
}
}
cuCtxPopCurrent(NULL);
cuvidCtxLockDestroy(m_ctxLock);
STOP_TIMER("Session Deinitialization Time: ");
NvDecoder::addDecoderSessionOverHead(getDecoderSessionID(), elapsedTime);
}
int NvDecoder::Decode(const uint8_t *pData, int nSize, int nFlags, int64_t nTimestamp)
{
m_nDecodedFrame = 0;
m_nDecodedFrameReturned = 0;
CUVIDSOURCEDATAPACKET packet = { 0 };
packet.payload = pData;
packet.payload_size = nSize;
packet.flags = nFlags | CUVID_PKT_TIMESTAMP;
packet.timestamp = nTimestamp;
if (!pData || nSize == 0) {
packet.flags |= CUVID_PKT_ENDOFSTREAM;
}
NVDEC_API_CALL(cuvidParseVideoData(m_hParser, &packet));
return m_nDecodedFrame;
}
uint8_t* NvDecoder::GetFrame(int64_t* pTimestamp)
{
if (m_nDecodedFrame > 0)
{
std::lock_guard<std::mutex> lock(m_mtxVPFrame);
m_nDecodedFrame--;
if (pTimestamp)
*pTimestamp = m_vTimestamp[m_nDecodedFrameReturned];
return m_vpFrame[m_nDecodedFrameReturned++];
}
return NULL;
}
uint8_t* NvDecoder::GetLockedFrame(int64_t* pTimestamp)
{
uint8_t *pFrame;
uint64_t timestamp;
if (m_nDecodedFrame > 0) {
std::lock_guard<std::mutex> lock(m_mtxVPFrame);
m_nDecodedFrame--;
pFrame = m_vpFrame[0];
m_vpFrame.erase(m_vpFrame.begin(), m_vpFrame.begin() + 1);
timestamp = m_vTimestamp[0];
m_vTimestamp.erase(m_vTimestamp.begin(), m_vTimestamp.begin() + 1);
if (pTimestamp)
*pTimestamp = timestamp;
return pFrame;
}
return NULL;
}
void NvDecoder::UnlockFrame(uint8_t **pFrame)
{
std::lock_guard<std::mutex> lock(m_mtxVPFrame);
m_vpFrame.insert(m_vpFrame.end(), &pFrame[0], &pFrame[1]);
// add a dummy entry for timestamp
uint64_t timestamp[2] = {0};
m_vTimestamp.insert(m_vTimestamp.end(), &timestamp[0], &timestamp[1]);
}
@@ -0,0 +1,362 @@
/*
* Copyright 2017-2022 NVIDIA Corporation. All rights reserved.
*
* Please refer to the NVIDIA end user license agreement (EULA) associated
* with this source code for terms and conditions that govern your use of
* this software. Any use, reproduction, disclosure, or distribution of
* this software and related documentation outside the terms of the EULA
* is strictly prohibited.
*
*/
#pragma once
#include <assert.h>
#include <stdint.h>
#include <mutex>
#include <vector>
#include <string>
#include <iostream>
#include <sstream>
#include <string.h>
#include <map>
#include "nvcuvid.h"
#include "Utils/NvCodecUtils.h"
#define MAX_FRM_CNT 32
typedef enum{
SEI_TYPE_TIME_CODE = 136,
SEI_TYPE_USER_DATA_UNREGISTERED = 5
}SEI_H264_HEVC_PAYLOAD_TYPE;
/**
* @brief Exception class for error reporting from the decode API.
*/
class NVDECException : public std::exception
{
public:
NVDECException(const std::string& errorStr, const CUresult errorCode)
: m_errorString(errorStr), m_errorCode(errorCode) {}
virtual ~NVDECException() throw() {}
virtual const char* what() const throw() { return m_errorString.c_str(); }
CUresult getErrorCode() const { return m_errorCode; }
const std::string& getErrorString() const { return m_errorString; }
static NVDECException makeNVDECException(const std::string& errorStr, const CUresult errorCode,
const std::string& functionName, const std::string& fileName, int lineNo);
private:
std::string m_errorString;
CUresult m_errorCode;
};
inline NVDECException NVDECException::makeNVDECException(const std::string& errorStr, const CUresult errorCode, const std::string& functionName,
const std::string& fileName, int lineNo)
{
std::ostringstream errorLog;
errorLog << functionName << " : " << errorStr << " at " << fileName << ":" << lineNo << std::endl;
NVDECException exception(errorLog.str(), errorCode);
return exception;
}
#define NVDEC_THROW_ERROR( errorStr, errorCode ) \
do \
{ \
throw NVDECException::makeNVDECException(errorStr, errorCode, __FUNCTION__, __FILE__, __LINE__); \
} while (0)
#define NVDEC_API_CALL( cuvidAPI ) \
do \
{ \
CUresult errorCode = cuvidAPI; \
if( errorCode != CUDA_SUCCESS) \
{ \
std::ostringstream errorLog; \
errorLog << #cuvidAPI << " returned error " << errorCode; \
throw NVDECException::makeNVDECException(errorLog.str(), errorCode, __FUNCTION__, __FILE__, __LINE__); \
} \
} while (0)
struct Rect {
int l, t, r, b;
};
struct Dim {
int w, h;
};
/**
* @brief Base class for decoder interface.
*/
class NvDecoder {
public:
/**
* @brief This function is used to initialize the decoder session.
* Application must call this function to initialize the decoder, before
* starting to decode any frames.
*/
NvDecoder(CUcontext cuContext, bool bUseDeviceFrame, cudaVideoCodec eCodec, bool bLowLatency = false,
bool bDeviceFramePitched = false, const Rect *pCropRect = NULL, const Dim *pResizeDim = NULL,
bool extract_user_SEI_Message = false, int maxWidth = 0, int maxHeight = 0, unsigned int clkRate = 1000,
bool force_zero_latency = false);
~NvDecoder();
/**
* @brief This function is used to get the current CUDA context.
*/
CUcontext GetContext() { return m_cuContext; }
/**
* @brief This function is used to get the output frame width.
* NV12/P016 output format width is 2 byte aligned because of U and V interleave
*/
int GetWidth() { assert(m_nWidth); return (m_eOutputFormat == cudaVideoSurfaceFormat_NV12 || m_eOutputFormat == cudaVideoSurfaceFormat_P016)
? (m_nWidth + 1) & ~1 : m_nWidth; }
/**
* @brief This function is used to get the actual decode width
*/
int GetDecodeWidth() { assert(m_nWidth); return m_nWidth; }
/**
* @brief This function is used to get the output frame height (Luma height).
*/
int GetHeight() { assert(m_nLumaHeight); return m_nLumaHeight; }
/**
* @brief This function is used to get the current chroma height.
*/
int GetChromaHeight() { assert(m_nChromaHeight); return m_nChromaHeight; }
/**
* @brief This function is used to get the number of chroma planes.
*/
int GetNumChromaPlanes() { assert(m_nNumChromaPlanes); return m_nNumChromaPlanes; }
/**
* @brief This function is used to get the current frame size based on pixel format.
*/
int GetFrameSize() { assert(m_nWidth); return GetWidth() * (m_nLumaHeight + (m_nChromaHeight * m_nNumChromaPlanes)) * m_nBPP; }
/**
* @brief This function is used to get the current frame Luma plane size.
*/
int GetLumaPlaneSize() { assert(m_nWidth); return GetWidth() * m_nLumaHeight * m_nBPP; }
/**
* @brief This function is used to get the current frame chroma plane size.
*/
int GetChromaPlaneSize() { assert(m_nWidth); return GetWidth() * (m_nChromaHeight * m_nNumChromaPlanes) * m_nBPP; }
/**
* @brief This function is used to get the pitch of the device buffer holding the decoded frame.
*/
int GetDeviceFramePitch() { assert(m_nWidth); return m_nDeviceFramePitch ? (int)m_nDeviceFramePitch : GetWidth() * m_nBPP; }
/**
* @brief This function is used to get the bit depth associated with the pixel format.
*/
int GetBitDepth() { assert(m_nWidth); return m_nBitDepthMinus8 + 8; }
/**
* @brief This function is used to get the bytes used per pixel.
*/
int GetBPP() { assert(m_nWidth); return m_nBPP; }
/**
* @brief This function is used to get the YUV chroma format
*/
cudaVideoSurfaceFormat GetOutputFormat() { return m_eOutputFormat; }
/**
* @brief This function is used to get information about the video stream (codec, display parameters etc)
*/
CUVIDEOFORMAT GetVideoFormatInfo() { assert(m_nWidth); return m_videoFormat; }
/**
* @brief This function is used to get codec string from codec id
*/
const char *GetCodecString(cudaVideoCodec eCodec);
/**
* @brief This function is used to print information about the video stream
*/
std::string GetVideoInfo() const { return m_videoInfo.str(); }
/**
* @brief This function decodes a frame and returns the number of frames that are available for
* display. All frames that are available for display should be read before making a subsequent decode call.
* @param pData - pointer to the data buffer that is to be decoded
* @param nSize - size of the data buffer in bytes
* @param nFlags - CUvideopacketflags for setting decode options
* @param nTimestamp - presentation timestamp
*/
int Decode(const uint8_t *pData, int nSize, int nFlags = 0, int64_t nTimestamp = 0);
/**
* @brief This function returns a decoded frame and timestamp. This function should be called in a loop for
* fetching all the frames that are available for display.
*/
uint8_t* GetFrame(int64_t* pTimestamp = nullptr);
/**
* @brief This function decodes a frame and returns the locked frame buffers
* This makes the buffers available for use by the application without the buffers
* getting overwritten, even if subsequent decode calls are made. The frame buffers
* remain locked, until UnlockFrame() is called
*/
uint8_t* GetLockedFrame(int64_t* pTimestamp = nullptr);
/**
* @brief This function unlocks the frame buffer and makes the frame buffers available for write again
* @param ppFrame - pointer to array of frames that are to be unlocked
* @param nFrame - number of frames to be unlocked
*/
void UnlockFrame(uint8_t **pFrame);
/**
* @brief This function allows app to set decoder reconfig params
* @param pCropRect - cropping rectangle coordinates
* @param pResizeDim - width and height of resized output
*/
int setReconfigParams(const Rect * pCropRect, const Dim * pResizeDim);
/**
* @brief This function allows app to set operating point for AV1 SVC clips
* @param opPoint - operating point of an AV1 scalable bitstream
* @param bDispAllLayers - Output all decoded frames of an AV1 scalable bitstream
*/
void SetOperatingPoint(const uint32_t opPoint, const bool bDispAllLayers) { m_nOperatingPoint = opPoint; m_bDispAllLayers = bDispAllLayers; }
// start a timer
void startTimer() { m_stDecode_time.Start(); }
// stop the timer
double stopTimer() { return m_stDecode_time.Stop(); }
void setDecoderSessionID(int sessionID) { decoderSessionID = sessionID; }
int getDecoderSessionID() { return decoderSessionID; }
// Session overhead refers to decoder initialization and deinitialization time
static void addDecoderSessionOverHead(int sessionID, int64_t duration) { sessionOverHead[sessionID] += duration; }
static int64_t getDecoderSessionOverHead(int sessionID) { return sessionOverHead[sessionID]; }
private:
int decoderSessionID; // Decoder session identifier. Used to gather session level stats.
static std::map<int, int64_t> sessionOverHead; // Records session overhead of initialization+deinitialization time. Format is (thread id, duration)
/**
* @brief Callback function to be registered for getting a callback when decoding of sequence starts
*/
static int CUDAAPI HandleVideoSequenceProc(void *pUserData, CUVIDEOFORMAT *pVideoFormat) { return ((NvDecoder *)pUserData)->HandleVideoSequence(pVideoFormat); }
/**
* @brief Callback function to be registered for getting a callback when a decoded frame is ready to be decoded
*/
static int CUDAAPI HandlePictureDecodeProc(void *pUserData, CUVIDPICPARAMS *pPicParams) { return ((NvDecoder *)pUserData)->HandlePictureDecode(pPicParams); }
/**
* @brief Callback function to be registered for getting a callback when a decoded frame is available for display
*/
static int CUDAAPI HandlePictureDisplayProc(void *pUserData, CUVIDPARSERDISPINFO *pDispInfo) { return ((NvDecoder *)pUserData)->HandlePictureDisplay(pDispInfo); }
/**
* @brief Callback function to be registered for getting a callback to get operating point when AV1 SVC sequence header start.
*/
static int CUDAAPI HandleOperatingPointProc(void *pUserData, CUVIDOPERATINGPOINTINFO *pOPInfo) { return ((NvDecoder *)pUserData)->GetOperatingPoint(pOPInfo); }
/**
* @brief Callback function to be registered for getting a callback when all the unregistered user SEI Messages are parsed for a frame.
*/
static int CUDAAPI HandleSEIMessagesProc(void *pUserData, CUVIDSEIMESSAGEINFO *pSEIMessageInfo) { return ((NvDecoder *)pUserData)->GetSEIMessage(pSEIMessageInfo); }
/**
* @brief This function gets called when a sequence is ready to be decoded. The function also gets called
when there is format change
*/
int HandleVideoSequence(CUVIDEOFORMAT *pVideoFormat);
/**
* @brief This function gets called when a picture is ready to be decoded. cuvidDecodePicture is called from this function
* to decode the picture
*/
int HandlePictureDecode(CUVIDPICPARAMS *pPicParams);
/**
* @brief This function gets called after a picture is decoded and available for display. Frames are fetched and stored in
internal buffer
*/
int HandlePictureDisplay(CUVIDPARSERDISPINFO *pDispInfo);
/**
* @brief This function gets called when AV1 sequence encounter more than one operating points
*/
int GetOperatingPoint(CUVIDOPERATINGPOINTINFO *pOPInfo);
/**
* @brief This function gets called when all unregistered user SEI messages are parsed for a frame
*/
int GetSEIMessage(CUVIDSEIMESSAGEINFO *pSEIMessageInfo);
/**
* @brief This function reconfigure decoder if there is a change in sequence params.
*/
int ReconfigureDecoder(CUVIDEOFORMAT *pVideoFormat);
private:
CUcontext m_cuContext = NULL;
CUvideoctxlock m_ctxLock;
CUvideoparser m_hParser = NULL;
CUvideodecoder m_hDecoder = NULL;
bool m_bUseDeviceFrame;
// dimension of the output
unsigned int m_nWidth = 0, m_nLumaHeight = 0, m_nChromaHeight = 0;
unsigned int m_nNumChromaPlanes = 0;
// height of the mapped surface
int m_nSurfaceHeight = 0;
int m_nSurfaceWidth = 0;
cudaVideoCodec m_eCodec = cudaVideoCodec_NumCodecs;
cudaVideoChromaFormat m_eChromaFormat = cudaVideoChromaFormat_420;
cudaVideoSurfaceFormat m_eOutputFormat = cudaVideoSurfaceFormat_NV12;
int m_nBitDepthMinus8 = 0;
int m_nBPP = 1;
CUVIDEOFORMAT m_videoFormat = {};
Rect m_displayRect = {};
// stock of frames
std::vector<uint8_t *> m_vpFrame;
// timestamps of decoded frames
std::vector<int64_t> m_vTimestamp;
int m_nDecodedFrame = 0, m_nDecodedFrameReturned = 0;
int m_nDecodePicCnt = 0, m_nPicNumInDecodeOrder[MAX_FRM_CNT];
CUVIDSEIMESSAGEINFO *m_pCurrSEIMessage = NULL;
CUVIDSEIMESSAGEINFO m_SEIMessagesDisplayOrder[MAX_FRM_CNT];
FILE *m_fpSEI = NULL;
bool m_bEndDecodeDone = false;
std::mutex m_mtxVPFrame;
int m_nFrameAlloc = 0;
CUstream m_cuvidStream = 0;
bool m_bDeviceFramePitched = false;
size_t m_nDeviceFramePitch = 0;
Rect m_cropRect = {};
Dim m_resizeDim = {};
std::ostringstream m_videoInfo;
unsigned int m_nMaxWidth = 0, m_nMaxHeight = 0;
bool m_bReconfigExternal = false;
bool m_bReconfigExtPPChange = false;
StopWatch m_stDecode_time;
unsigned int m_nOperatingPoint = 0;
bool m_bDispAllLayers = false;
// In H.264, there is an inherent display latency for video contents
// which do not have num_reorder_frames=0 in the VUI. This applies to
// All-Intra and IPPP sequences as well. If the user wants zero display
// latency for All-Intra and IPPP sequences, the below flag will enable
// the display callback immediately after the decode callback.
bool m_bForce_zero_latency = false;
bool m_bExtractSEIMessage = false;
};
@@ -0,0 +1,530 @@
/*
* Copyright 2017-2022 NVIDIA Corporation. All rights reserved.
*
* Please refer to the NVIDIA end user license agreement (EULA) associated
* with this source code for terms and conditions that govern your use of
* this software. Any use, reproduction, disclosure, or distribution of
* this software and related documentation outside the terms of the EULA
* is strictly prohibited.
*
*/
#pragma once
#include <stdint.h>
#include <string.h>
#include <iostream>
#include <mutex>
#include <sstream>
#include <string>
#include <vector>
#include "Utils/NvCodecUtils.h"
#include "nvEncodeAPI.h"
/**
* @brief Exception class for error reporting from NvEncodeAPI calls.
*/
class NVENCException : public std::exception {
public:
NVENCException(const std::string& errorStr, const NVENCSTATUS errorCode)
: m_errorString(errorStr), m_errorCode(errorCode) {}
virtual ~NVENCException() throw() {}
virtual const char* what() const throw() { return m_errorString.c_str(); }
NVENCSTATUS getErrorCode() const { return m_errorCode; }
const std::string& getErrorString() const { return m_errorString; }
static NVENCException makeNVENCException(const std::string& errorStr,
const NVENCSTATUS errorCode,
const std::string& functionName,
const std::string& fileName,
int lineNo);
private:
std::string m_errorString;
NVENCSTATUS m_errorCode;
};
inline NVENCException NVENCException::makeNVENCException(
const std::string& errorStr,
const NVENCSTATUS errorCode,
const std::string& functionName,
const std::string& fileName,
int lineNo) {
std::ostringstream errorLog;
errorLog << functionName << " : " << errorStr << " at " << fileName << ":"
<< lineNo << std::endl;
NVENCException exception(errorLog.str(), errorCode);
return exception;
}
#define NVENC_THROW_ERROR(errorStr, errorCode) \
do { \
throw NVENCException::makeNVENCException( \
errorStr, errorCode, __FUNCTION__, __FILE__, __LINE__); \
} while (0)
#define NVENC_API_CALL(nvencAPI) \
do { \
NVENCSTATUS errorCode = nvencAPI; \
if (errorCode != NV_ENC_SUCCESS) { \
std::ostringstream errorLog; \
errorLog << #nvencAPI << " returned error " << errorCode; \
throw NVENCException::makeNVENCException( \
errorLog.str(), errorCode, __FUNCTION__, __FILE__, __LINE__); \
} \
} while (0)
struct NvEncInputFrame {
void* inputPtr = nullptr;
uint32_t chromaOffsets[2];
uint32_t numChromaPlanes;
uint32_t pitch;
uint32_t chromaPitch;
NV_ENC_BUFFER_FORMAT bufferFormat;
NV_ENC_INPUT_RESOURCE_TYPE resourceType;
};
struct NvEncExternalInputFrame {
void* resource = nullptr;
NV_ENC_INPUT_RESOURCE_TYPE resourceType = NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR;
int width = 0;
int height = 0;
int pitch = 0;
uint32_t subResourceIndex = 0;
NV_ENC_BUFFER_FORMAT bufferFormat = NV_ENC_BUFFER_FORMAT_NV12;
NV_ENC_FENCE_POINT_D3D12* inputFencePoint = nullptr;
};
/**
* @brief Shared base class for different encoder interfaces.
*/
class NvEncoder {
public:
/**
* @brief This function is used to initialize the encoder session.
* Application must call this function to initialize the encoder, before
* starting to encode any frames.
*/
virtual void CreateEncoder(const NV_ENC_INITIALIZE_PARAMS* pEncodeParams);
/**
* @brief This function is used to destroy the encoder session.
* Application must call this function to destroy the encoder session and
* clean up any allocated resources. The application must call EndEncode()
* function to get any queued encoded frames before calling DestroyEncoder().
*/
virtual void DestroyEncoder();
/**
* @brief This function is used to reconfigure an existing encoder session.
* Application can use this function to dynamically change the bitrate,
* resolution and other QOS parameters. If the application changes the
* resolution, it must set NV_ENC_RECONFIGURE_PARAMS::forceIDR.
*/
bool Reconfigure(const NV_ENC_RECONFIGURE_PARAMS* pReconfigureParams);
/**
* @brief This function is used to get the next available input buffer.
* Applications must call this function to obtain a pointer to the next
* input buffer. The application must copy the uncompressed data to the
* input buffer and then call EncodeFrame() function to encode it.
*/
const NvEncInputFrame* GetNextInputFrame();
/**
* @brief This function is used to encode a frame.
* Applications must call EncodeFrame() function to encode the uncompressed
* data, which has been copied to an input buffer obtained from the
* GetNextInputFrame() function.
*/
virtual void EncodeFrame(std::vector<std::vector<uint8_t>>& vPacket,
NV_ENC_PIC_PARAMS* pPicParams = nullptr);
/**
* @brief Encode one externally-owned CUDA, D3D, or OpenGL resource.
*
* The caller retains ownership of the resource. This method temporarily
* registers and maps it with NVENC, submits the frame, drains the matching
* bitstream packet, then unregisters it. It is intended for zero-copy
* capture paths where the frame already lives in GPU memory.
*/
virtual void EncodeExternalFrame(
const NvEncExternalInputFrame& inputFrame,
std::vector<std::vector<uint8_t>>& vPacket,
NV_ENC_PIC_PARAMS* pPicParams = nullptr);
/**
* @brief This function to flush the encoder queue.
* The encoder might be queuing frames for B picture encoding or lookahead;
* the application must call EndEncode() to get all the queued encoded frames
* from the encoder. The application must call this function before
* destroying an encoder session.
*/
virtual void EndEncode(std::vector<std::vector<uint8_t>>& vPacket);
/**
* @brief This function is used to query hardware encoder capabilities.
* Applications can call this function to query capabilities like maximum
* encode dimensions, support for lookahead or the ME-only mode etc.
*/
int GetCapabilityValue(GUID guidCodec, NV_ENC_CAPS capsToQuery);
/**
* @brief This function is used to get the current device on which encoder
* is running.
*/
void* GetDevice() const { return m_pDevice; }
/**
* @brief This function is used to get the current device type which encoder
* is running.
*/
NV_ENC_DEVICE_TYPE GetDeviceType() const { return m_eDeviceType; }
/**
* @brief This function is used to get the current encode width.
* The encode width can be modified by Reconfigure() function.
*/
int GetEncodeWidth() const { return m_nWidth; }
/**
* @brief This function is used to get the current encode height.
* The encode height can be modified by Reconfigure() function.
*/
int GetEncodeHeight() const { return m_nHeight; }
/**
* @brief This function is used to get the current frame size based on
* pixel format.
*/
int GetFrameSize() const;
/**
* @brief This function is used to initialize config parameters based on
* given codec and preset guids.
* The application can call this function to get the default configuration
* for a certain preset. The application can either use these parameters
* directly or override them with application-specific settings before
* using them in CreateEncoder() function.
*/
void CreateDefaultEncoderParams(
NV_ENC_INITIALIZE_PARAMS* pIntializeParams,
GUID codecGuid,
GUID presetGuid,
NV_ENC_TUNING_INFO tuningInfo = NV_ENC_TUNING_INFO_UNDEFINED);
/**
* @brief This function is used to get the current initialization
* parameters, which had been used to configure the encoder session. The
* initialization parameters are modified if the application calls
* Reconfigure() function.
*/
void GetInitializeParams(NV_ENC_INITIALIZE_PARAMS* pInitializeParams);
/**
* @brief This function is used to run motion estimation
* This is used to run motion estimation on a a pair of frames. The
* application must copy the reference frame data to the buffer obtained
* by calling GetNextReferenceFrame(), and copy the input frame data to
* the buffer obtained by calling GetNextInputFrame() before calling the
* RunMotionEstimation() function.
*/
void RunMotionEstimation(std::vector<uint8_t>& mvData);
/**
* @brief This function is used to get an available reference frame.
* Application must call this function to get a pointer to reference buffer,
* to be used in the subsequent RunMotionEstimation() function.
*/
const NvEncInputFrame* GetNextReferenceFrame();
/**
* @brief This function is used to get sequence and picture parameter
* headers. Application can call this function after encoder is initialized to
* get SPS and PPS nalus for the current encoder instance. The sequence header
* data might change when application calls Reconfigure() function.
*/
void GetSequenceParams(std::vector<uint8_t>& seqParams);
/**
* @brief NvEncoder class virtual destructor.
*/
virtual ~NvEncoder();
public:
/**
* @brief This a static function to get chroma offsets for YUV planar
* formats.
*/
static void GetChromaSubPlaneOffsets(const NV_ENC_BUFFER_FORMAT bufferFormat,
const uint32_t pitch,
const uint32_t height,
std::vector<uint32_t>& chromaOffsets);
/**
* @brief This a static function to get the chroma plane pitch for YUV planar
* formats.
*/
static uint32_t GetChromaPitch(const NV_ENC_BUFFER_FORMAT bufferFormat,
const uint32_t lumaPitch);
/**
* @brief This a static function to get the number of chroma planes for YUV
* planar formats.
*/
static uint32_t GetNumChromaPlanes(const NV_ENC_BUFFER_FORMAT bufferFormat);
/**
* @brief This a static function to get the chroma plane width in bytes for
* YUV planar formats.
*/
static uint32_t GetChromaWidthInBytes(const NV_ENC_BUFFER_FORMAT bufferFormat,
const uint32_t lumaWidth);
/**
* @brief This a static function to get the chroma planes height in bytes for
* YUV planar formats.
*/
static uint32_t GetChromaHeight(const NV_ENC_BUFFER_FORMAT bufferFormat,
const uint32_t lumaHeight);
/**
* @brief This a static function to get the width in bytes for the frame.
* For YUV planar format this is the width in bytes of the luma plane.
*/
static uint32_t GetWidthInBytes(const NV_ENC_BUFFER_FORMAT bufferFormat,
const uint32_t width);
/**
* @brief This function returns the number of allocated buffers.
*/
uint32_t GetEncoderBufferCount() const { return m_nEncoderBuffer; }
protected:
/**
* @brief NvEncoder class constructor.
* NvEncoder class constructor cannot be called directly by the application.
*/
NvEncoder(NV_ENC_DEVICE_TYPE eDeviceType,
void* pDevice,
uint32_t nWidth,
uint32_t nHeight,
NV_ENC_BUFFER_FORMAT eBufferFormat,
uint32_t nOutputDelay,
bool bMotionEstimationOnly,
bool bOutputInVideoMemory = false,
bool bDX12Encode = false,
bool bUseIVFContainer = true);
/**
* @brief This function is used to check if hardware encoder is properly
* initialized.
*/
bool IsHWEncoderInitialized() const {
return m_hEncoder != NULL && m_bEncoderInitialized;
}
/**
* @brief This function is used to register CUDA, D3D or OpenGL input buffers
* with NvEncodeAPI. This is non public function and is called by derived
* class for allocating and registering input buffers.
*/
void RegisterInputResources(std::vector<void*> inputframes,
NV_ENC_INPUT_RESOURCE_TYPE eResourceType,
int width,
int height,
int pitch,
NV_ENC_BUFFER_FORMAT bufferFormat,
bool bReferenceFrame = false);
/**
* @brief This function is used to unregister resources which had been
* previously registered for encoding using RegisterInputResources() function.
*/
void UnregisterInputResources();
/**
* @brief This function is used to register CUDA, D3D or OpenGL input or
* output buffers with NvEncodeAPI.
*/
NV_ENC_REGISTERED_PTR RegisterResource(
void* pBuffer,
NV_ENC_INPUT_RESOURCE_TYPE eResourceType,
int width,
int height,
int pitch,
NV_ENC_BUFFER_FORMAT bufferFormat,
NV_ENC_BUFFER_USAGE bufferUsage = NV_ENC_INPUT_IMAGE,
NV_ENC_FENCE_POINT_D3D12* pInputFencePoint = NULL,
uint32_t subResourceIndex = 0);
/**
* @brief This function returns maximum width used to open the encoder
* session. All encode input buffers are allocated using maximum dimensions.
*/
uint32_t GetMaxEncodeWidth() const { return m_nMaxEncodeWidth; }
/**
* @brief This function returns maximum height used to open the encoder
* session. All encode input buffers are allocated using maximum dimensions.
*/
uint32_t GetMaxEncodeHeight() const { return m_nMaxEncodeHeight; }
/**
* @brief This function returns the completion event.
*/
void* GetCompletionEvent(uint32_t eventIdx) {
return (m_vpCompletionEvent.size() == m_nEncoderBuffer)
? m_vpCompletionEvent[eventIdx]
: nullptr;
}
/**
* @brief This function returns the current pixel format.
*/
NV_ENC_BUFFER_FORMAT GetPixelFormat() const { return m_eBufferFormat; }
/**
* @brief This function is used to submit the encode commands to the
* NVENC hardware.
*/
NVENCSTATUS DoEncode(NV_ENC_INPUT_PTR inputBuffer,
NV_ENC_OUTPUT_PTR outputBuffer,
NV_ENC_PIC_PARAMS* pPicParams);
/**
* @brief This function is used to submit the encode commands to the
* NVENC hardware for ME only mode.
*/
NVENCSTATUS DoMotionEstimation(NV_ENC_INPUT_PTR inputBuffer,
NV_ENC_INPUT_PTR inputBufferForReference,
NV_ENC_OUTPUT_PTR outputBuffer);
/**
* @brief This function is used to map the input buffers to NvEncodeAPI.
*/
void MapResources(uint32_t bfrIdx);
/**
* @brief This function is used to wait for completion of encode command.
*/
void WaitForCompletionEvent(int iEvent);
/**
* @brief This function is used to send EOS to HW encoder.
*/
void SendEOS();
private:
/**
* @brief This is a private function which is used to check if there is any
buffering done by encoder.
* The encoder generally buffers data to encode B frames or for lookahead
* or pipelining.
*/
bool IsZeroDelay() { return m_nOutputDelay == 0; }
/**
* @brief This is a private function which is used to load the encode api
* shared library.
*/
void LoadNvEncApi();
/**
* @brief This is a private function which is used to get the output packets
* from the encoder HW.
* This is called by DoEncode() function. If there is buffering enabled,
* this may return without any output data.
*/
void GetEncodedPacket(std::vector<NV_ENC_OUTPUT_PTR>& vOutputBuffer,
std::vector<std::vector<uint8_t>>& vPacket,
bool bOutputDelay);
/**
* @brief This is a private function which is used to initialize the
* bitstream buffers. This is only used in the encoding mode.
*/
void InitializeBitstreamBuffer();
/**
* @brief This is a private function which is used to destroy the bitstream
* buffers. This is only used in the encoding mode.
*/
void DestroyBitstreamBuffer();
/**
* @brief This is a private function which is used to initialize MV output
* buffers. This is only used in ME-only Mode.
*/
void InitializeMVOutputBuffer();
/**
* @brief This is a private function which is used to destroy MV output
* buffers. This is only used in ME-only Mode.
*/
void DestroyMVOutputBuffer();
/**
* @brief This is a private function which is used to destroy HW encoder.
*/
void DestroyHWEncoder();
/**
* @brief This function is used to flush the encoder queue.
*/
void FlushEncoder();
private:
/**
* @brief This is a pure virtual function which is used to allocate input
* buffers. The derived classes must implement this function.
*/
virtual void AllocateInputBuffers(int32_t numInputBuffers) = 0;
/**
* @brief This is a pure virtual function which is used to destroy input
* buffers. The derived classes must implement this function.
*/
virtual void ReleaseInputBuffers() = 0;
protected:
bool m_bMotionEstimationOnly = false;
bool m_bOutputInVideoMemory = false;
bool m_bIsDX12Encode = false;
void* m_hEncoder = nullptr;
NV_ENCODE_API_FUNCTION_LIST m_nvenc;
NV_ENC_INITIALIZE_PARAMS m_initializeParams = {};
std::vector<NvEncInputFrame> m_vInputFrames;
std::vector<NV_ENC_REGISTERED_PTR> m_vRegisteredResources;
std::vector<NvEncInputFrame> m_vReferenceFrames;
std::vector<NV_ENC_REGISTERED_PTR> m_vRegisteredResourcesForReference;
std::vector<NV_ENC_INPUT_PTR> m_vMappedInputBuffers;
std::vector<NV_ENC_INPUT_PTR> m_vMappedRefBuffers;
std::vector<void*> m_vpCompletionEvent;
int32_t m_iToSend = 0;
int32_t m_iGot = 0;
int32_t m_nEncoderBuffer = 0;
int32_t m_nOutputDelay = 0;
IVFUtils m_IVFUtils;
bool m_bWriteIVFFileHeader = true;
bool m_bUseIVFContainer = true;
private:
uint32_t m_nWidth;
uint32_t m_nHeight;
NV_ENC_BUFFER_FORMAT m_eBufferFormat;
void* m_pDevice;
NV_ENC_DEVICE_TYPE m_eDeviceType;
NV_ENC_CONFIG m_encodeConfig = {};
bool m_bEncoderInitialized = false;
uint32_t m_nExtraOutputDelay =
3; // To ensure encode and graphics can work in parallel,
// m_nExtraOutputDelay should be set to at least 1
std::vector<NV_ENC_OUTPUT_PTR> m_vBitstreamOutputBuffer;
std::vector<NV_ENC_OUTPUT_PTR> m_vMVDataOutputBuffer;
uint32_t m_nMaxEncodeWidth = 0;
uint32_t m_nMaxEncodeHeight = 0;
void* m_hModule = nullptr;
};
@@ -0,0 +1,271 @@
/*
* Copyright 2017-2022 NVIDIA Corporation. All rights reserved.
*
* Please refer to the NVIDIA end user license agreement (EULA) associated
* with this source code for terms and conditions that govern your use of
* this software. Any use, reproduction, disclosure, or distribution of
* this software and related documentation outside the terms of the EULA
* is strictly prohibited.
*
*/
#include "NvEncoderCuda.h"
NvEncoderCuda::NvEncoderCuda(CUcontext cuContext,
uint32_t nWidth,
uint32_t nHeight,
NV_ENC_BUFFER_FORMAT eBufferFormat,
uint32_t nExtraOutputDelay,
bool bMotionEstimationOnly,
bool bOutputInVideoMemory,
bool bUseIVFContainer)
: NvEncoder(NV_ENC_DEVICE_TYPE_CUDA,
cuContext,
nWidth,
nHeight,
eBufferFormat,
nExtraOutputDelay,
bMotionEstimationOnly,
bOutputInVideoMemory,
false,
bUseIVFContainer),
m_cuContext(cuContext) {
if (!m_hEncoder) {
NVENC_THROW_ERROR("Encoder Initialization failed",
NV_ENC_ERR_INVALID_DEVICE);
}
if (!m_cuContext) {
NVENC_THROW_ERROR("Invalid Cuda Context", NV_ENC_ERR_INVALID_DEVICE);
}
}
NvEncoderCuda::~NvEncoderCuda() {
ReleaseCudaResources();
}
void NvEncoderCuda::AllocateInputBuffers(int32_t numInputBuffers) {
if (!IsHWEncoderInitialized()) {
NVENC_THROW_ERROR("Encoder intialization failed",
NV_ENC_ERR_ENCODER_NOT_INITIALIZED);
}
// for MEOnly mode we need to allocate seperate set of buffers for reference
// frame
int numCount = m_bMotionEstimationOnly ? 2 : 1;
for (int count = 0; count < numCount; count++) {
CUDA_DRVAPI_CALL(cuCtxPushCurrent(m_cuContext));
std::vector<void*> inputFrames;
for (int i = 0; i < numInputBuffers; i++) {
CUdeviceptr pDeviceFrame;
uint32_t chromaHeight =
GetNumChromaPlanes(GetPixelFormat()) *
GetChromaHeight(GetPixelFormat(), GetMaxEncodeHeight());
if (GetPixelFormat() == NV_ENC_BUFFER_FORMAT_YV12 ||
GetPixelFormat() == NV_ENC_BUFFER_FORMAT_IYUV)
chromaHeight = GetChromaHeight(GetPixelFormat(), GetMaxEncodeHeight());
CUDA_DRVAPI_CALL(cuMemAllocPitch(
(CUdeviceptr*)&pDeviceFrame, &m_cudaPitch,
GetWidthInBytes(GetPixelFormat(), GetMaxEncodeWidth()),
GetMaxEncodeHeight() + chromaHeight, 16));
inputFrames.push_back((void*)pDeviceFrame);
}
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
RegisterInputResources(
inputFrames, NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR,
GetMaxEncodeWidth(), GetMaxEncodeHeight(), (int)m_cudaPitch,
GetPixelFormat(), (count == 1) ? true : false);
}
}
void NvEncoderCuda::SetIOCudaStreams(NV_ENC_CUSTREAM_PTR inputStream,
NV_ENC_CUSTREAM_PTR outputStream) {
NVENC_API_CALL(
m_nvenc.nvEncSetIOCudaStreams(m_hEncoder, inputStream, outputStream));
}
void NvEncoderCuda::ReleaseInputBuffers() {
ReleaseCudaResources();
}
void NvEncoderCuda::ReleaseCudaResources() {
if (!m_hEncoder) {
return;
}
if (!m_cuContext) {
return;
}
UnregisterInputResources();
cuCtxPushCurrent(m_cuContext);
for (uint32_t i = 0; i < m_vInputFrames.size(); ++i) {
if (m_vInputFrames[i].inputPtr) {
cuMemFree(reinterpret_cast<CUdeviceptr>(m_vInputFrames[i].inputPtr));
}
}
m_vInputFrames.clear();
for (uint32_t i = 0; i < m_vReferenceFrames.size(); ++i) {
if (m_vReferenceFrames[i].inputPtr) {
cuMemFree(reinterpret_cast<CUdeviceptr>(m_vReferenceFrames[i].inputPtr));
}
}
m_vReferenceFrames.clear();
cuCtxPopCurrent(NULL);
m_cuContext = nullptr;
}
void NvEncoderCuda::CopyToDeviceFrame(CUcontext device,
void* pSrcFrame,
uint32_t nSrcPitch,
CUdeviceptr pDstFrame,
uint32_t dstPitch,
int width,
int height,
CUmemorytype srcMemoryType,
NV_ENC_BUFFER_FORMAT pixelFormat,
const uint32_t dstChromaOffsets[],
uint32_t numChromaPlanes,
bool bUnAlignedDeviceCopy,
CUstream stream) {
if (srcMemoryType != CU_MEMORYTYPE_HOST &&
srcMemoryType != CU_MEMORYTYPE_DEVICE) {
NVENC_THROW_ERROR("Invalid source memory type for copy",
NV_ENC_ERR_INVALID_PARAM);
}
CUDA_DRVAPI_CALL(cuCtxPushCurrent(device));
uint32_t srcPitch =
nSrcPitch ? nSrcPitch : NvEncoder::GetWidthInBytes(pixelFormat, width);
CUDA_MEMCPY2D m = {0};
m.srcMemoryType = srcMemoryType;
if (srcMemoryType == CU_MEMORYTYPE_HOST) {
m.srcHost = pSrcFrame;
} else {
m.srcDevice = (CUdeviceptr)pSrcFrame;
}
m.srcPitch = srcPitch;
m.dstMemoryType = CU_MEMORYTYPE_DEVICE;
m.dstDevice = pDstFrame;
m.dstPitch = dstPitch;
m.WidthInBytes = NvEncoder::GetWidthInBytes(pixelFormat, width);
m.Height = height;
if (bUnAlignedDeviceCopy && srcMemoryType == CU_MEMORYTYPE_DEVICE) {
CUDA_DRVAPI_CALL(cuMemcpy2DUnaligned(&m));
} else {
CUDA_DRVAPI_CALL(stream == NULL ? cuMemcpy2D(&m)
: cuMemcpy2DAsync(&m, stream));
}
std::vector<uint32_t> srcChromaOffsets;
NvEncoder::GetChromaSubPlaneOffsets(pixelFormat, srcPitch, height,
srcChromaOffsets);
uint32_t chromaHeight = NvEncoder::GetChromaHeight(pixelFormat, height);
uint32_t destChromaPitch = NvEncoder::GetChromaPitch(pixelFormat, dstPitch);
uint32_t srcChromaPitch = NvEncoder::GetChromaPitch(pixelFormat, srcPitch);
uint32_t chromaWidthInBytes =
NvEncoder::GetChromaWidthInBytes(pixelFormat, width);
for (uint32_t i = 0; i < numChromaPlanes; ++i) {
if (chromaHeight) {
if (srcMemoryType == CU_MEMORYTYPE_HOST) {
m.srcHost = ((uint8_t*)pSrcFrame + srcChromaOffsets[i]);
} else {
m.srcDevice = (CUdeviceptr)((uint8_t*)pSrcFrame + srcChromaOffsets[i]);
}
m.srcPitch = srcChromaPitch;
m.dstDevice = (CUdeviceptr)((uint8_t*)pDstFrame + dstChromaOffsets[i]);
m.dstPitch = destChromaPitch;
m.WidthInBytes = chromaWidthInBytes;
m.Height = chromaHeight;
if (bUnAlignedDeviceCopy && srcMemoryType == CU_MEMORYTYPE_DEVICE) {
CUDA_DRVAPI_CALL(cuMemcpy2DUnaligned(&m));
} else {
CUDA_DRVAPI_CALL(stream == NULL ? cuMemcpy2D(&m)
: cuMemcpy2DAsync(&m, stream));
}
}
}
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
}
void NvEncoderCuda::CopyToDeviceFrame(CUcontext device,
void* pSrcFrame,
uint32_t nSrcPitch,
CUdeviceptr pDstFrame,
uint32_t dstPitch,
int width,
int height,
CUmemorytype srcMemoryType,
NV_ENC_BUFFER_FORMAT pixelFormat,
CUdeviceptr dstChromaDevicePtrs[],
uint32_t dstChromaPitch,
uint32_t numChromaPlanes,
bool bUnAlignedDeviceCopy) {
if (srcMemoryType != CU_MEMORYTYPE_HOST &&
srcMemoryType != CU_MEMORYTYPE_DEVICE) {
NVENC_THROW_ERROR("Invalid source memory type for copy",
NV_ENC_ERR_INVALID_PARAM);
}
CUDA_DRVAPI_CALL(cuCtxPushCurrent(device));
uint32_t srcPitch =
nSrcPitch ? nSrcPitch : NvEncoder::GetWidthInBytes(pixelFormat, width);
CUDA_MEMCPY2D m = {0};
m.srcMemoryType = srcMemoryType;
if (srcMemoryType == CU_MEMORYTYPE_HOST) {
m.srcHost = pSrcFrame;
} else {
m.srcDevice = (CUdeviceptr)pSrcFrame;
}
m.srcPitch = srcPitch;
m.dstMemoryType = CU_MEMORYTYPE_DEVICE;
m.dstDevice = pDstFrame;
m.dstPitch = dstPitch;
m.WidthInBytes = NvEncoder::GetWidthInBytes(pixelFormat, width);
m.Height = height;
if (bUnAlignedDeviceCopy && srcMemoryType == CU_MEMORYTYPE_DEVICE) {
CUDA_DRVAPI_CALL(cuMemcpy2DUnaligned(&m));
} else {
CUDA_DRVAPI_CALL(cuMemcpy2D(&m));
}
std::vector<uint32_t> srcChromaOffsets;
NvEncoder::GetChromaSubPlaneOffsets(pixelFormat, srcPitch, height,
srcChromaOffsets);
uint32_t chromaHeight = NvEncoder::GetChromaHeight(pixelFormat, height);
uint32_t srcChromaPitch = NvEncoder::GetChromaPitch(pixelFormat, srcPitch);
uint32_t chromaWidthInBytes =
NvEncoder::GetChromaWidthInBytes(pixelFormat, width);
for (uint32_t i = 0; i < numChromaPlanes; ++i) {
if (chromaHeight) {
if (srcMemoryType == CU_MEMORYTYPE_HOST) {
m.srcHost = ((uint8_t*)pSrcFrame + srcChromaOffsets[i]);
} else {
m.srcDevice = (CUdeviceptr)((uint8_t*)pSrcFrame + srcChromaOffsets[i]);
}
m.srcPitch = srcChromaPitch;
m.dstDevice = dstChromaDevicePtrs[i];
m.dstPitch = dstChromaPitch;
m.WidthInBytes = chromaWidthInBytes;
m.Height = chromaHeight;
if (bUnAlignedDeviceCopy && srcMemoryType == CU_MEMORYTYPE_DEVICE) {
CUDA_DRVAPI_CALL(cuMemcpy2DUnaligned(&m));
} else {
CUDA_DRVAPI_CALL(cuMemcpy2D(&m));
}
}
}
CUDA_DRVAPI_CALL(cuCtxPopCurrent(NULL));
}
@@ -0,0 +1,123 @@
/*
* Copyright 2017-2022 NVIDIA Corporation. All rights reserved.
*
* Please refer to the NVIDIA end user license agreement (EULA) associated
* with this source code for terms and conditions that govern your use of
* this software. Any use, reproduction, disclosure, or distribution of
* this software and related documentation outside the terms of the EULA
* is strictly prohibited.
*
*/
#pragma once
#include <cuda.h>
#include <stdint.h>
#include <mutex>
#include <vector>
#include "NvEncoder.h"
#define CUDA_DRVAPI_CALL(call) \
do { \
CUresult err__ = call; \
if (err__ != CUDA_SUCCESS) { \
const char* szErrName = NULL; \
cuGetErrorName(err__, &szErrName); \
std::ostringstream errorLog; \
errorLog << "CUDA driver API error " << szErrName; \
throw NVENCException::makeNVENCException( \
errorLog.str(), NV_ENC_ERR_GENERIC, __FUNCTION__, __FILE__, \
__LINE__); \
} \
} while (0)
/**
* @brief Encoder for CUDA device memory.
*/
class NvEncoderCuda : public NvEncoder {
public:
NvEncoderCuda(CUcontext cuContext,
uint32_t nWidth,
uint32_t nHeight,
NV_ENC_BUFFER_FORMAT eBufferFormat,
uint32_t nExtraOutputDelay = 3,
bool bMotionEstimationOnly = false,
bool bOPInVideoMemory = false,
bool bUseIVFContainer = true);
virtual ~NvEncoderCuda();
/**
* @brief This is a static function to copy input data from host memory to
* device memory. This function assumes YUV plane is a single contiguous
* memory segment.
*/
static void CopyToDeviceFrame(CUcontext device,
void* pSrcFrame,
uint32_t nSrcPitch,
CUdeviceptr pDstFrame,
uint32_t dstPitch,
int width,
int height,
CUmemorytype srcMemoryType,
NV_ENC_BUFFER_FORMAT pixelFormat,
const uint32_t dstChromaOffsets[],
uint32_t numChromaPlanes,
bool bUnAlignedDeviceCopy = false,
CUstream stream = NULL);
/**
* @brief This is a static function to copy input data from host memory to
* device memory. Application must pass a seperate device pointer for each YUV
* plane.
*/
static void CopyToDeviceFrame(CUcontext device,
void* pSrcFrame,
uint32_t nSrcPitch,
CUdeviceptr pDstFrame,
uint32_t dstPitch,
int width,
int height,
CUmemorytype srcMemoryType,
NV_ENC_BUFFER_FORMAT pixelFormat,
CUdeviceptr dstChromaPtr[],
uint32_t dstChromaPitch,
uint32_t numChromaPlanes,
bool bUnAlignedDeviceCopy = false);
/**
* @brief This function sets input and output CUDA streams
*/
void SetIOCudaStreams(NV_ENC_CUSTREAM_PTR inputStream,
NV_ENC_CUSTREAM_PTR outputStream);
protected:
/**
* @brief This function is used to release the input buffers allocated for
* encoding. This function is an override of virtual function
* NvEncoder::ReleaseInputBuffers().
*/
virtual void ReleaseInputBuffers() override;
private:
/**
* @brief This function is used to allocate input buffers for encoding.
* This function is an override of virtual function
* NvEncoder::AllocateInputBuffers().
*/
virtual void AllocateInputBuffers(int32_t numInputBuffers) override;
private:
/**
* @brief This is a private function to release CUDA device memory used for
* encoding.
*/
void ReleaseCudaResources();
protected:
CUcontext m_cuContext;
private:
size_t m_cudaPitch = 0;
};
@@ -0,0 +1,2 @@
Source code under the directory are copied from this repository.
https://github.com/NVIDIA/video-sdk-samples/tree/master/Samples
@@ -0,0 +1,240 @@
/*
* Copyright 2017-2022 NVIDIA Corporation. All rights reserved.
*
* Please refer to the NVIDIA end user license agreement (EULA) associated
* with this source code for terms and conditions that govern your use of
* this software. Any use, reproduction, disclosure, or distribution of
* this software and related documentation outside the terms of the EULA
* is strictly prohibited.
*
*/
#pragma once
#include <iostream>
#include <fstream>
#include <string>
#include <sstream>
#include <mutex>
#include <time.h>
#ifdef _WIN32
#include <winsock2.h>
#include <windows.h>
#pragma comment(lib, "ws2_32.lib")
#undef ERROR
#else
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#define SOCKET int
#define INVALID_SOCKET -1
#endif
enum LogLevel {
TRACE,
INFO,
WARNING,
ERROR,
FATAL
};
namespace simplelogger{
class Logger {
public:
Logger(LogLevel level, bool bPrintTimeStamp) : level(level), bPrintTimeStamp(bPrintTimeStamp) {}
virtual ~Logger() {}
virtual std::ostream& GetStream() = 0;
virtual void FlushStream() {}
bool ShouldLogFor(LogLevel l) {
return l >= level;
}
char* GetLead(LogLevel l, const char *szFile, int nLine, const char *szFunc) {
if (l < TRACE || l > FATAL) {
sprintf(szLead, "[?????] ");
return szLead;
}
const char *szLevels[] = {"TRACE", "INFO", "WARN", "ERROR", "FATAL"};
if (bPrintTimeStamp) {
time_t t = time(NULL);
struct tm *ptm = localtime(&t);
sprintf(szLead, "[%-5s][%02d:%02d:%02d] ",
szLevels[l], ptm->tm_hour, ptm->tm_min, ptm->tm_sec);
} else {
sprintf(szLead, "[%-5s] ", szLevels[l]);
}
return szLead;
}
void EnterCriticalSection() {
mtx.lock();
}
void LeaveCriticalSection() {
mtx.unlock();
}
private:
LogLevel level;
char szLead[80];
bool bPrintTimeStamp;
std::mutex mtx;
};
class LoggerFactory {
public:
static Logger* CreateFileLogger(std::string strFilePath,
LogLevel level = INFO, bool bPrintTimeStamp = true) {
return new FileLogger(strFilePath, level, bPrintTimeStamp);
}
static Logger* CreateConsoleLogger(LogLevel level = INFO,
bool bPrintTimeStamp = true) {
return new ConsoleLogger(level, bPrintTimeStamp);
}
static Logger* CreateUdpLogger(char *szHost, unsigned uPort, LogLevel level = INFO,
bool bPrintTimeStamp = true) {
return new UdpLogger(szHost, uPort, level, bPrintTimeStamp);
}
private:
LoggerFactory() {}
class FileLogger : public Logger {
public:
FileLogger(std::string strFilePath, LogLevel level, bool bPrintTimeStamp)
: Logger(level, bPrintTimeStamp) {
pFileOut = new std::ofstream();
pFileOut->open(strFilePath.c_str());
}
~FileLogger() {
pFileOut->close();
}
std::ostream& GetStream() {
return *pFileOut;
}
private:
std::ofstream *pFileOut;
};
class ConsoleLogger : public Logger {
public:
ConsoleLogger(LogLevel level, bool bPrintTimeStamp)
: Logger(level, bPrintTimeStamp) {}
std::ostream& GetStream() {
return std::cout;
}
};
class UdpLogger : public Logger {
private:
class UdpOstream : public std::ostream {
public:
UdpOstream(char *szHost, unsigned short uPort) : std::ostream(&sb), socket(INVALID_SOCKET){
#ifdef _WIN32
WSADATA w;
if (WSAStartup(0x0101, &w) != 0) {
fprintf(stderr, "WSAStartup() failed.\n");
return;
}
#endif
socket = ::socket(AF_INET, SOCK_DGRAM, 0);
if (socket == INVALID_SOCKET) {
#ifdef _WIN32
WSACleanup();
#endif
fprintf(stderr, "socket() failed.\n");
return;
}
#ifdef _WIN32
unsigned int b1, b2, b3, b4;
sscanf(szHost, "%u.%u.%u.%u", &b1, &b2, &b3, &b4);
struct in_addr addr = {(unsigned char)b1, (unsigned char)b2, (unsigned char)b3, (unsigned char)b4};
#else
struct in_addr addr = {inet_addr(szHost)};
#endif
struct sockaddr_in s = {AF_INET, htons(uPort), addr};
server = s;
}
~UdpOstream() throw() {
if (socket == INVALID_SOCKET) {
return;
}
#ifdef _WIN32
closesocket(socket);
WSACleanup();
#else
close(socket);
#endif
}
void Flush() {
if (sendto(socket, sb.str().c_str(), (int)sb.str().length() + 1,
0, (struct sockaddr *)&server, (int)sizeof(sockaddr_in)) == -1) {
fprintf(stderr, "sendto() failed.\n");
}
sb.str("");
}
private:
std::stringbuf sb;
SOCKET socket;
struct sockaddr_in server;
};
public:
UdpLogger(char *szHost, unsigned uPort, LogLevel level, bool bPrintTimeStamp)
: Logger(level, bPrintTimeStamp), udpOut(szHost, (unsigned short)uPort) {}
UdpOstream& GetStream() {
return udpOut;
}
virtual void FlushStream() {
udpOut.Flush();
}
private:
UdpOstream udpOut;
};
};
class LogTransaction {
public:
LogTransaction(Logger *pLogger, LogLevel level, const char *szFile, const int nLine, const char *szFunc) : pLogger(pLogger), level(level) {
if (!pLogger) {
std::cout << "[-----] ";
return;
}
if (!pLogger->ShouldLogFor(level)) {
return;
}
pLogger->EnterCriticalSection();
pLogger->GetStream() << pLogger->GetLead(level, szFile, nLine, szFunc);
}
~LogTransaction() {
if (!pLogger) {
std::cout << std::endl;
return;
}
if (!pLogger->ShouldLogFor(level)) {
return;
}
pLogger->GetStream() << std::endl;
pLogger->FlushStream();
pLogger->LeaveCriticalSection();
if (level == FATAL) {
exit(1);
}
}
std::ostream& GetStream() {
if (!pLogger) {
return std::cout;
}
if (!pLogger->ShouldLogFor(level)) {
return ossNull;
}
return pLogger->GetStream();
}
private:
Logger *pLogger;
LogLevel level;
std::ostringstream ossNull;
};
}
extern simplelogger::Logger *logger;
#define LOG(level) simplelogger::LogTransaction(logger, level, __FILE__, __LINE__, __FUNCTION__).GetStream()
@@ -0,0 +1,537 @@
/*
* Copyright 2017-2022 NVIDIA Corporation. All rights reserved.
*
* Please refer to the NVIDIA end user license agreement (EULA) associated
* with this source code for terms and conditions that govern your use of
* this software. Any use, reproduction, disclosure, or distribution of
* this software and related documentation outside the terms of the EULA
* is strictly prohibited.
*
*/
//---------------------------------------------------------------------------
//! \file NvCodecUtils.h
//! \brief Miscellaneous classes and error checking functions.
//!
//! Used by Transcode/Encode samples apps for reading input files, mutithreading, performance measurement or colorspace conversion while decoding.
//---------------------------------------------------------------------------
#pragma once
#include <iomanip>
#include <chrono>
#include <sys/stat.h>
#include <assert.h>
#include <stdint.h>
#include <string.h>
#include <ios>
#include <sstream>
#include <thread>
#include <list>
#include <vector>
#include <condition_variable>
#include "Logger.h"
extern simplelogger::Logger *logger;
#ifdef __cuda_cuda_h__
inline bool check(CUresult e, int iLine, const char *szFile) {
if (e != CUDA_SUCCESS) {
const char *szErrName = NULL;
cuGetErrorName(e, &szErrName);
LOG(FATAL) << "CUDA driver API error " << szErrName << " at line " << iLine << " in file " << szFile;
return false;
}
return true;
}
#endif
#ifdef __CUDA_RUNTIME_H__
inline bool check(cudaError_t e, int iLine, const char *szFile) {
if (e != cudaSuccess) {
LOG(FATAL) << "CUDA runtime API error " << cudaGetErrorName(e) << " at line " << iLine << " in file " << szFile;
return false;
}
return true;
}
#endif
#ifdef _NV_ENCODEAPI_H_
inline bool check(NVENCSTATUS e, int iLine, const char *szFile) {
const char *aszErrName[] = {
"NV_ENC_SUCCESS",
"NV_ENC_ERR_NO_ENCODE_DEVICE",
"NV_ENC_ERR_UNSUPPORTED_DEVICE",
"NV_ENC_ERR_INVALID_ENCODERDEVICE",
"NV_ENC_ERR_INVALID_DEVICE",
"NV_ENC_ERR_DEVICE_NOT_EXIST",
"NV_ENC_ERR_INVALID_PTR",
"NV_ENC_ERR_INVALID_EVENT",
"NV_ENC_ERR_INVALID_PARAM",
"NV_ENC_ERR_INVALID_CALL",
"NV_ENC_ERR_OUT_OF_MEMORY",
"NV_ENC_ERR_ENCODER_NOT_INITIALIZED",
"NV_ENC_ERR_UNSUPPORTED_PARAM",
"NV_ENC_ERR_LOCK_BUSY",
"NV_ENC_ERR_NOT_ENOUGH_BUFFER",
"NV_ENC_ERR_INVALID_VERSION",
"NV_ENC_ERR_MAP_FAILED",
"NV_ENC_ERR_NEED_MORE_INPUT",
"NV_ENC_ERR_ENCODER_BUSY",
"NV_ENC_ERR_EVENT_NOT_REGISTERD",
"NV_ENC_ERR_GENERIC",
"NV_ENC_ERR_INCOMPATIBLE_CLIENT_KEY",
"NV_ENC_ERR_UNIMPLEMENTED",
"NV_ENC_ERR_RESOURCE_REGISTER_FAILED",
"NV_ENC_ERR_RESOURCE_NOT_REGISTERED",
"NV_ENC_ERR_RESOURCE_NOT_MAPPED",
};
if (e != NV_ENC_SUCCESS) {
LOG(FATAL) << "NVENC error " << aszErrName[e] << " at line " << iLine << " in file " << szFile;
return false;
}
return true;
}
#endif
#ifdef _WINERROR_
inline bool check(HRESULT e, int iLine, const char *szFile) {
if (e != S_OK) {
std::stringstream stream;
stream << std::hex << std::uppercase << e;
LOG(FATAL) << "HRESULT error 0x" << stream.str() << " at line " << iLine << " in file " << szFile;
return false;
}
return true;
}
#endif
#if defined(__gl_h_) || defined(__GL_H__)
inline bool check(GLenum e, int iLine, const char *szFile) {
if (e != 0) {
LOG(ERROR) << "GLenum error " << e << " at line " << iLine << " in file " << szFile;
return false;
}
return true;
}
#endif
inline bool check(int e, int iLine, const char *szFile) {
if (e < 0) {
LOG(ERROR) << "General error " << e << " at line " << iLine << " in file " << szFile;
return false;
}
return true;
}
#define ck(call) check(call, __LINE__, __FILE__)
#define MAKE_FOURCC( ch0, ch1, ch2, ch3 ) \
( (uint32_t)(uint8_t)(ch0) | ( (uint32_t)(uint8_t)(ch1) << 8 ) | \
( (uint32_t)(uint8_t)(ch2) << 16 ) | ( (uint32_t)(uint8_t)(ch3) << 24 ) )
/**
* @brief Wrapper class around std::thread
*/
class NvThread
{
public:
NvThread() = default;
NvThread(const NvThread&) = delete;
NvThread& operator=(const NvThread& other) = delete;
NvThread(std::thread&& thread) : t(std::move(thread))
{
}
NvThread(NvThread&& thread) : t(std::move(thread.t))
{
}
NvThread& operator=(NvThread&& other)
{
t = std::move(other.t);
return *this;
}
~NvThread()
{
join();
}
void join()
{
if (t.joinable())
{
t.join();
}
}
private:
std::thread t;
};
#ifndef _WIN32
#define _stricmp strcasecmp
#define _stat64 stat64
#endif
/**
* @brief Utility class to allocate buffer memory. Helps avoid I/O during the encode/decode loop in case of performance tests.
*/
class BufferedFileReader {
public:
/**
* @brief Constructor function to allocate appropriate memory and copy file contents into it
*/
BufferedFileReader(const char *szFileName, bool bPartial = false) {
struct _stat64 st;
if (_stat64(szFileName, &st) != 0) {
return;
}
nSize = st.st_size;
while (nSize) {
try {
pBuf = new uint8_t[(size_t)nSize];
if (nSize != st.st_size) {
LOG(WARNING) << "File is too large - only " << std::setprecision(4) << 100.0 * nSize / st.st_size << "% is loaded";
}
break;
} catch(std::bad_alloc) {
if (!bPartial) {
LOG(ERROR) << "Failed to allocate memory in BufferedReader";
return;
}
nSize = (uint32_t)(nSize * 0.9);
}
}
std::ifstream fpIn(szFileName, std::ifstream::in | std::ifstream::binary);
if (!fpIn)
{
LOG(ERROR) << "Unable to open input file: " << szFileName;
return;
}
std::streamsize nRead = fpIn.read(reinterpret_cast<char*>(pBuf), nSize).gcount();
fpIn.close();
assert(nRead == nSize);
}
~BufferedFileReader() {
if (pBuf) {
delete[] pBuf;
}
}
bool GetBuffer(uint8_t **ppBuf, uint64_t *pnSize) {
if (!pBuf) {
return false;
}
*ppBuf = pBuf;
*pnSize = nSize;
return true;
}
private:
uint8_t *pBuf = NULL;
uint64_t nSize = 0;
};
/**
* @brief Template class to facilitate color space conversion
*/
template<typename T>
class YuvConverter {
public:
YuvConverter(int nWidth, int nHeight) : nWidth(nWidth), nHeight(nHeight) {
pQuad = new T[((nWidth + 1) / 2) * ((nHeight + 1) / 2)];
}
~YuvConverter() {
delete[] pQuad;
}
void PlanarToUVInterleaved(T *pFrame, int nPitch = 0) {
if (nPitch == 0) {
nPitch = nWidth;
}
// sizes of source surface plane
int nSizePlaneY = nPitch * nHeight;
int nSizePlaneU = ((nPitch + 1) / 2) * ((nHeight + 1) / 2);
int nSizePlaneV = nSizePlaneU;
T *puv = pFrame + nSizePlaneY;
if (nPitch == nWidth) {
memcpy(pQuad, puv, nSizePlaneU * sizeof(T));
} else {
for (int i = 0; i < (nHeight + 1) / 2; i++) {
memcpy(pQuad + ((nWidth + 1) / 2) * i, puv + ((nPitch + 1) / 2) * i, ((nWidth + 1) / 2) * sizeof(T));
}
}
T *pv = puv + nSizePlaneU;
for (int y = 0; y < (nHeight + 1) / 2; y++) {
for (int x = 0; x < (nWidth + 1) / 2; x++) {
puv[y * nPitch + x * 2] = pQuad[y * ((nWidth + 1) / 2) + x];
puv[y * nPitch + x * 2 + 1] = pv[y * ((nPitch + 1) / 2) + x];
}
}
}
void UVInterleavedToPlanar(T *pFrame, int nPitch = 0) {
if (nPitch == 0) {
nPitch = nWidth;
}
// sizes of source surface plane
int nSizePlaneY = nPitch * nHeight;
int nSizePlaneU = ((nPitch + 1) / 2) * ((nHeight + 1) / 2);
int nSizePlaneV = nSizePlaneU;
T *puv = pFrame + nSizePlaneY,
*pu = puv,
*pv = puv + nSizePlaneU;
// split chroma from interleave to planar
for (int y = 0; y < (nHeight + 1) / 2; y++) {
for (int x = 0; x < (nWidth + 1) / 2; x++) {
pu[y * ((nPitch + 1) / 2) + x] = puv[y * nPitch + x * 2];
pQuad[y * ((nWidth + 1) / 2) + x] = puv[y * nPitch + x * 2 + 1];
}
}
if (nPitch == nWidth) {
memcpy(pv, pQuad, nSizePlaneV * sizeof(T));
} else {
for (int i = 0; i < (nHeight + 1) / 2; i++) {
memcpy(pv + ((nPitch + 1) / 2) * i, pQuad + ((nWidth + 1) / 2) * i, ((nWidth + 1) / 2) * sizeof(T));
}
}
}
private:
T *pQuad;
int nWidth, nHeight;
};
/**
* @brief Class for writing IVF format header for AV1 codec
*/
class IVFUtils {
public:
void WriteFileHeader(std::vector<uint8_t> &vPacket, uint32_t nFourCC, uint32_t nWidth, uint32_t nHeight, uint32_t nFrameRateNum, uint32_t nFrameRateDen, uint32_t nFrameCnt)
{
char header[32];
header[0] = 'D';
header[1] = 'K';
header[2] = 'I';
header[3] = 'F';
mem_put_le16(header + 4, 0); // version
mem_put_le16(header + 6, 32); // header size
mem_put_le32(header + 8, nFourCC); // fourcc
mem_put_le16(header + 12, nWidth); // width
mem_put_le16(header + 14, nHeight); // height
mem_put_le32(header + 16, nFrameRateNum); // rate
mem_put_le32(header + 20, nFrameRateDen); // scale
mem_put_le32(header + 24, nFrameCnt); // length
mem_put_le32(header + 28, 0); // unused
vPacket.insert(vPacket.end(), &header[0], &header[32]);
}
void WriteFrameHeader(std::vector<uint8_t> &vPacket, size_t nFrameSize, int64_t pts)
{
char header[12];
mem_put_le32(header, (int)nFrameSize);
mem_put_le32(header + 4, (int)(pts & 0xFFFFFFFF));
mem_put_le32(header + 8, (int)(pts >> 32));
vPacket.insert(vPacket.end(), &header[0], &header[12]);
}
private:
static inline void mem_put_le32(void *vmem, int val)
{
unsigned char *mem = (unsigned char *)vmem;
mem[0] = (unsigned char)((val >> 0) & 0xff);
mem[1] = (unsigned char)((val >> 8) & 0xff);
mem[2] = (unsigned char)((val >> 16) & 0xff);
mem[3] = (unsigned char)((val >> 24) & 0xff);
}
static inline void mem_put_le16(void *vmem, int val)
{
unsigned char *mem = (unsigned char *)vmem;
mem[0] = (unsigned char)((val >> 0) & 0xff);
mem[1] = (unsigned char)((val >> 8) & 0xff);
}
};
/**
* @brief Utility class to measure elapsed time in seconds between the block of executed code
*/
class StopWatch {
public:
void Start() {
t0 = std::chrono::high_resolution_clock::now();
}
double Stop() {
return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::high_resolution_clock::now().time_since_epoch() - t0.time_since_epoch()).count() / 1.0e9;
}
private:
std::chrono::high_resolution_clock::time_point t0;
};
template<typename T>
class ConcurrentQueue
{
public:
ConcurrentQueue() {}
ConcurrentQueue(size_t size) : maxSize(size) {}
ConcurrentQueue(const ConcurrentQueue&) = delete;
ConcurrentQueue& operator=(const ConcurrentQueue&) = delete;
void setSize(size_t s) {
maxSize = s;
}
void push_back(const T& value) {
// Do not use a std::lock_guard here. We will need to explicitly
// unlock before notify_one as the other waiting thread will
// automatically try to acquire mutex once it wakes up
// (which will happen on notify_one)
std::unique_lock<std::mutex> lock(m_mutex);
auto wasEmpty = m_List.empty();
while (full()) {
m_cond.wait(lock);
}
m_List.push_back(value);
if (wasEmpty && !m_List.empty()) {
lock.unlock();
m_cond.notify_one();
}
}
T pop_front() {
std::unique_lock<std::mutex> lock(m_mutex);
while (m_List.empty()) {
m_cond.wait(lock);
}
auto wasFull = full();
T data = std::move(m_List.front());
m_List.pop_front();
if (wasFull && !full()) {
lock.unlock();
m_cond.notify_one();
}
return data;
}
T front() {
std::unique_lock<std::mutex> lock(m_mutex);
while (m_List.empty()) {
m_cond.wait(lock);
}
return m_List.front();
}
size_t size() {
std::unique_lock<std::mutex> lock(m_mutex);
return m_List.size();
}
bool empty() {
std::unique_lock<std::mutex> lock(m_mutex);
return m_List.empty();
}
void clear() {
std::unique_lock<std::mutex> lock(m_mutex);
m_List.clear();
}
private:
bool full() {
if (m_List.size() == maxSize)
return true;
return false;
}
private:
std::list<T> m_List;
std::mutex m_mutex;
std::condition_variable m_cond;
size_t maxSize;
};
inline void CheckInputFile(const char *szInFilePath) {
std::ifstream fpIn(szInFilePath, std::ios::in | std::ios::binary);
if (fpIn.fail()) {
std::ostringstream err;
err << "Unable to open input file: " << szInFilePath << std::endl;
throw std::invalid_argument(err.str());
}
}
inline void ValidateResolution(int nWidth, int nHeight) {
if (nWidth <= 0 || nHeight <= 0) {
std::ostringstream err;
err << "Please specify positive non zero resolution as -s WxH. Current resolution is " << nWidth << "x" << nHeight << std::endl;
throw std::invalid_argument(err.str());
}
}
template <class COLOR32>
void Nv12ToColor32(uint8_t *dpNv12, int nNv12Pitch, uint8_t *dpBgra, int nBgraPitch, int nWidth, int nHeight, int iMatrix = 0);
template <class COLOR64>
void Nv12ToColor64(uint8_t *dpNv12, int nNv12Pitch, uint8_t *dpBgra, int nBgraPitch, int nWidth, int nHeight, int iMatrix = 0);
template <class COLOR32>
void P016ToColor32(uint8_t *dpP016, int nP016Pitch, uint8_t *dpBgra, int nBgraPitch, int nWidth, int nHeight, int iMatrix = 4);
template <class COLOR64>
void P016ToColor64(uint8_t *dpP016, int nP016Pitch, uint8_t *dpBgra, int nBgraPitch, int nWidth, int nHeight, int iMatrix = 4);
template <class COLOR32>
void YUV444ToColor32(uint8_t *dpYUV444, int nPitch, uint8_t *dpBgra, int nBgraPitch, int nWidth, int nHeight, int iMatrix = 0);
template <class COLOR64>
void YUV444ToColor64(uint8_t *dpYUV444, int nPitch, uint8_t *dpBgra, int nBgraPitch, int nWidth, int nHeight, int iMatrix = 0);
template <class COLOR32>
void YUV444P16ToColor32(uint8_t *dpYUV444, int nPitch, uint8_t *dpBgra, int nBgraPitch, int nWidth, int nHeight, int iMatrix = 4);
template <class COLOR64>
void YUV444P16ToColor64(uint8_t *dpYUV444, int nPitch, uint8_t *dpBgra, int nBgraPitch, int nWidth, int nHeight, int iMatrix = 4);
template <class COLOR32>
void Nv12ToColorPlanar(uint8_t *dpNv12, int nNv12Pitch, uint8_t *dpBgrp, int nBgrpPitch, int nWidth, int nHeight, int iMatrix = 0);
template <class COLOR32>
void P016ToColorPlanar(uint8_t *dpP016, int nP016Pitch, uint8_t *dpBgrp, int nBgrpPitch, int nWidth, int nHeight, int iMatrix = 4);
template <class COLOR32>
void YUV444ToColorPlanar(uint8_t *dpYUV444, int nPitch, uint8_t *dpBgrp, int nBgrpPitch, int nWidth, int nHeight, int iMatrix = 0);
template <class COLOR32>
void YUV444P16ToColorPlanar(uint8_t *dpYUV444, int nPitch, uint8_t *dpBgrp, int nBgrpPitch, int nWidth, int nHeight, int iMatrix = 4);
void Bgra64ToP016(uint8_t *dpBgra, int nBgraPitch, uint8_t *dpP016, int nP016Pitch, int nWidth, int nHeight, int iMatrix = 4);
void ConvertUInt8ToUInt16(uint8_t *dpUInt8, uint16_t *dpUInt16, int nSrcPitch, int nDestPitch, int nWidth, int nHeight);
void ConvertUInt16ToUInt8(uint16_t *dpUInt16, uint8_t *dpUInt8, int nSrcPitch, int nDestPitch, int nWidth, int nHeight);
void ResizeNv12(unsigned char *dpDstNv12, int nDstPitch, int nDstWidth, int nDstHeight, unsigned char *dpSrcNv12, int nSrcPitch, int nSrcWidth, int nSrcHeight, unsigned char *dpDstNv12UV = nullptr);
void ResizeP016(unsigned char *dpDstP016, int nDstPitch, int nDstWidth, int nDstHeight, unsigned char *dpSrcP016, int nSrcPitch, int nSrcWidth, int nSrcHeight, unsigned char *dpDstP016UV = nullptr);
void ScaleYUV420(unsigned char *dpDstY, unsigned char* dpDstU, unsigned char* dpDstV, int nDstPitch, int nDstChromaPitch, int nDstWidth, int nDstHeight,
unsigned char *dpSrcY, unsigned char* dpSrcU, unsigned char* dpSrcV, int nSrcPitch, int nSrcChromaPitch, int nSrcWidth, int nSrcHeight, bool bSemiplanar);
#ifdef __cuda_cuda_h__
void ComputeCRC(uint8_t *pBuffer, uint32_t *crcValue, CUstream_st *outputCUStream);
#endif
@@ -0,0 +1,230 @@
/*
* SPDX-License-Identifier: MIT
*
* Minimal vendored CUDA Driver API declarations used by Fluxer's native
* LiveKit/WebRTC NVENC bridge. This is intentionally not the CUDA Toolkit
* header; it keeps the build independent of a system CUDA SDK while still
* compiling against the NVIDIA driver API that is loaded dynamically at
* runtime.
*/
#ifndef __cuda_cuda_h__
#define __cuda_cuda_h__
#include <stddef.h>
#ifdef _WIN32
#define CUDAAPI __stdcall
#else
#define CUDAAPI
#endif
#ifdef __cplusplus
extern "C" {
#endif
#ifndef CUDA_VERSION
#define CUDA_VERSION 12000
#endif
typedef enum CUresult_enum {
CUDA_SUCCESS = 0,
CUDA_ERROR_INVALID_VALUE = 1,
CUDA_ERROR_OUT_OF_MEMORY = 2,
CUDA_ERROR_NOT_INITIALIZED = 3,
CUDA_ERROR_DEINITIALIZED = 4,
CUDA_ERROR_NOT_SUPPORTED = 801
} CUresult;
typedef int CUdevice;
typedef struct CUctx_st* CUcontext;
typedef struct CUstream_st* CUstream;
typedef struct CUarray_st* CUarray;
typedef struct CUgraphicsResource_st* CUgraphicsResource;
typedef unsigned long long CUdeviceptr;
typedef enum CUmemorytype_enum {
CU_MEMORYTYPE_HOST = 0x01,
CU_MEMORYTYPE_DEVICE = 0x02,
CU_MEMORYTYPE_ARRAY = 0x03,
CU_MEMORYTYPE_UNIFIED = 0x04
} CUmemorytype;
typedef enum CUdevice_attribute_enum {
CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75
} CUdevice_attribute;
typedef enum CUeglFrameType_enum {
CU_EGL_FRAME_TYPE_ARRAY = 0,
CU_EGL_FRAME_TYPE_PITCH = 1
} CUeglFrameType;
typedef enum CUeglColorFormat_enum {
CU_EGL_COLOR_FORMAT_YUV420_PLANAR = 0x00,
CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR = 0x01,
CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER = 0x02,
CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR = 0x03,
CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER = 0x04,
CU_EGL_COLOR_FORMAT_ARGB = 0x05,
CU_EGL_COLOR_FORMAT_RGBA = 0x06,
CU_EGL_COLOR_FORMAT_L = 0x07,
CU_EGL_COLOR_FORMAT_R = 0x08,
CU_EGL_COLOR_FORMAT_YUV444_PLANAR = 0x09,
CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR = 0x0a,
CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR = 0x0b,
CU_EGL_COLOR_FORMAT_Y = 0x0c,
CU_EGL_COLOR_FORMAT_YUVY = 0x0d,
CU_EGL_COLOR_FORMAT_UYVY = 0x0e,
CU_EGL_COLOR_FORMAT_ABGR = 0x0f,
CU_EGL_COLOR_FORMAT_BGRA = 0x10,
CU_EGL_COLOR_FORMAT_A = 0x11,
CU_EGL_COLOR_FORMAT_RG = 0x12,
CU_EGL_COLOR_FORMAT_AYUV = 0x13,
CU_EGL_COLOR_FORMAT_YVU444_PLANAR = 0x14,
CU_EGL_COLOR_FORMAT_YVU422_PLANAR = 0x15,
CU_EGL_COLOR_FORMAT_YUV422_PLANAR = 0x16,
CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR = 0x17,
CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR = 0x18,
CU_EGL_COLOR_FORMAT_YUYV = 0x19,
CU_EGL_COLOR_FORMAT_UYVY_ER = 0x1a,
CU_EGL_COLOR_FORMAT_YUYV_ER = 0x1b,
CU_EGL_COLOR_FORMAT_YUVA = 0x1c,
CU_EGL_COLOR_FORMAT_AYUV_ER = 0x1d,
CU_EGL_COLOR_FORMAT_YUVA_ER = 0x1e,
CU_EGL_COLOR_FORMAT_LAST = 0x1f
} CUeglColorFormat;
typedef enum CUarray_format_enum {
CU_AD_FORMAT_UNSIGNED_INT8 = 0x01,
CU_AD_FORMAT_UNSIGNED_INT16 = 0x02,
CU_AD_FORMAT_UNSIGNED_INT32 = 0x03,
CU_AD_FORMAT_SIGNED_INT8 = 0x08,
CU_AD_FORMAT_SIGNED_INT16 = 0x09,
CU_AD_FORMAT_SIGNED_INT32 = 0x0a,
CU_AD_FORMAT_HALF = 0x10,
CU_AD_FORMAT_FLOAT = 0x20
} CUarray_format;
typedef struct CUDA_MEMCPY2D_st {
size_t srcXInBytes;
size_t srcY;
CUmemorytype srcMemoryType;
const void* srcHost;
CUdeviceptr srcDevice;
CUarray srcArray;
size_t srcPitch;
size_t dstXInBytes;
size_t dstY;
CUmemorytype dstMemoryType;
void* dstHost;
CUdeviceptr dstDevice;
CUarray dstArray;
size_t dstPitch;
size_t WidthInBytes;
size_t Height;
} CUDA_MEMCPY2D;
typedef struct CUDA_ARRAY3D_DESCRIPTOR_st {
size_t Width;
size_t Height;
size_t Depth;
CUarray_format Format;
unsigned int NumChannels;
unsigned int Flags;
} CUDA_ARRAY3D_DESCRIPTOR;
typedef struct CUDA_RESOURCE_DESC_st {
int resType;
union {
struct {
CUarray hArray;
} array;
struct {
CUdeviceptr devPtr;
CUarray_format format;
unsigned int numChannels;
size_t sizeInBytes;
} linear;
struct {
CUdeviceptr devPtr;
CUarray_format format;
unsigned int numChannels;
size_t width;
size_t height;
size_t pitchInBytes;
} pitch2D;
struct {
unsigned int reserved[32];
} reserved;
} res;
unsigned int flags;
} CUDA_RESOURCE_DESC;
typedef struct CUeglFrame_st {
union {
CUarray pArray[3];
void* pPitch[3];
} frame;
unsigned int width;
unsigned int height;
unsigned int depth;
unsigned int pitch;
unsigned int planeCount;
unsigned int numChannels;
CUeglFrameType frameType;
CUeglColorFormat eglColorFormat;
CUarray_format cuFormat;
} CUeglFrame;
#define CU_STREAM_DEFAULT 0
#define CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE 0x00
CUresult CUDAAPI cuInit(unsigned int flags);
CUresult CUDAAPI cuDriverGetVersion(int* driverVersion);
CUresult CUDAAPI cuGetErrorName(CUresult error, const char** pStr);
CUresult CUDAAPI cuDeviceGetCount(int* count);
CUresult CUDAAPI cuDeviceGet(CUdevice* device, int ordinal);
CUresult CUDAAPI cuDeviceGetName(char* name, int len, CUdevice dev);
CUresult CUDAAPI cuDeviceGetAttribute(int* pi, CUdevice_attribute attrib, CUdevice dev);
CUresult CUDAAPI cuCtxCreate(CUcontext* pctx, unsigned int flags, CUdevice dev);
CUresult CUDAAPI cuCtxCreate_v2(CUcontext* pctx, unsigned int flags, CUdevice dev);
CUresult CUDAAPI cuCtxDestroy(CUcontext ctx);
CUresult CUDAAPI cuCtxDestroy_v2(CUcontext ctx);
CUresult CUDAAPI cuCtxGetCurrent(CUcontext* pctx);
CUresult CUDAAPI cuCtxSetCurrent(CUcontext ctx);
CUresult CUDAAPI cuCtxGetDevice(CUdevice* device);
CUresult CUDAAPI cuCtxPushCurrent(CUcontext ctx);
CUresult CUDAAPI cuCtxPushCurrent_v2(CUcontext ctx);
CUresult CUDAAPI cuCtxPopCurrent(CUcontext* pctx);
CUresult CUDAAPI cuCtxPopCurrent_v2(CUcontext* pctx);
CUresult CUDAAPI cuMemAlloc(CUdeviceptr* dptr, size_t bytesize);
CUresult CUDAAPI cuMemAllocPitch(CUdeviceptr* dptr,
size_t* pPitch,
size_t WidthInBytes,
size_t Height,
unsigned int ElementSizeBytes);
CUresult CUDAAPI cuMemFree(CUdeviceptr dptr);
CUresult CUDAAPI cuMemFree_v2(CUdeviceptr dptr);
CUresult CUDAAPI cuMemcpy2D(const CUDA_MEMCPY2D* pCopy);
CUresult CUDAAPI cuMemcpy2D_v2(const CUDA_MEMCPY2D* pCopy);
CUresult CUDAAPI cuMemcpy2DUnaligned(const CUDA_MEMCPY2D* pCopy);
CUresult CUDAAPI cuMemcpy2DUnaligned_v2(const CUDA_MEMCPY2D* pCopy);
CUresult CUDAAPI cuMemcpy2DAsync(const CUDA_MEMCPY2D* pCopy, CUstream hStream);
CUresult CUDAAPI cuMemcpy2DAsync_v2(const CUDA_MEMCPY2D* pCopy, CUstream hStream);
CUresult CUDAAPI cuStreamCreate(CUstream* phStream, unsigned int Flags);
CUresult CUDAAPI cuStreamSynchronize(CUstream hStream);
CUresult CUDAAPI cuArrayDestroy(CUarray hArray);
CUresult CUDAAPI cuGraphicsEGLRegisterImage(CUgraphicsResource* pCudaResource,
void* image,
unsigned int flags);
CUresult CUDAAPI cuGraphicsResourceGetMappedEglFrame(CUeglFrame* eglFrame,
CUgraphicsResource resource,
unsigned int index,
unsigned int mipLevel);
CUresult CUDAAPI cuGraphicsUnregisterResource(CUgraphicsResource resource);
#ifdef __cplusplus
}
#endif
#endif
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,501 @@
/*
* This copyright notice applies to this header file only:
*
* Copyright (c) 2010-2022 NVIDIA Corporation
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the software, and to permit persons to whom the
* software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
/********************************************************************************************************************/
//! \file nvcuvid.h
//! NVDECODE API provides video decoding interface to NVIDIA GPU devices.
//! \date 2015-2022
//! This file contains the interface constants, structure definitions and function prototypes.
/********************************************************************************************************************/
#if !defined(__NVCUVID_H__)
#define __NVCUVID_H__
#include "cuviddec.h"
#if defined(__cplusplus)
extern "C" {
#endif /* __cplusplus */
#define MAX_CLOCK_TS 3
/***********************************************/
//!
//! High-level helper APIs for video sources
//!
/***********************************************/
typedef void *CUvideosource;
typedef void *CUvideoparser;
typedef long long CUvideotimestamp;
/************************************************************************/
//! \enum cudaVideoState
//! Video source state enums
//! Used in cuvidSetVideoSourceState and cuvidGetVideoSourceState APIs
/************************************************************************/
typedef enum {
cudaVideoState_Error = -1, /**< Error state (invalid source) */
cudaVideoState_Stopped = 0, /**< Source is stopped (or reached end-of-stream) */
cudaVideoState_Started = 1 /**< Source is running and delivering data */
} cudaVideoState;
/************************************************************************/
//! \enum cudaAudioCodec
//! Audio compression enums
//! Used in CUAUDIOFORMAT structure
/************************************************************************/
typedef enum {
cudaAudioCodec_MPEG1=0, /**< MPEG-1 Audio */
cudaAudioCodec_MPEG2, /**< MPEG-2 Audio */
cudaAudioCodec_MP3, /**< MPEG-1 Layer III Audio */
cudaAudioCodec_AC3, /**< Dolby Digital (AC3) Audio */
cudaAudioCodec_LPCM, /**< PCM Audio */
cudaAudioCodec_AAC, /**< AAC Audio */
} cudaAudioCodec;
/************************************************************************/
//! \ingroup STRUCTS
//! \struct HEVCTIMECODESET
//! Used to store Time code extracted from Time code SEI in HEVC codec
/************************************************************************/
typedef struct _HEVCTIMECODESET
{
unsigned int time_offset_value;
unsigned short n_frames;
unsigned char clock_timestamp_flag;
unsigned char units_field_based_flag;
unsigned char counting_type;
unsigned char full_timestamp_flag;
unsigned char discontinuity_flag;
unsigned char cnt_dropped_flag;
unsigned char seconds_value;
unsigned char minutes_value;
unsigned char hours_value;
unsigned char seconds_flag;
unsigned char minutes_flag;
unsigned char hours_flag;
unsigned char time_offset_length;
unsigned char reserved;
} HEVCTIMECODESET;
/************************************************************************/
//! \ingroup STRUCTS
//! \struct HEVCSEITIMECODE
//! Used to extract Time code SEI in HEVC codec
/************************************************************************/
typedef struct _HEVCSEITIMECODE
{
HEVCTIMECODESET time_code_set[MAX_CLOCK_TS];
unsigned char num_clock_ts;
} HEVCSEITIMECODE;
/**********************************************************************************/
//! \ingroup STRUCTS
//! \struct CUSEIMESSAGE;
//! Used in CUVIDSEIMESSAGEINFO structure
/**********************************************************************************/
typedef struct _CUSEIMESSAGE
{
unsigned char sei_message_type; /**< OUT: SEI Message Type */
unsigned char reserved[3];
unsigned int sei_message_size; /**< OUT: SEI Message Size */
} CUSEIMESSAGE;
/************************************************************************************************/
//! \ingroup STRUCTS
//! \struct CUVIDEOFORMAT
//! Video format
//! Used in cuvidGetSourceVideoFormat API
/************************************************************************************************/
typedef struct
{
cudaVideoCodec codec; /**< OUT: Compression format */
/**
* OUT: frame rate = numerator / denominator (for example: 30000/1001)
*/
struct {
/**< OUT: frame rate numerator (0 = unspecified or variable frame rate) */
unsigned int numerator;
/**< OUT: frame rate denominator (0 = unspecified or variable frame rate) */
unsigned int denominator;
} frame_rate;
unsigned char progressive_sequence; /**< OUT: 0=interlaced, 1=progressive */
unsigned char bit_depth_luma_minus8; /**< OUT: high bit depth luma. E.g, 2 for 10-bitdepth, 4 for 12-bitdepth */
unsigned char bit_depth_chroma_minus8; /**< OUT: high bit depth chroma. E.g, 2 for 10-bitdepth, 4 for 12-bitdepth */
unsigned char min_num_decode_surfaces; /**< OUT: Minimum number of decode surfaces to be allocated for correct
decoding. The client can send this value in ulNumDecodeSurfaces
(in CUVIDDECODECREATEINFO structure).
This guarantees correct functionality and optimal video memory
usage but not necessarily the best performance, which depends on
the design of the overall application. The optimal number of
decode surfaces (in terms of performance and memory utilization)
should be decided by experimentation for each application, but it
cannot go below min_num_decode_surfaces.
If this value is used for ulNumDecodeSurfaces then it must be
returned to parser during sequence callback. */
unsigned int coded_width; /**< OUT: coded frame width in pixels */
unsigned int coded_height; /**< OUT: coded frame height in pixels */
/**
* area of the frame that should be displayed
* typical example:
* coded_width = 1920, coded_height = 1088
* display_area = { 0,0,1920,1080 }
*/
struct {
int left; /**< OUT: left position of display rect */
int top; /**< OUT: top position of display rect */
int right; /**< OUT: right position of display rect */
int bottom; /**< OUT: bottom position of display rect */
} display_area;
cudaVideoChromaFormat chroma_format; /**< OUT: Chroma format */
unsigned int bitrate; /**< OUT: video bitrate (bps, 0=unknown) */
/**
* OUT: Display Aspect Ratio = x:y (4:3, 16:9, etc)
*/
struct {
int x;
int y;
} display_aspect_ratio;
/**
* Video Signal Description
* Refer section E.2.1 (VUI parameters semantics) of H264 spec file
*/
struct {
unsigned char video_format : 3; /**< OUT: 0-Component, 1-PAL, 2-NTSC, 3-SECAM, 4-MAC, 5-Unspecified */
unsigned char video_full_range_flag : 1; /**< OUT: indicates the black level and luma and chroma range */
unsigned char reserved_zero_bits : 4; /**< Reserved bits */
unsigned char color_primaries; /**< OUT: chromaticity coordinates of source primaries */
unsigned char transfer_characteristics; /**< OUT: opto-electronic transfer characteristic of the source picture */
unsigned char matrix_coefficients; /**< OUT: used in deriving luma and chroma signals from RGB primaries */
} video_signal_description;
unsigned int seqhdr_data_length; /**< OUT: Additional bytes following (CUVIDEOFORMATEX) */
} CUVIDEOFORMAT;
/****************************************************************/
//! \ingroup STRUCTS
//! \struct CUVIDOPERATINGPOINTINFO
//! Operating point information of scalable bitstream
/****************************************************************/
typedef struct
{
cudaVideoCodec codec;
union
{
struct
{
unsigned char operating_points_cnt;
unsigned char reserved24_bits[3];
unsigned short operating_points_idc[32];
} av1;
unsigned char CodecReserved[1024];
};
} CUVIDOPERATINGPOINTINFO;
/**********************************************************************************/
//! \ingroup STRUCTS
//! \struct CUVIDSEIMESSAGEINFO
//! Used in cuvidParseVideoData API with PFNVIDSEIMSGCALLBACK pfnGetSEIMsg
/**********************************************************************************/
typedef struct _CUVIDSEIMESSAGEINFO
{
void *pSEIData; /**< OUT: SEI Message Data */
CUSEIMESSAGE *pSEIMessage; /**< OUT: SEI Message Info */
unsigned int sei_message_count; /**< OUT: SEI Message Count */
unsigned int picIdx; /**< OUT: SEI Message Pic Index */
} CUVIDSEIMESSAGEINFO;
/****************************************************************/
//! \ingroup STRUCTS
//! \struct CUVIDAV1SEQHDR
//! AV1 specific sequence header information
/****************************************************************/
typedef struct {
unsigned int max_width;
unsigned int max_height;
unsigned char reserved[1016];
} CUVIDAV1SEQHDR;
/****************************************************************/
//! \ingroup STRUCTS
//! \struct CUVIDEOFORMATEX
//! Video format including raw sequence header information
//! Used in cuvidGetSourceVideoFormat API
/****************************************************************/
typedef struct
{
CUVIDEOFORMAT format; /**< OUT: CUVIDEOFORMAT structure */
union {
CUVIDAV1SEQHDR av1;
unsigned char raw_seqhdr_data[1024]; /**< OUT: Sequence header data */
};
} CUVIDEOFORMATEX;
/****************************************************************/
//! \ingroup STRUCTS
//! \struct CUAUDIOFORMAT
//! Audio formats
//! Used in cuvidGetSourceAudioFormat API
/****************************************************************/
typedef struct
{
cudaAudioCodec codec; /**< OUT: Compression format */
unsigned int channels; /**< OUT: number of audio channels */
unsigned int samplespersec; /**< OUT: sampling frequency */
unsigned int bitrate; /**< OUT: For uncompressed, can also be used to determine bits per sample */
unsigned int reserved1; /**< Reserved for future use */
unsigned int reserved2; /**< Reserved for future use */
} CUAUDIOFORMAT;
/***************************************************************/
//! \enum CUvideopacketflags
//! Data packet flags
//! Used in CUVIDSOURCEDATAPACKET structure
/***************************************************************/
typedef enum {
CUVID_PKT_ENDOFSTREAM = 0x01, /**< Set when this is the last packet for this stream */
CUVID_PKT_TIMESTAMP = 0x02, /**< Timestamp is valid */
CUVID_PKT_DISCONTINUITY = 0x04, /**< Set when a discontinuity has to be signalled */
CUVID_PKT_ENDOFPICTURE = 0x08, /**< Set when the packet contains exactly one frame or one field */
CUVID_PKT_NOTIFY_EOS = 0x10, /**< If this flag is set along with CUVID_PKT_ENDOFSTREAM, an additional (dummy)
display callback will be invoked with null value of CUVIDPARSERDISPINFO which
should be interpreted as end of the stream. */
} CUvideopacketflags;
/*****************************************************************************/
//! \ingroup STRUCTS
//! \struct CUVIDSOURCEDATAPACKET
//! Data Packet
//! Used in cuvidParseVideoData API
//! IN for cuvidParseVideoData
/*****************************************************************************/
typedef struct _CUVIDSOURCEDATAPACKET
{
unsigned long flags; /**< IN: Combination of CUVID_PKT_XXX flags */
unsigned long payload_size; /**< IN: number of bytes in the payload (may be zero if EOS flag is set) */
const unsigned char *payload; /**< IN: Pointer to packet payload data (may be NULL if EOS flag is set) */
CUvideotimestamp timestamp; /**< IN: Presentation time stamp (10MHz clock), only valid if
CUVID_PKT_TIMESTAMP flag is set */
} CUVIDSOURCEDATAPACKET;
// Callback for packet delivery
typedef int (CUDAAPI *PFNVIDSOURCECALLBACK)(void *, CUVIDSOURCEDATAPACKET *);
/**************************************************************************************************************************/
//! \ingroup STRUCTS
//! \struct CUVIDSOURCEPARAMS
//! Describes parameters needed in cuvidCreateVideoSource API
//! NVDECODE API is intended for HW accelerated video decoding so CUvideosource doesn't have audio demuxer for all supported
//! containers. It's recommended to clients to use their own or third party demuxer if audio support is needed.
/**************************************************************************************************************************/
typedef struct _CUVIDSOURCEPARAMS
{
unsigned int ulClockRate; /**< IN: Time stamp units in Hz (0=default=10000000Hz) */
unsigned int bAnnexb : 1; /**< IN: AV1 annexB stream */
unsigned int uReserved : 31; /**< Reserved for future use - set to zero */
unsigned int uReserved1[6]; /**< Reserved for future use - set to zero */
void *pUserData; /**< IN: User private data passed in to the data handlers */
PFNVIDSOURCECALLBACK pfnVideoDataHandler; /**< IN: Called to deliver video packets */
PFNVIDSOURCECALLBACK pfnAudioDataHandler; /**< IN: Called to deliver audio packets. */
void *pvReserved2[8]; /**< Reserved for future use - set to NULL */
} CUVIDSOURCEPARAMS;
/**********************************************/
//! \ingroup ENUMS
//! \enum CUvideosourceformat_flags
//! CUvideosourceformat_flags
//! Used in cuvidGetSourceVideoFormat API
/**********************************************/
typedef enum {
CUVID_FMT_EXTFORMATINFO = 0x100 /**< Return extended format structure (CUVIDEOFORMATEX) */
} CUvideosourceformat_flags;
#if !defined(__APPLE__)
/***************************************************************************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidCreateVideoSource(CUvideosource *pObj, const char *pszFileName, CUVIDSOURCEPARAMS *pParams)
//! Create CUvideosource object. CUvideosource spawns demultiplexer thread that provides two callbacks:
//! pfnVideoDataHandler() and pfnAudioDataHandler()
//! NVDECODE API is intended for HW accelerated video decoding so CUvideosource doesn't have audio demuxer for all supported
//! containers. It's recommended to clients to use their own or third party demuxer if audio support is needed.
/***************************************************************************************************************************/
CUresult CUDAAPI cuvidCreateVideoSource(CUvideosource *pObj, const char *pszFileName, CUVIDSOURCEPARAMS *pParams);
/***************************************************************************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidCreateVideoSourceW(CUvideosource *pObj, const wchar_t *pwszFileName, CUVIDSOURCEPARAMS *pParams)
//! Create video source
/***************************************************************************************************************************/
CUresult CUDAAPI cuvidCreateVideoSourceW(CUvideosource *pObj, const wchar_t *pwszFileName, CUVIDSOURCEPARAMS *pParams);
/********************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidDestroyVideoSource(CUvideosource obj)
//! Destroy video source
/********************************************************************/
CUresult CUDAAPI cuvidDestroyVideoSource(CUvideosource obj);
/******************************************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidSetVideoSourceState(CUvideosource obj, cudaVideoState state)
//! Set video source state to:
//! cudaVideoState_Started - to signal the source to run and deliver data
//! cudaVideoState_Stopped - to stop the source from delivering the data
//! cudaVideoState_Error - invalid source
/******************************************************************************************/
CUresult CUDAAPI cuvidSetVideoSourceState(CUvideosource obj, cudaVideoState state);
/******************************************************************************************/
//! \ingroup FUNCTS
//! \fn cudaVideoState CUDAAPI cuvidGetVideoSourceState(CUvideosource obj)
//! Get video source state
//! Returns:
//! cudaVideoState_Started - if Source is running and delivering data
//! cudaVideoState_Stopped - if Source is stopped or reached end-of-stream
//! cudaVideoState_Error - if Source is in error state
/******************************************************************************************/
cudaVideoState CUDAAPI cuvidGetVideoSourceState(CUvideosource obj);
/******************************************************************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidGetSourceVideoFormat(CUvideosource obj, CUVIDEOFORMAT *pvidfmt, unsigned int flags)
//! Gets video source format in pvidfmt, flags is set to combination of CUvideosourceformat_flags as per requirement
/******************************************************************************************************************/
CUresult CUDAAPI cuvidGetSourceVideoFormat(CUvideosource obj, CUVIDEOFORMAT *pvidfmt, unsigned int flags);
/**************************************************************************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidGetSourceAudioFormat(CUvideosource obj, CUAUDIOFORMAT *paudfmt, unsigned int flags)
//! Get audio source format
//! NVDECODE API is intended for HW accelerated video decoding so CUvideosource doesn't have audio demuxer for all supported
//! containers. It's recommended to clients to use their own or third party demuxer if audio support is needed.
/**************************************************************************************************************************/
CUresult CUDAAPI cuvidGetSourceAudioFormat(CUvideosource obj, CUAUDIOFORMAT *paudfmt, unsigned int flags);
#endif
/**********************************************************************************/
//! \ingroup STRUCTS
//! \struct CUVIDPARSERDISPINFO
//! Used in cuvidParseVideoData API with PFNVIDDISPLAYCALLBACK pfnDisplayPicture
/**********************************************************************************/
typedef struct _CUVIDPARSERDISPINFO
{
int picture_index; /**< OUT: Index of the current picture */
int progressive_frame; /**< OUT: 1 if progressive frame; 0 otherwise */
int top_field_first; /**< OUT: 1 if top field is displayed first; 0 otherwise */
int repeat_first_field; /**< OUT: Number of additional fields (1=ivtc, 2=frame doubling, 4=frame tripling,
-1=unpaired field) */
CUvideotimestamp timestamp; /**< OUT: Presentation time stamp */
} CUVIDPARSERDISPINFO;
/***********************************************************************************************************************/
//! Parser callbacks
//! The parser will call these synchronously from within cuvidParseVideoData(), whenever there is sequence change or a picture
//! is ready to be decoded and/or displayed. First argument in functions is "void *pUserData" member of structure CUVIDSOURCEPARAMS
//! Return values from these callbacks are interpreted as below. If the callbacks return failure, it will be propagated by
//! cuvidParseVideoData() to the application.
//! Parser picks default operating point as 0 and outputAllLayers flag as 0 if PFNVIDOPPOINTCALLBACK is not set or return value is
//! -1 or invalid operating point.
//! PFNVIDSEQUENCECALLBACK : 0: fail, 1: succeeded, > 1: override dpb size of parser (set by CUVIDPARSERPARAMS::ulMaxNumDecodeSurfaces
//! while creating parser)
//! PFNVIDDECODECALLBACK : 0: fail, >=1: succeeded
//! PFNVIDDISPLAYCALLBACK : 0: fail, >=1: succeeded
//! PFNVIDOPPOINTCALLBACK : <0: fail, >=0: succeeded (bit 0-9: OperatingPoint, bit 10-10: outputAllLayers, bit 11-30: reserved)
//! PFNVIDSEIMSGCALLBACK : 0: fail, >=1: succeeded
/***********************************************************************************************************************/
typedef int (CUDAAPI *PFNVIDSEQUENCECALLBACK)(void *, CUVIDEOFORMAT *);
typedef int (CUDAAPI *PFNVIDDECODECALLBACK)(void *, CUVIDPICPARAMS *);
typedef int (CUDAAPI *PFNVIDDISPLAYCALLBACK)(void *, CUVIDPARSERDISPINFO *);
typedef int (CUDAAPI *PFNVIDOPPOINTCALLBACK)(void *, CUVIDOPERATINGPOINTINFO*);
typedef int (CUDAAPI *PFNVIDSEIMSGCALLBACK) (void *, CUVIDSEIMESSAGEINFO *);
/**************************************/
//! \ingroup STRUCTS
//! \struct CUVIDPARSERPARAMS
//! Used in cuvidCreateVideoParser API
/**************************************/
typedef struct _CUVIDPARSERPARAMS
{
cudaVideoCodec CodecType; /**< IN: cudaVideoCodec_XXX */
unsigned int ulMaxNumDecodeSurfaces; /**< IN: Max # of decode surfaces (parser will cycle through these) */
unsigned int ulClockRate; /**< IN: Timestamp units in Hz (0=default=10000000Hz) */
unsigned int ulErrorThreshold; /**< IN: % Error threshold (0-100) for calling pfnDecodePicture (100=always
IN: call pfnDecodePicture even if picture bitstream is fully corrupted) */
unsigned int ulMaxDisplayDelay; /**< IN: Max display queue delay (improves pipelining of decode with display)
0=no delay (recommended values: 2..4) */
unsigned int bAnnexb : 1; /**< IN: AV1 annexB stream */
unsigned int uReserved : 31; /**< Reserved for future use - set to zero */
unsigned int uReserved1[4]; /**< IN: Reserved for future use - set to 0 */
void *pUserData; /**< IN: User data for callbacks */
PFNVIDSEQUENCECALLBACK pfnSequenceCallback; /**< IN: Called before decoding frames and/or whenever there is a fmt change */
PFNVIDDECODECALLBACK pfnDecodePicture; /**< IN: Called when a picture is ready to be decoded (decode order) */
PFNVIDDISPLAYCALLBACK pfnDisplayPicture; /**< IN: Called whenever a picture is ready to be displayed (display order) */
PFNVIDOPPOINTCALLBACK pfnGetOperatingPoint; /**< IN: Called from AV1 sequence header to get operating point of a AV1
scalable bitstream */
PFNVIDSEIMSGCALLBACK pfnGetSEIMsg; /**< IN: Called when all SEI messages are parsed for particular frame */
void *pvReserved2[5]; /**< Reserved for future use - set to NULL */
CUVIDEOFORMATEX *pExtVideoInfo; /**< IN: [Optional] sequence header data from system layer */
} CUVIDPARSERPARAMS;
/************************************************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidCreateVideoParser(CUvideoparser *pObj, CUVIDPARSERPARAMS *pParams)
//! Create video parser object and initialize
/************************************************************************************************/
CUresult CUDAAPI cuvidCreateVideoParser(CUvideoparser *pObj, CUVIDPARSERPARAMS *pParams);
/************************************************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidParseVideoData(CUvideoparser obj, CUVIDSOURCEDATAPACKET *pPacket)
//! Parse the video data from source data packet in pPacket
//! Extracts parameter sets like SPS, PPS, bitstream etc. from pPacket and
//! calls back pfnDecodePicture with CUVIDPICPARAMS data for kicking of HW decoding
//! calls back pfnSequenceCallback with CUVIDEOFORMAT data for initial sequence header or when
//! the decoder encounters a video format change
//! calls back pfnDisplayPicture with CUVIDPARSERDISPINFO data to display a video frame
/************************************************************************************************/
CUresult CUDAAPI cuvidParseVideoData(CUvideoparser obj, CUVIDSOURCEDATAPACKET *pPacket);
/************************************************************************************************/
//! \ingroup FUNCTS
//! \fn CUresult CUDAAPI cuvidDestroyVideoParser(CUvideoparser obj)
//! Destroy the video parser
/************************************************************************************************/
CUresult CUDAAPI cuvidDestroyVideoParser(CUvideoparser obj);
/**********************************************************************************************/
#if defined(__cplusplus)
}
#endif /* __cplusplus */
#endif // __NVCUVID_H__
@@ -0,0 +1,201 @@
#include "cuda_context.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#if defined(WIN32)
#include <windows.h>
#else
#include <dlfcn.h>
#endif
#include <iostream>
#if defined(WIN32)
static const char CUDA_DYNAMIC_LIBRARY[] = "nvcuda.dll";
#else
static const char CUDA_DYNAMIC_LIBRARY[] = "libcuda.so.1";
#endif
namespace livekit_ffi {
#define __CUCTX_CUDA_CALL(call, ret) \
CUresult err__ = call; \
if (err__ != CUDA_SUCCESS) { \
const char* szErrName = NULL; \
cuGetErrorName(err__, &szErrName); \
RTC_LOG(LS_ERROR) << "CudaContext error " << szErrName; \
return ret; \
}
#define CUCTX_CUDA_CALL_ERROR(call) \
do { \
__CUCTX_CUDA_CALL(call, err__); \
} while (0)
static void* s_module_ptr = nullptr;
static const int kRequiredDriverVersion = 11000;
static bool load_cuda_modules() {
if (s_module_ptr)
return true;
#if defined(WIN32)
// dll delay load
HMODULE module = LoadLibrary(TEXT("nvcuda.dll"));
if (!module) {
RTC_LOG(LS_INFO) << "nvcuda.dll is not found.";
return false;
}
s_module_ptr = module;
#elif defined(__linux__)
s_module_ptr = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_GLOBAL);
if (!s_module_ptr)
return false;
// Close handle immediately because going to call `dlopen` again
// in the implib module when cuda api called on Linux.
dlclose(s_module_ptr);
s_module_ptr = nullptr;
#endif
return true;
}
static bool check_cuda_device() {
int device_count = 0;
int driver_version = 0;
CUCTX_CUDA_CALL_ERROR(cuDriverGetVersion(&driver_version));
if (kRequiredDriverVersion > driver_version) {
RTC_LOG(LS_ERROR)
<< "CUDA driver version is not higher than the required version. "
<< driver_version;
return false;
}
CUresult result = cuInit(0);
if (result != CUDA_SUCCESS) {
RTC_LOG(LS_ERROR) << "Failed to initialize CUDA.";
return false;
}
result = cuDeviceGetCount(&device_count);
if (result != CUDA_SUCCESS) {
RTC_LOG(LS_ERROR) << "Failed to get CUDA device count.";
return false;
}
if (device_count == 0) {
RTC_LOG(LS_ERROR) << "No CUDA devices found.";
return false;
}
return true;
}
CudaContext* CudaContext::GetInstance() {
static CudaContext instance;
return &instance;
}
bool CudaContext::IsAvailable() {
return load_cuda_modules() && check_cuda_device();
}
bool CudaContext::Initialize() {
// Initialize CUDA context
bool success = load_cuda_modules();
if (!success) {
RTC_LOG(LS_ERROR) << "Failed to load CUDA modules. maybe the NVIDIA driver "
"is not installed?";
return false;
}
int num_devices = 0;
CUdevice cu_device = 0;
CUcontext context = nullptr;
int driverVersion = 0;
CUCTX_CUDA_CALL_ERROR(cuDriverGetVersion(&driverVersion));
if (kRequiredDriverVersion > driverVersion) {
RTC_LOG(LS_ERROR)
<< "CUDA driver version is not higher than the required version. "
<< driverVersion;
return false;
}
CUresult result = cuInit(0);
if (result != CUDA_SUCCESS) {
RTC_LOG(LS_ERROR) << "Failed to initialize CUDA.";
return false;
}
result = cuDeviceGetCount(&num_devices);
if (result != CUDA_SUCCESS) {
RTC_LOG(LS_ERROR) << "Failed to get CUDA device count.";
return false;
}
if (num_devices == 0) {
RTC_LOG(LS_ERROR) << "No CUDA devices found.";
return false;
}
CUCTX_CUDA_CALL_ERROR(cuDeviceGet(&cu_device, 0));
char device_name[80];
CUCTX_CUDA_CALL_ERROR(
cuDeviceGetName(device_name, sizeof(device_name), cu_device));
RTC_LOG(LS_INFO) << "CUDA device name: " << device_name;
#if CUDA_VERSION >= 13000
CUCTX_CUDA_CALL_ERROR(cuCtxCreate(&context, nullptr, 0, cu_device));
#else
CUCTX_CUDA_CALL_ERROR(cuCtxCreate(&context, 0, cu_device));
#endif
if (context == nullptr) {
RTC_LOG(LS_ERROR) << "Failed to create CUDA context.";
return false;
}
cu_device_ = cu_device;
cu_context_ = context;
return true;
}
CUcontext CudaContext::GetContext() const {
RTC_DCHECK(cu_context_ != nullptr);
// Ensure the context is current
CUcontext current;
if (cuCtxGetCurrent(&current) != CUDA_SUCCESS) {
throw;
}
if (cu_context_ == current) {
return cu_context_;
}
if (cuCtxSetCurrent(cu_context_) != CUDA_SUCCESS) {
throw;
}
return cu_context_;
}
void CudaContext::Shutdown() {
// Shutdown CUDA context
if (cu_context_) {
cuCtxDestroy(cu_context_);
cu_context_ = nullptr;
}
if (s_module_ptr) {
#if defined(WIN32)
FreeLibrary((HMODULE)s_module_ptr);
#elif defined(__linux__)
dlclose(s_module_ptr);
#endif
s_module_ptr = nullptr;
}
}
} // namespace livekit_ffi
@@ -0,0 +1,29 @@
#ifndef WEBRTC_SYS_NVIDIA_CUDA_CONTEXT_H
#define WEBRTC_SYS_NVIDIA_CUDA_CONTEXT_H
#include <cuda.h>
namespace livekit_ffi {
class CudaContext {
public:
CudaContext() = default;
~CudaContext() = default;
static bool IsAvailable();
static CudaContext* GetInstance();
bool Initialize();
bool IsInitialized() const { return cu_context_ != nullptr; }
CUcontext GetContext() const;
void Shutdown();
private:
CUdevice cu_device_ = 0;
CUcontext cu_context_ = nullptr;
};
} // namespace livekit_ffi
#endif // WEBRTC_SYS_NVIDIA_CUDA_CONTEXT_H
@@ -0,0 +1,240 @@
#if defined(_WIN32)
#include "cuda.h"
#include <windows.h>
namespace {
HMODULE CudaModule() {
static HMODULE module = LoadLibraryA("nvcuda.dll");
return module;
}
FARPROC CudaProc(const char* name) {
HMODULE module = CudaModule();
if (!module) {
return nullptr;
}
return GetProcAddress(module, name);
}
template <typename Fn>
Fn Resolve(const char* primary, const char* fallback = nullptr) {
FARPROC proc = CudaProc(primary);
if (!proc && fallback) {
proc = CudaProc(fallback);
}
return reinterpret_cast<Fn>(proc);
}
template <typename Fn, typename... Args>
CUresult Call(Fn fn, Args... args) {
if (!fn) {
return CUDA_ERROR_NOT_INITIALIZED;
}
return fn(args...);
}
} // namespace
extern "C" {
CUresult CUDAAPI cuInit(unsigned int flags) {
using Fn = CUresult(CUDAAPI*)(unsigned int);
return Call(Resolve<Fn>("cuInit"), flags);
}
CUresult CUDAAPI cuDriverGetVersion(int* driverVersion) {
using Fn = CUresult(CUDAAPI*)(int*);
return Call(Resolve<Fn>("cuDriverGetVersion"), driverVersion);
}
CUresult CUDAAPI cuGetErrorName(CUresult error, const char** pStr) {
using Fn = CUresult(CUDAAPI*)(CUresult, const char**);
Fn fn = Resolve<Fn>("cuGetErrorName");
if (fn) {
return fn(error, pStr);
}
if (pStr) {
*pStr = "CUDA driver API unavailable";
}
return CUDA_ERROR_NOT_INITIALIZED;
}
CUresult CUDAAPI cuDeviceGetCount(int* count) {
using Fn = CUresult(CUDAAPI*)(int*);
return Call(Resolve<Fn>("cuDeviceGetCount"), count);
}
CUresult CUDAAPI cuDeviceGet(CUdevice* device, int ordinal) {
using Fn = CUresult(CUDAAPI*)(CUdevice*, int);
return Call(Resolve<Fn>("cuDeviceGet"), device, ordinal);
}
CUresult CUDAAPI cuDeviceGetName(char* name, int len, CUdevice dev) {
using Fn = CUresult(CUDAAPI*)(char*, int, CUdevice);
return Call(Resolve<Fn>("cuDeviceGetName"), name, len, dev);
}
CUresult CUDAAPI cuDeviceGetAttribute(int* pi,
CUdevice_attribute attrib,
CUdevice dev) {
using Fn = CUresult(CUDAAPI*)(int*, CUdevice_attribute, CUdevice);
return Call(Resolve<Fn>("cuDeviceGetAttribute"), pi, attrib, dev);
}
CUresult CUDAAPI cuCtxCreate(CUcontext* pctx,
unsigned int flags,
CUdevice dev) {
using Fn = CUresult(CUDAAPI*)(CUcontext*, unsigned int, CUdevice);
return Call(Resolve<Fn>("cuCtxCreate_v2", "cuCtxCreate"), pctx, flags, dev);
}
CUresult CUDAAPI cuCtxCreate_v2(CUcontext* pctx,
unsigned int flags,
CUdevice dev) {
return cuCtxCreate(pctx, flags, dev);
}
CUresult CUDAAPI cuCtxDestroy(CUcontext ctx) {
using Fn = CUresult(CUDAAPI*)(CUcontext);
return Call(Resolve<Fn>("cuCtxDestroy_v2", "cuCtxDestroy"), ctx);
}
CUresult CUDAAPI cuCtxDestroy_v2(CUcontext ctx) {
return cuCtxDestroy(ctx);
}
CUresult CUDAAPI cuCtxGetCurrent(CUcontext* pctx) {
using Fn = CUresult(CUDAAPI*)(CUcontext*);
return Call(Resolve<Fn>("cuCtxGetCurrent"), pctx);
}
CUresult CUDAAPI cuCtxSetCurrent(CUcontext ctx) {
using Fn = CUresult(CUDAAPI*)(CUcontext);
return Call(Resolve<Fn>("cuCtxSetCurrent"), ctx);
}
CUresult CUDAAPI cuCtxGetDevice(CUdevice* device) {
using Fn = CUresult(CUDAAPI*)(CUdevice*);
return Call(Resolve<Fn>("cuCtxGetDevice"), device);
}
CUresult CUDAAPI cuCtxPushCurrent(CUcontext ctx) {
using Fn = CUresult(CUDAAPI*)(CUcontext);
return Call(Resolve<Fn>("cuCtxPushCurrent_v2", "cuCtxPushCurrent"), ctx);
}
CUresult CUDAAPI cuCtxPushCurrent_v2(CUcontext ctx) {
return cuCtxPushCurrent(ctx);
}
CUresult CUDAAPI cuCtxPopCurrent(CUcontext* pctx) {
using Fn = CUresult(CUDAAPI*)(CUcontext*);
return Call(Resolve<Fn>("cuCtxPopCurrent_v2", "cuCtxPopCurrent"), pctx);
}
CUresult CUDAAPI cuCtxPopCurrent_v2(CUcontext* pctx) {
return cuCtxPopCurrent(pctx);
}
CUresult CUDAAPI cuMemAlloc(CUdeviceptr* dptr, size_t bytesize) {
using Fn = CUresult(CUDAAPI*)(CUdeviceptr*, size_t);
return Call(Resolve<Fn>("cuMemAlloc_v2", "cuMemAlloc"), dptr, bytesize);
}
CUresult CUDAAPI cuMemAllocPitch(CUdeviceptr* dptr,
size_t* pPitch,
size_t WidthInBytes,
size_t Height,
unsigned int ElementSizeBytes) {
using Fn = CUresult(CUDAAPI*)(CUdeviceptr*, size_t*, size_t, size_t,
unsigned int);
return Call(Resolve<Fn>("cuMemAllocPitch_v2", "cuMemAllocPitch"), dptr,
pPitch, WidthInBytes, Height, ElementSizeBytes);
}
CUresult CUDAAPI cuMemFree(CUdeviceptr dptr) {
using Fn = CUresult(CUDAAPI*)(CUdeviceptr);
return Call(Resolve<Fn>("cuMemFree_v2", "cuMemFree"), dptr);
}
CUresult CUDAAPI cuMemFree_v2(CUdeviceptr dptr) {
return cuMemFree(dptr);
}
CUresult CUDAAPI cuMemcpy2D(const CUDA_MEMCPY2D* pCopy) {
using Fn = CUresult(CUDAAPI*)(const CUDA_MEMCPY2D*);
return Call(Resolve<Fn>("cuMemcpy2D_v2", "cuMemcpy2D"), pCopy);
}
CUresult CUDAAPI cuMemcpy2D_v2(const CUDA_MEMCPY2D* pCopy) {
return cuMemcpy2D(pCopy);
}
CUresult CUDAAPI cuMemcpy2DUnaligned(const CUDA_MEMCPY2D* pCopy) {
using Fn = CUresult(CUDAAPI*)(const CUDA_MEMCPY2D*);
return Call(Resolve<Fn>("cuMemcpy2DUnaligned_v2",
"cuMemcpy2DUnaligned"),
pCopy);
}
CUresult CUDAAPI cuMemcpy2DUnaligned_v2(const CUDA_MEMCPY2D* pCopy) {
return cuMemcpy2DUnaligned(pCopy);
}
CUresult CUDAAPI cuMemcpy2DAsync(const CUDA_MEMCPY2D* pCopy,
CUstream hStream) {
using Fn = CUresult(CUDAAPI*)(const CUDA_MEMCPY2D*, CUstream);
return Call(Resolve<Fn>("cuMemcpy2DAsync_v2", "cuMemcpy2DAsync"), pCopy,
hStream);
}
CUresult CUDAAPI cuMemcpy2DAsync_v2(const CUDA_MEMCPY2D* pCopy,
CUstream hStream) {
return cuMemcpy2DAsync(pCopy, hStream);
}
CUresult CUDAAPI cuStreamCreate(CUstream* phStream, unsigned int Flags) {
using Fn = CUresult(CUDAAPI*)(CUstream*, unsigned int);
return Call(Resolve<Fn>("cuStreamCreate"), phStream, Flags);
}
CUresult CUDAAPI cuStreamSynchronize(CUstream hStream) {
using Fn = CUresult(CUDAAPI*)(CUstream);
return Call(Resolve<Fn>("cuStreamSynchronize"), hStream);
}
CUresult CUDAAPI cuArrayDestroy(CUarray hArray) {
using Fn = CUresult(CUDAAPI*)(CUarray);
return Call(Resolve<Fn>("cuArrayDestroy_v2", "cuArrayDestroy"), hArray);
}
CUresult CUDAAPI cuGraphicsEGLRegisterImage(CUgraphicsResource* pCudaResource,
void* image,
unsigned int flags) {
using Fn = CUresult(CUDAAPI*)(CUgraphicsResource*, void*, unsigned int);
return Call(Resolve<Fn>("cuGraphicsEGLRegisterImage"), pCudaResource, image,
flags);
}
CUresult CUDAAPI cuGraphicsResourceGetMappedEglFrame(
CUeglFrame* eglFrame,
CUgraphicsResource resource,
unsigned int index,
unsigned int mipLevel) {
using Fn = CUresult(CUDAAPI*)(CUeglFrame*, CUgraphicsResource, unsigned int,
unsigned int);
return Call(Resolve<Fn>("cuGraphicsResourceGetMappedEglFrame"), eglFrame,
resource, index, mipLevel);
}
CUresult CUDAAPI cuGraphicsUnregisterResource(CUgraphicsResource resource) {
using Fn = CUresult(CUDAAPI*)(CUgraphicsResource);
return Call(Resolve<Fn>("cuGraphicsUnregisterResource"), resource);
}
} // extern "C"
#endif // defined(_WIN32)
@@ -0,0 +1,193 @@
#include "h264_decoder_impl.h"
#include <api/video/i420_buffer.h>
#include <api/video/video_codec_type.h>
#include <modules/video_coding/include/video_error_codes.h>
#include <third_party/libyuv/include/libyuv/convert.h>
#include "NvDecoder/NvDecoder.h"
#include "Utils/NvCodecUtils.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
ColorSpace ExtractH264ColorSpace(const CUVIDEOFORMAT& format) {
return ColorSpace(
static_cast<ColorSpace::PrimaryID>(
format.video_signal_description.color_primaries),
static_cast<ColorSpace::TransferID>(
format.video_signal_description.transfer_characteristics),
static_cast<ColorSpace::MatrixID>(
format.video_signal_description.matrix_coefficients),
static_cast<ColorSpace::RangeID>(
format.video_signal_description.video_full_range_flag));
}
NvidiaH264DecoderImpl::NvidiaH264DecoderImpl(CUcontext context)
: cu_context_(context),
decoder_(nullptr),
is_configured_decoder_(false),
decoded_complete_callback_(nullptr),
buffer_pool_(false) {}
NvidiaH264DecoderImpl::~NvidiaH264DecoderImpl() {
Release();
}
VideoDecoder::DecoderInfo NvidiaH264DecoderImpl::GetDecoderInfo() const {
VideoDecoder::DecoderInfo info;
info.implementation_name = "NVIDIA H264 Decoder";
info.is_hardware_accelerated = true;
return info;
}
bool NvidiaH264DecoderImpl::Configure(const Settings& settings) {
if (settings.codec_type() != kVideoCodecH264) {
RTC_LOG(LS_ERROR)
<< "initialization failed on codectype is not kVideoCodecH264";
return false;
}
if (!settings.max_render_resolution().Valid()) {
RTC_LOG(LS_ERROR)
<< "initialization failed on codec_settings width < 0 or height < 0";
return false;
}
settings_ = settings;
const CUresult result = cuCtxSetCurrent(cu_context_);
if (!ck(result)) {
RTC_LOG(LS_ERROR) << "initialization failed on cuCtxSetCurrent result"
<< result;
return false;
}
// todo(kazuki): Max resolution is differred each architecture.
// Refer to the table in Video Decoder Capabilities.
// https://docs.nvidia.com/video-technologies/video-codec-sdk/nvdec-video-decoder-api-prog-guide
int maxWidth = 4096;
int maxHeight = 4096;
// bUseDeviceFrame: allocate in memory or cuda device memory
decoder_ = std::make_unique<NvDecoder>(
cu_context_, false, cudaVideoCodec_H264, true, false, nullptr, nullptr,
false, maxWidth, maxHeight);
return true;
}
int32_t NvidiaH264DecoderImpl::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
this->decoded_complete_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t NvidiaH264DecoderImpl::Release() {
buffer_pool_.Release();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t NvidiaH264DecoderImpl::Decode(const EncodedImage& input_image,
bool missing_frames,
int64_t render_time_ms) {
CUcontext current;
if (!ck(cuCtxGetCurrent(&current))) {
RTC_LOG(LS_ERROR) << "decode failed on cuCtxGetCurrent is failed";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (current != cu_context_) {
RTC_LOG(LS_ERROR)
<< "decode failed on not match current context and hold context";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (decoded_complete_callback_ == nullptr) {
RTC_LOG(LS_ERROR) << "decode failed on not set m_decodedCompleteCallback";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (!input_image.data() || !input_image.size()) {
RTC_LOG(LS_ERROR) << "decode failed on input image is null";
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
h264_bitstream_parser_.ParseBitstream(input_image);
std::optional<int> qp = h264_bitstream_parser_.GetLastSliceQp();
std::optional<SpsParser::SpsState> sps = h264_bitstream_parser_.sps();
if (is_configured_decoder_) {
if (!sps ||
sps.value().width != static_cast<uint32_t>(decoder_->GetWidth()) ||
sps.value().height != static_cast<uint32_t>(decoder_->GetHeight())) {
decoder_->setReconfigParams(nullptr, nullptr);
}
}
int nFrameReturnd = 0;
try {
do {
nFrameReturnd = decoder_->Decode(
input_image.data(), static_cast<int>(input_image.size()),
CUVID_PKT_TIMESTAMP, input_image.RtpTimestamp());
} while (nFrameReturnd == 0);
} catch (const NVDECException& e) {
RTC_LOG(LS_ERROR) << "NVDEC H264 decode failed: " << e.what();
decoder_.reset();
is_configured_decoder_ = false;
return WEBRTC_VIDEO_CODEC_ERROR;
}
is_configured_decoder_ = true;
// todo: support other output format
// Chromium's H264 Encoder is output on NV12, so currently only NV12 is
// supported.
if (decoder_->GetOutputFormat() != cudaVideoSurfaceFormat_NV12) {
RTC_LOG(LS_ERROR) << "not supported this format: "
<< decoder_->GetOutputFormat();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// Pass on color space from input frame if explicitly specified.
const ColorSpace& color_space =
input_image.ColorSpace()
? *input_image.ColorSpace()
: ExtractH264ColorSpace(decoder_->GetVideoFormatInfo());
for (int i = 0; i < nFrameReturnd; i++) {
int64_t timeStamp;
uint8_t* pFrame = decoder_->GetFrame(&timeStamp);
webrtc::scoped_refptr<webrtc::I420Buffer> i420_buffer =
buffer_pool_.CreateI420Buffer(decoder_->GetWidth(),
decoder_->GetHeight());
int result;
{
result = libyuv::NV12ToI420(
pFrame, decoder_->GetDeviceFramePitch(),
pFrame + decoder_->GetHeight() * decoder_->GetDeviceFramePitch(),
decoder_->GetDeviceFramePitch(), i420_buffer->MutableDataY(),
i420_buffer->StrideY(), i420_buffer->MutableDataU(),
i420_buffer->StrideU(), i420_buffer->MutableDataV(),
i420_buffer->StrideV(), decoder_->GetWidth(), decoder_->GetHeight());
}
if (result) {
RTC_LOG(LS_INFO) << "libyuv::NV12ToI420 failed. error:" << result;
}
VideoFrame decoded_frame =
VideoFrame::Builder()
.set_video_frame_buffer(i420_buffer)
.set_timestamp_rtp(static_cast<uint32_t>(timeStamp))
.set_color_space(color_space)
.build();
// todo: measurement decoding time
std::optional<int32_t> decodetime;
decoded_complete_callback_->Decoded(decoded_frame, decodetime, qp);
}
return WEBRTC_VIDEO_CODEC_OK;
}
} // end namespace webrtc
@@ -0,0 +1,57 @@
#ifndef WEBRTC_NVIDIA_H264_DECODER_IMPL_H_
#define WEBRTC_NVIDIA_H264_DECODER_IMPL_H_
#include <api/video_codecs/h264_profile_level_id.h>
#include <api/video_codecs/sdp_video_format.h>
#include <api/video_codecs/video_decoder.h>
#include <api/video_codecs/video_decoder_factory.h>
#include <api/video_codecs/video_encoder.h>
#include <api/video_codecs/video_encoder_factory.h>
#include <common_video/h264/h264_bitstream_parser.h>
#include <common_video/h264/pps_parser.h>
#include <common_video/h264/sps_parser.h>
#include <common_video/include/video_frame_buffer_pool.h>
#include <cuda.h>
#include <media/base/codec.h>
#include "NvDecoder/NvDecoder.h"
namespace webrtc {
class H264BitstreamParserEx : public ::webrtc::H264BitstreamParser {
public:
std::optional<SpsParser::SpsState> sps() { return sps_; }
std::optional<PpsParser::PpsState> pps() { return pps_; }
};
class NvidiaH264DecoderImpl : public VideoDecoder {
public:
NvidiaH264DecoderImpl(CUcontext context);
NvidiaH264DecoderImpl(const NvidiaH264DecoderImpl&) = delete;
NvidiaH264DecoderImpl& operator=(const NvidiaH264DecoderImpl&) = delete;
~NvidiaH264DecoderImpl() override;
bool Configure(const Settings& settings) override;
int32_t Decode(const EncodedImage& input_image,
bool missing_frames,
int64_t render_time_ms) override;
int32_t RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) override;
int32_t Release() override;
DecoderInfo GetDecoderInfo() const override;
private:
CUcontext cu_context_;
std::unique_ptr<NvDecoder> decoder_;
bool is_configured_decoder_;
Settings settings_;
DecodedImageCallback* decoded_complete_callback_ = nullptr;
webrtc::VideoFrameBufferPool buffer_pool_;
H264BitstreamParserEx h264_bitstream_parser_;
};
} // end namespace webrtc
#endif // WEBRTC_NVIDIA_H264_DECODER_IMPL_H_
@@ -0,0 +1,550 @@
#include "h264_encoder_impl.h"
#include <algorithm>
#include <cstring>
#include <limits>
#include <string>
#include <vector>
#include "absl/strings/match.h"
#include "absl/types/optional.h"
#include "api/video/i420_buffer.h"
#include "api/video/nv12_buffer.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/scalability_mode.h"
#include <common_video/h264/h264_common.h>
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/svc/create_scalability_structure.h"
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
#include "modules/video_coding/utility/simulcast_utility.h"
#include "native_gpu_encode_bridge.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/metrics.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/scale.h"
namespace webrtc {
// Used by histograms. Values of entries should not be changed.
enum H264EncoderImplEvent {
kH264EncoderEventInit = 0,
kH264EncoderEventError = 1,
kH264EncoderEventMax = 16,
};
namespace {
struct Nv12HostFrame {
const uint8_t* data = nullptr;
uint32_t stride = 0;
std::vector<uint8_t> owned;
};
bool PrepareNv12HostFrame(const VideoFrame& input_frame,
Nv12HostFrame* out) {
const auto input_buffer = input_frame.video_frame_buffer();
if (!input_buffer) {
return false;
}
const int width = input_frame.width();
const int height = input_frame.height();
if (width < 2 || height < 2 || (width % 2) != 0 || (height % 2) != 0) {
return false;
}
if (input_buffer->type() == VideoFrameBuffer::Type::kNV12) {
const NV12BufferInterface* nv12 = input_buffer->GetNV12();
if (!nv12 || nv12->width() != width || nv12->height() != height) {
return false;
}
out->data = nv12->DataY();
out->stride = nv12->StrideY();
return out->data != nullptr && nv12->DataUV() != nullptr &&
nv12->StrideY() == nv12->StrideUV() &&
nv12->DataUV() == nv12->DataY() + nv12->StrideY() * height;
}
webrtc::scoped_refptr<I420BufferInterface> i420 = input_buffer->ToI420();
if (!i420 || i420->width() != width || i420->height() != height) {
return false;
}
const int chroma_width = (width + 1) / 2;
const int chroma_height = (height + 1) / 2;
out->stride = width;
out->owned.assign(width * height + width * chroma_height, 0);
uint8_t* y = out->owned.data();
uint8_t* uv = y + width * height;
for (int row = 0; row < height; ++row) {
memcpy(y + row * width, i420->DataY() + row * i420->StrideY(), width);
}
for (int row = 0; row < chroma_height; ++row) {
const uint8_t* src_u = i420->DataU() + row * i420->StrideU();
const uint8_t* src_v = i420->DataV() + row * i420->StrideV();
uint8_t* dst_uv = uv + row * width;
for (int col = 0; col < chroma_width; ++col) {
dst_uv[col * 2] = src_u[col];
dst_uv[col * 2 + 1] = src_v[col];
}
}
out->data = out->owned.data();
return true;
}
} // namespace
NV_ENC_LEVEL H264LevelToNvEncLevel(webrtc::H264Level level) {
switch (level) {
case H264Level::kLevel1_b:
return NV_ENC_LEVEL_H264_1b;
case H264Level::kLevel1:
return NV_ENC_LEVEL_H264_1;
case H264Level::kLevel1_1:
return NV_ENC_LEVEL_H264_11;
case H264Level::kLevel1_2:
return NV_ENC_LEVEL_H264_12;
case H264Level::kLevel1_3:
return NV_ENC_LEVEL_H264_13;
case H264Level::kLevel2:
return NV_ENC_LEVEL_H264_2;
case H264Level::kLevel2_1:
return NV_ENC_LEVEL_H264_21;
case H264Level::kLevel2_2:
return NV_ENC_LEVEL_H264_22;
case H264Level::kLevel3:
return NV_ENC_LEVEL_H264_3;
case H264Level::kLevel3_1:
return NV_ENC_LEVEL_H264_31;
case H264Level::kLevel3_2:
return NV_ENC_LEVEL_H264_32;
case H264Level::kLevel4:
return NV_ENC_LEVEL_H264_4;
case H264Level::kLevel4_1:
return NV_ENC_LEVEL_H264_41;
case H264Level::kLevel4_2:
return NV_ENC_LEVEL_H264_42;
case H264Level::kLevel5:
return NV_ENC_LEVEL_H264_5;
case H264Level::kLevel5_1:
return NV_ENC_LEVEL_H264_51;
case H264Level::kLevel5_2:
return NV_ENC_LEVEL_H264_52;
}
return NV_ENC_LEVEL_AUTOSELECT; // Default value.
}
NvidiaH264EncoderImpl::NvidiaH264EncoderImpl(
const webrtc::Environment& env,
CUcontext context,
CUmemorytype memory_type,
NV_ENC_BUFFER_FORMAT nv_format,
const SdpVideoFormat& format)
: env_(env),
encoder_(nullptr),
cu_context_(context),
cu_memory_type_(memory_type),
cu_scaled_array_(nullptr),
nv_format_(nv_format),
packetization_mode_(
H264EncoderSettings::Parse(format).packetization_mode),
format_(format) {
std::string hexString = format_.parameters.at("profile-level-id");
std::optional<webrtc::H264ProfileLevelId> profile_level_id =
webrtc::ParseH264ProfileLevelId(hexString.c_str());
if (profile_level_id.has_value()) {
profile_ = profile_level_id->profile;
level_ = profile_level_id->level;
}
nv_enc_level_ = NV_ENC_LEVEL_AUTOSELECT;
if (level_ != H264Level::kLevel1_b) {
// Convert H264Level to NV_ENC_LEVEL.
nv_enc_level_ = webrtc::H264LevelToNvEncLevel(level_);
}
RTC_CHECK_NE(cu_memory_type_, CU_MEMORYTYPE_HOST);
}
NvidiaH264EncoderImpl::~NvidiaH264EncoderImpl() {
Release();
}
void NvidiaH264EncoderImpl::ReportInit() {
if (has_reported_init_)
return;
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
kH264EncoderEventInit, kH264EncoderEventMax);
has_reported_init_ = true;
}
void NvidiaH264EncoderImpl::ReportError() {
if (has_reported_error_)
return;
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event",
kH264EncoderEventError, kH264EncoderEventMax);
has_reported_error_ = true;
}
int32_t NvidiaH264EncoderImpl::InitEncode(
const VideoCodec* inst,
const VideoEncoder::Settings& settings) {
if (!inst || inst->codecType != kVideoCodecH264) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inst->maxFramerate == 0) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (inst->width < 1 || inst->height < 1) {
ReportError();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
int32_t release_ret = Release();
if (release_ret != WEBRTC_VIDEO_CODEC_OK) {
ReportError();
return release_ret;
}
codec_ = *inst;
// Code expects simulcastStream resolutions to be correct, make sure they are
// filled even when there are no simulcast layers.
if (codec_.numberOfSimulcastStreams == 0) {
codec_.simulcastStream[0].width = codec_.width;
codec_.simulcastStream[0].height = codec_.height;
}
// Initialize encoded image. Default buffer size: size of unencoded data.
const size_t new_capacity =
CalcBufferSize(VideoType::kI420, codec_.width, codec_.height);
encoded_image_.SetEncodedData(EncodedImageBuffer::Create(new_capacity));
encoded_image_._encodedWidth = codec_.width;
encoded_image_._encodedHeight = codec_.height;
encoded_image_.set_size(0);
configuration_.sending = false;
configuration_.frame_dropping_on = codec_.GetFrameDropEnabled();
configuration_.key_frame_interval = codec_.H264()->keyFrameInterval;
configuration_.width = codec_.width;
configuration_.height = codec_.height;
configuration_.max_frame_rate = codec_.maxFramerate;
configuration_.target_bps = codec_.startBitrate * 1000;
configuration_.max_bps = codec_.maxBitrate * 1000;
const CUresult result = cuCtxSetCurrent(cu_context_);
if (result != CUDA_SUCCESS) {
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
// Some NVIDIA GPUs have a limited Encode Session count.
// We can't get the Session count, so catching NvEncThrow to avoid the crash.
// refer:
// https://developer.nvidia.com/video-encode-and-decode-gpu-support-matrix-new
try {
if (cu_memory_type_ == CU_MEMORYTYPE_DEVICE) {
encoder_ = std::make_unique<NvEncoderCuda>(cu_context_, codec_.width,
codec_.height, nv_format_, 0);
} else {
RTC_DCHECK_NOTREACHED();
}
} catch (const NVENCException& e) {
// Surface initialization failure to WebRTC through the codec error return.
RTC_LOG(LS_ERROR) << "Failed Initialize NvEncoder " << e.what();
return WEBRTC_VIDEO_CODEC_ERROR;
}
nv_initialize_params_.version = NV_ENC_INITIALIZE_PARAMS_VER;
nv_encode_config_.version = NV_ENC_CONFIG_VER;
nv_initialize_params_.encodeConfig = &nv_encode_config_;
GUID encodeGuid = NV_ENC_CODEC_H264_GUID;
GUID presetGuid = NV_ENC_PRESET_P4_GUID;
encoder_->CreateDefaultEncoderParams(&nv_initialize_params_, encodeGuid,
presetGuid,
NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY);
nv_initialize_params_.frameRateNum =
static_cast<uint32_t>(configuration_.max_frame_rate);
nv_initialize_params_.frameRateDen = 1;
nv_initialize_params_.bufferFormat = nv_format_;
nv_encode_config_.profileGUID = nv_profile_guid_;
nv_encode_config_.gopLength = NVENC_INFINITE_GOPLENGTH;
nv_encode_config_.frameIntervalP = 1;
nv_encode_config_.encodeCodecConfig.h264Config.level = nv_enc_level_;
nv_encode_config_.encodeCodecConfig.h264Config.idrPeriod =
NVENC_INFINITE_GOPLENGTH;
nv_encode_config_.rcParams.version = NV_ENC_RC_PARAMS_VER;
nv_encode_config_.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR;
nv_encode_config_.rcParams.averageBitRate = configuration_.target_bps;
nv_encode_config_.rcParams.vbvBufferSize =
(nv_encode_config_.rcParams.averageBitRate *
nv_initialize_params_.frameRateDen /
nv_initialize_params_.frameRateNum) *
5;
nv_encode_config_.rcParams.vbvInitialDelay =
nv_encode_config_.rcParams.vbvBufferSize;
try {
encoder_->CreateEncoder(&nv_initialize_params_);
} catch (const NVENCException& e) {
RTC_LOG(LS_ERROR) << "Failed Initialize NvEncoder " << e.what();
return WEBRTC_VIDEO_CODEC_ERROR;
}
RTC_LOG(LS_INFO) << "NVIDIA H264 NVENC initialized: "
<< codec_.width << "x" << codec_.height
<< " @ " << codec_.maxFramerate << "fps, target_bps="
<< configuration_.target_bps;
SimulcastRateAllocator init_allocator(env_, codec_);
VideoBitrateAllocation allocation =
init_allocator.Allocate(VideoBitrateAllocationParameters(
DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate));
SetRates(RateControlParameters(allocation, codec_.maxFramerate));
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t NvidiaH264EncoderImpl::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
encoded_image_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t NvidiaH264EncoderImpl::Release() {
if (encoder_) {
encoder_->DestroyEncoder();
encoder_ = nullptr;
}
if (cu_scaled_array_) {
cuArrayDestroy(cu_scaled_array_);
cu_scaled_array_ = nullptr;
}
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t NvidiaH264EncoderImpl::Encode(
const VideoFrame& input_frame,
const std::vector<VideoFrameType>* frame_types) {
if (!encoder_) {
ReportError();
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (!encoded_image_callback_) {
RTC_LOG(LS_WARNING)
<< "InitEncode() has been called, but a callback function "
"has not been set with RegisterEncodeCompleteCallback()";
ReportError();
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
bool is_keyframe_needed = false;
if (configuration_.key_frame_request && configuration_.sending) {
is_keyframe_needed = true;
}
bool send_key_frame =
is_keyframe_needed ||
(frame_types && (*frame_types)[0] == VideoFrameType::kVideoFrameKey);
if (send_key_frame) {
is_keyframe_needed = true;
configuration_.key_frame_request = false;
}
RTC_DCHECK_EQ(configuration_.width, input_frame.width());
RTC_DCHECK_EQ(configuration_.height, input_frame.height());
if (!configuration_.sending) {
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
if (frame_types != nullptr) {
// Skip frame?
if ((*frame_types)[0] == VideoFrameType::kEmptyFrame) {
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
}
NV_ENC_PIC_PARAMS pic_params = NV_ENC_PIC_PARAMS();
pic_params.version = NV_ENC_PIC_PARAMS_VER;
pic_params.encodePicFlags = 0;
if (is_keyframe_needed) {
pic_params.encodePicFlags = NV_ENC_PIC_FLAG_FORCEINTRA |
NV_ENC_PIC_FLAG_FORCEIDR |
NV_ENC_PIC_FLAG_OUTPUT_SPSPPS;
configuration_.key_frame_request = false;
}
try {
std::vector<std::vector<uint8_t>> bit_stream;
if (TryEncodeNativeGpuFrame(encoder_.get(), cu_context_, input_frame,
&pic_params, &bit_stream)) {
for (std::vector<uint8_t>& packet : bit_stream) {
int32_t result = ProcessEncodedFrame(packet, input_frame);
if (result != WEBRTC_VIDEO_CODEC_OK) {
return result;
}
}
return WEBRTC_VIDEO_CODEC_OK;
}
} catch (const NVENCException& e) {
RTC_LOG(LS_ERROR) << "Failed native GPU EncodeFrame NvEncoder "
<< e.what();
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
Nv12HostFrame frame_buffer;
if (!PrepareNv12HostFrame(input_frame, &frame_buffer)) {
const auto failed_buffer = input_frame.video_frame_buffer();
RTC_LOG(LS_ERROR) << "Failed to convert "
<< (failed_buffer
? VideoFrameBufferTypeToString(
failed_buffer->type())
: "null")
<< " image to NV12. Can't encode frame.";
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
try {
const NvEncInputFrame* nv_enc_input_frame = encoder_->GetNextInputFrame();
if (cu_memory_type_ == CU_MEMORYTYPE_DEVICE) {
if (nv_enc_input_frame->bufferFormat != NV_ENC_BUFFER_FORMAT_NV12) {
RTC_LOG(LS_ERROR) << "NVIDIA encoder expected NV12 input";
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
NvEncoderCuda::CopyToDeviceFrame(
cu_context_, (void*)frame_buffer.data, frame_buffer.stride,
reinterpret_cast<CUdeviceptr>(nv_enc_input_frame->inputPtr),
nv_enc_input_frame->pitch, input_frame.width(), input_frame.height(),
CU_MEMORYTYPE_HOST, nv_enc_input_frame->bufferFormat,
nv_enc_input_frame->chromaOffsets, nv_enc_input_frame->numChromaPlanes);
}
std::vector<std::vector<uint8_t>> bit_stream;
encoder_->EncodeFrame(bit_stream, &pic_params);
for (std::vector<uint8_t>& packet : bit_stream) {
int32_t result = ProcessEncodedFrame(packet, input_frame);
if (result != WEBRTC_VIDEO_CODEC_OK) {
return result;
}
}
} catch (const NVENCException& e) {
RTC_LOG(LS_ERROR) << "Failed EncodeFrame NvEncoder " << e.what();
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t NvidiaH264EncoderImpl::ProcessEncodedFrame(
std::vector<uint8_t>& packet,
const ::webrtc::VideoFrame& inputFrame) {
encoded_image_._encodedWidth = encoder_->GetEncodeWidth();
encoded_image_._encodedHeight = encoder_->GetEncodeHeight();
encoded_image_.SetRtpTimestamp(inputFrame.rtp_timestamp());
encoded_image_.SetSimulcastIndex(0);
encoded_image_.ntp_time_ms_ = inputFrame.ntp_time_ms();
encoded_image_.capture_time_ms_ = inputFrame.render_time_ms();
encoded_image_.rotation_ = inputFrame.rotation();
encoded_image_.content_type_ = VideoContentType::UNSPECIFIED;
encoded_image_.timing_.flags = VideoSendTiming::kInvalid;
encoded_image_._frameType = VideoFrameType::kVideoFrameDelta;
encoded_image_.SetColorSpace(inputFrame.color_space());
std::vector<H264::NaluIndex> naluIndices =
H264::FindNaluIndices(MakeArrayView(packet.data(), packet.size()));
for (uint32_t i = 0; i < naluIndices.size(); i++) {
const H264::NaluType naluType =
H264::ParseNaluType(packet[naluIndices[i].payload_start_offset]);
if (naluType == H264::kIdr) {
encoded_image_._frameType = VideoFrameType::kVideoFrameKey;
break;
}
}
encoded_image_.SetEncodedData(
EncodedImageBuffer::Create(packet.data(), packet.size()));
encoded_image_.set_size(packet.size());
h264_bitstream_parser_.ParseBitstream(encoded_image_);
encoded_image_.qp_ = h264_bitstream_parser_.GetLastSliceQp().value_or(-1);
CodecSpecificInfo codecInfo;
codecInfo.codecType = kVideoCodecH264;
codecInfo.codecSpecific.H264.packetization_mode =
H264PacketizationMode::NonInterleaved;
const auto result =
encoded_image_callback_->OnEncodedImage(encoded_image_, &codecInfo);
if (result.error != EncodedImageCallback::Result::OK) {
RTC_LOG(LS_ERROR) << "Encode m_encodedCompleteCallback failed "
<< result.error;
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
VideoEncoder::EncoderInfo NvidiaH264EncoderImpl::GetEncoderInfo() const {
EncoderInfo info;
info.supports_native_handle = false;
info.implementation_name = "NVIDIA H264 Encoder";
info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
info.is_hardware_accelerated = true;
info.supports_simulcast = false;
info.preferred_pixel_formats = {VideoFrameBuffer::Type::kNV12,
VideoFrameBuffer::Type::kI420};
return info;
}
void NvidiaH264EncoderImpl::SetRates(
const RateControlParameters& parameters) {
if (!encoder_) {
RTC_LOG(LS_WARNING) << "SetRates() while uninitialized.";
return;
}
if (parameters.framerate_fps < 1.0) {
RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps;
return;
}
if (parameters.bitrate.get_sum_bps() == 0) {
configuration_.SetStreamState(false);
return;
}
codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps);
codec_.maxBitrate = parameters.bitrate.GetSpatialLayerSum(0);
configuration_.target_bps = parameters.bitrate.GetSpatialLayerSum(0);
configuration_.max_frame_rate = parameters.framerate_fps;
if (configuration_.target_bps) {
configuration_.SetStreamState(true);
} else {
configuration_.SetStreamState(false);
}
}
void NvidiaH264EncoderImpl::LayerConfig::SetStreamState(bool send_stream) {
if (send_stream && !sending) {
// Need a key frame if we have not sent this stream before.
key_frame_request = true;
}
sending = send_stream;
}
} // namespace webrtc
@@ -0,0 +1,99 @@
#ifndef WEBRTC_NVIDIA_H264_ENCODER_IMPL_H_
#define WEBRTC_NVIDIA_H264_ENCODER_IMPL_H_
#include <cuda.h>
#include <memory>
#include <vector>
#include "NvEncoder/NvEncoder.h"
#include "NvEncoder/NvEncoderCuda.h"
#include "absl/container/inlined_vector.h"
#include "api/transport/rtp/dependency_descriptor.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/scalability_mode.h"
#include "api/video_codecs/video_encoder.h"
#include "common_video/h264/h264_bitstream_parser.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/svc/scalable_video_controller.h"
#include "modules/video_coding/utility/quality_scaler.h"
namespace webrtc {
class NvidiaH264EncoderImpl : public VideoEncoder {
public:
struct LayerConfig {
int simulcast_idx = 0;
int width = -1;
int height = -1;
bool sending = true;
bool key_frame_request = false;
float max_frame_rate = 0;
uint32_t target_bps = 0;
uint32_t max_bps = 0;
bool frame_dropping_on = false;
int key_frame_interval = 0;
int num_temporal_layers = 1;
void SetStreamState(bool send_stream);
};
public:
NvidiaH264EncoderImpl(const webrtc::Environment& env,
CUcontext context,
CUmemorytype memory_type,
NV_ENC_BUFFER_FORMAT nv_format,
const SdpVideoFormat& format);
~NvidiaH264EncoderImpl() override;
int32_t InitEncode(const VideoCodec* codec_settings,
const Settings& settings) override;
int32_t RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) override;
int32_t Release() override;
int32_t Encode(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) override;
void SetRates(const RateControlParameters& rc_parameters) override;
EncoderInfo GetEncoderInfo() const override;
private:
int32_t ProcessEncodedFrame(std::vector<uint8_t>& packet,
const ::webrtc::VideoFrame& inputFrame);
private:
const webrtc::Environment& env_;
EncodedImageCallback* encoded_image_callback_ = nullptr;
std::unique_ptr<NvEncoder> encoder_;
CUcontext cu_context_;
CUmemorytype cu_memory_type_;
CUarray cu_scaled_array_;
NV_ENC_BUFFER_FORMAT nv_format_;
NV_ENC_INITIALIZE_PARAMS nv_initialize_params_;
NV_ENC_CONFIG nv_encode_config_;
GUID nv_profile_guid_;
NV_ENC_LEVEL nv_enc_level_;
LayerConfig configuration_;
EncodedImage encoded_image_;
H264PacketizationMode packetization_mode_;
VideoCodec codec_;
void ReportInit();
void ReportError();
bool has_reported_init_ = false;
bool has_reported_error_ = false;
webrtc::H264BitstreamParser h264_bitstream_parser_;
const SdpVideoFormat format_;
H264Profile profile_ = H264Profile::kProfileConstrainedBaseline;
H264Level level_ = H264Level::kLevel1_b;
};
} // namespace webrtc
#endif // WEBRTC_NVIDIA_H264_ENCODER_IMPL_H_
@@ -0,0 +1,168 @@
#include "h265_decoder_impl.h"
#include <api/video/i420_buffer.h>
#include <api/video/video_codec_type.h>
#include <modules/video_coding/include/video_error_codes.h>
#include <third_party/libyuv/include/libyuv/convert.h>
#include "NvDecoder/NvDecoder.h"
#include "Utils/NvCodecUtils.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
static ColorSpace ExtractColorSpaceFromFormat(const CUVIDEOFORMAT& format) {
return ColorSpace(
static_cast<ColorSpace::PrimaryID>(
format.video_signal_description.color_primaries),
static_cast<ColorSpace::TransferID>(
format.video_signal_description.transfer_characteristics),
static_cast<ColorSpace::MatrixID>(
format.video_signal_description.matrix_coefficients),
static_cast<ColorSpace::RangeID>(
format.video_signal_description.video_full_range_flag));
}
NvidiaH265DecoderImpl::NvidiaH265DecoderImpl(CUcontext context)
: cu_context_(context),
decoder_(nullptr),
is_configured_decoder_(false),
decoded_complete_callback_(nullptr),
buffer_pool_(false) {}
NvidiaH265DecoderImpl::~NvidiaH265DecoderImpl() { Release(); }
VideoDecoder::DecoderInfo NvidiaH265DecoderImpl::GetDecoderInfo() const {
VideoDecoder::DecoderInfo info;
info.implementation_name = "NVIDIA H265 Decoder";
info.is_hardware_accelerated = true;
return info;
}
bool NvidiaH265DecoderImpl::Configure(const Settings& settings) {
if (settings.codec_type() != kVideoCodecH265) {
RTC_LOG(LS_ERROR) << "initialization failed: codec type is not H265";
return false;
}
if (!settings.max_render_resolution().Valid()) {
RTC_LOG(LS_ERROR)
<< "initialization failed on codec_settings width < 0 or height < 0";
return false;
}
settings_ = settings;
const CUresult result = cuCtxSetCurrent(cu_context_);
if (!ck(result)) {
RTC_LOG(LS_ERROR) << "initialization failed on cuCtxSetCurrent result"
<< result;
return false;
}
int maxWidth = 4096;
int maxHeight = 4096;
decoder_ = std::make_unique<NvDecoder>(
cu_context_, false, cudaVideoCodec_HEVC, true, false, nullptr, nullptr,
false, maxWidth, maxHeight);
return true;
}
int32_t NvidiaH265DecoderImpl::RegisterDecodeCompleteCallback(
DecodedImageCallback* callback) {
decoded_complete_callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t NvidiaH265DecoderImpl::Release() {
buffer_pool_.Release();
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t NvidiaH265DecoderImpl::Decode(const EncodedImage& input_image,
bool /*missing_frames*/,
int64_t /*render_time_ms*/) {
CUcontext current;
if (!ck(cuCtxGetCurrent(&current))) {
RTC_LOG(LS_ERROR) << "decode failed on cuCtxGetCurrent";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (current != cu_context_) {
RTC_LOG(LS_ERROR)
<< "decode failed: current context does not match held context";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (decoded_complete_callback_ == nullptr) {
RTC_LOG(LS_ERROR) << "decode failed: decoded_complete_callback_ not set";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (!input_image.data() || !input_image.size()) {
RTC_LOG(LS_ERROR) << "decode failed: input image is null";
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
int nFrameReturned = 0;
try {
do {
nFrameReturned = decoder_->Decode(
input_image.data(), static_cast<int>(input_image.size()),
CUVID_PKT_TIMESTAMP, input_image.RtpTimestamp());
} while (nFrameReturned == 0);
} catch (const NVDECException& e) {
RTC_LOG(LS_ERROR) << "NVDEC H265 decode failed: " << e.what();
decoder_.reset();
is_configured_decoder_ = false;
return WEBRTC_VIDEO_CODEC_ERROR;
}
is_configured_decoder_ = true;
if (decoder_->GetOutputFormat() != cudaVideoSurfaceFormat_NV12) {
RTC_LOG(LS_ERROR) << "not supported output format: "
<< decoder_->GetOutputFormat();
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
const ColorSpace& color_space =
input_image.ColorSpace() ? *input_image.ColorSpace()
: ExtractColorSpaceFromFormat(
decoder_->GetVideoFormatInfo());
for (int i = 0; i < nFrameReturned; i++) {
int64_t timeStamp;
uint8_t* pFrame = decoder_->GetFrame(&timeStamp);
webrtc::scoped_refptr<webrtc::I420Buffer> i420_buffer =
buffer_pool_.CreateI420Buffer(decoder_->GetWidth(),
decoder_->GetHeight());
int result = libyuv::NV12ToI420(
pFrame, decoder_->GetDeviceFramePitch(),
pFrame + decoder_->GetHeight() * decoder_->GetDeviceFramePitch(),
decoder_->GetDeviceFramePitch(), i420_buffer->MutableDataY(),
i420_buffer->StrideY(), i420_buffer->MutableDataU(),
i420_buffer->StrideU(), i420_buffer->MutableDataV(),
i420_buffer->StrideV(), decoder_->GetWidth(), decoder_->GetHeight());
if (result) {
RTC_LOG(LS_INFO) << "libyuv::NV12ToI420 failed. error:" << result;
}
VideoFrame decoded_frame = VideoFrame::Builder()
.set_video_frame_buffer(i420_buffer)
.set_timestamp_rtp(static_cast<uint32_t>(
timeStamp))
.set_color_space(color_space)
.build();
std::optional<int32_t> decodetime;
std::optional<int> qp; // Not parsed for H265 currently
decoded_complete_callback_->Decoded(decoded_frame, decodetime, qp);
}
return WEBRTC_VIDEO_CODEC_OK;
}
} // end namespace webrtc

Some files were not shown because too many files have changed in this diff Show More