diff --git a/Cargo.toml b/Cargo.toml index 61ea04719c..bbeb1b3125 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["alvr/*"] [workspace.package] -version = "20.0.0-dev00" +version = "20.0.0-dev01" edition = "2021" rust-version = "1.64" authors = ["alvr-org"] diff --git a/alvr/client_core/cpp/bindings.h b/alvr/client_core/cpp/bindings.h index 48218889b1..fab0ce9fbc 100644 --- a/alvr/client_core/cpp/bindings.h +++ b/alvr/client_core/cpp/bindings.h @@ -61,9 +61,9 @@ extern "C" void renderStreamNative(void *streamHardwareBuffer, const int swapcha // nal.h extern "C" void initializeNalParser(int codec, bool enableFec); +extern "C" void notifyNewDecoder(); extern "C" bool processNalPacket(VideoFrame header, const unsigned char *payload, int payloadSize, bool &outHadFecFailure); -extern "C" void (*createDecoder)(const char *csd_0, int length); extern "C" void (*pushNal)(const char *buffer, int length, unsigned long long frameIndex); \ No newline at end of file diff --git a/alvr/client_core/cpp/nal.cpp b/alvr/client_core/cpp/nal.cpp index e0085121e8..1a476c3530 100644 --- a/alvr/client_core/cpp/nal.cpp +++ b/alvr/client_core/cpp/nal.cpp @@ -5,15 +5,11 @@ #include "bindings.h" #include "fec.h" -static const std::byte NAL_TYPE_SPS = static_cast(7); -static const std::byte H265_NAL_TYPE_VPS = static_cast(32); - enum ALVR_CODEC { ALVR_CODEC_H264 = 0, ALVR_CODEC_H265 = 1, }; -void (*createDecoder)(const char *csd_0, int length); void (*pushNal)(const char *buffer, int length, unsigned long long frameIndex); namespace { @@ -28,29 +24,8 @@ void initializeNalParser(int codec, bool enableFec) { m_queue = FECQueue(); } -int findVPSSPS(const std::byte *frameBuffer, int frameByteSize) { - int zeroes = 0; - int foundNals = 0; - for (int i = 0; i < frameByteSize; i++) { - if (frameBuffer[i] == std::byte(0)) { - zeroes++; - } else if (frameBuffer[i] == std::byte(1)) { - if (zeroes >= 2) { - foundNals++; - if (m_codec == ALVR_CODEC_H264 && foundNals >= 3) { - // Find end of SPS+PPS on H.264. - return i - 3; - } else if (m_codec == ALVR_CODEC_H265 && foundNals >= 4) { - // Find end of VPS+SPS+PPS on H.264. - return i - 3; - } - } - zeroes = 0; - } else { - zeroes = 0; - } - } - return -1; +void notifyNewDecoder() { + m_queue.clearFecFailure(); } bool processNalPacket(VideoFrame header, @@ -73,31 +48,8 @@ bool processNalPacket(VideoFrame header, frameByteSize = payloadSize; } - std::byte NALType; - if (m_codec == ALVR_CODEC_H264) - NALType = frameBuffer[4] & std::byte(0x1F); - else - NALType = (frameBuffer[4] >> 1) & std::byte(0x3F); + pushNal((const char *)&frameBuffer[0], frameByteSize, header.trackingFrameIndex); - if ((m_codec == ALVR_CODEC_H264 && NALType == NAL_TYPE_SPS) || - (m_codec == ALVR_CODEC_H265 && NALType == H265_NAL_TYPE_VPS)) { - // This frame contains (VPS + )SPS + PPS + IDR on NVENC H.264 (H.265) stream. - // (VPS + )SPS + PPS has short size (8bytes + 28bytes in some environment), so we can - // assume SPS + PPS is contained in first fragment. - - int end = findVPSSPS(frameBuffer, frameByteSize); - if (end == -1) { - // Invalid frame. - return false; - } - createDecoder((const char *)&frameBuffer[0], end); - pushNal( - (const char *)&frameBuffer[end], frameByteSize - end, header.trackingFrameIndex); - - m_queue.clearFecFailure(); - } else { - pushNal((const char *)&frameBuffer[0], frameByteSize, header.trackingFrameIndex); - } return true; } return false; diff --git a/alvr/client_core/src/connection.rs b/alvr/client_core/src/connection.rs index a0d7eb3d1c..bef8cfe254 100644 --- a/alvr/client_core/src/connection.rs +++ b/alvr/client_core/src/connection.rs @@ -1,10 +1,13 @@ #![allow(clippy::if_same_then_else)] use crate::{ - decoder::DECODER_INIT_CONFIG, platform, sockets::AnnouncerSocket, - statistics::StatisticsManager, storage::Config, AlvrEvent, VideoFrame, CONTROL_CHANNEL_SENDER, - DISCONNECT_NOTIFIER, EVENT_QUEUE, IS_ALIVE, IS_RESUMED, IS_STREAMING, STATISTICS_MANAGER, - STATISTICS_SENDER, TRACKING_SENDER, + decoder::{self, DECODER_INIT_CONFIG}, + platform, + sockets::AnnouncerSocket, + statistics::StatisticsManager, + storage::Config, + AlvrEvent, VideoFrame, CONTROL_CHANNEL_SENDER, DISCONNECT_NOTIFIER, EVENT_QUEUE, IS_ALIVE, + IS_RESUMED, IS_STREAMING, STATISTICS_MANAGER, STATISTICS_SENDER, TRACKING_SENDER, }; use alvr_audio::{AudioDevice, AudioDeviceType}; use alvr_common::{glam::UVec2, prelude::*, ALVR_VERSION}; @@ -587,6 +590,9 @@ async fn stream_pipeline( let control_receive_loop = async move { loop { match control_receiver.recv().await { + Ok(ServerControlPacket::InitializeDecoder { config_buffer }) => { + decoder::create_decoder(config_buffer); + } Ok(ServerControlPacket::Restarting) => { info!("{SERVER_RESTART_MESSAGE}"); set_hud_message(SERVER_RESTART_MESSAGE); diff --git a/alvr/client_core/src/decoder.rs b/alvr/client_core/src/decoder.rs index 0a163ad0c6..45f5596997 100644 --- a/alvr/client_core/src/decoder.rs +++ b/alvr/client_core/src/decoder.rs @@ -1,4 +1,4 @@ -use crate::{AlvrCodec, AlvrEvent, EVENT_QUEUE, STATISTICS_MANAGER}; +use crate::{AlvrCodec, AlvrEvent, EVENT_QUEUE}; use alvr_common::{once_cell::sync::Lazy, parking_lot::Mutex, RelaxedAtomic}; use alvr_session::{CodecType, MediacodecDataType}; use std::{collections::VecDeque, ffi::c_char, ptr, time::Duration}; @@ -41,17 +41,14 @@ static NAL_QUEUE: Lazy>> = static LAST_ENQUEUED_TIMESTAMPS: Lazy>> = Lazy::new(|| Mutex::new(VecDeque::new())); -pub extern "C" fn create_decoder(buffer: *const c_char, length: i32) { - let mut csd_0 = vec![0; length as _]; - unsafe { ptr::copy_nonoverlapping(buffer, csd_0.as_mut_ptr() as _, length as _) }; - +pub fn create_decoder(config_buffer: Vec) { let config = DECODER_INIT_CONFIG.lock(); if EXTERNAL_DECODER.value() { // duration == 0 is the flag to identify the config NALS NAL_QUEUE.lock().push_back(ReconstructedNal { timestamp: Duration::ZERO, - data: csd_0, + data: config_buffer, }); EVENT_QUEUE.lock().push_back(AlvrEvent::CreateDecoder { codec: if matches!(config.codec, CodecType::H264) { @@ -63,13 +60,16 @@ pub extern "C" fn create_decoder(buffer: *const c_char, length: i32) { } else { #[cfg(target_os = "android")] if DECODER_ENQUEUER.lock().is_none() { - let (enqueuer, dequeuer) = - crate::platform::video_decoder_split(config.clone(), &csd_0, |target_timestamp| { - if let Some(stats) = &mut *STATISTICS_MANAGER.lock() { + let (enqueuer, dequeuer) = crate::platform::video_decoder_split( + config.clone(), + &config_buffer, + |target_timestamp| { + if let Some(stats) = &mut *crate::STATISTICS_MANAGER.lock() { stats.report_frame_decoded(target_timestamp); } - }) - .unwrap(); + }, + ) + .unwrap(); *DECODER_ENQUEUER.lock() = Some(enqueuer); *DECODER_DEQUEUER.lock() = Some(dequeuer); @@ -79,6 +79,8 @@ pub extern "C" fn create_decoder(buffer: *const c_char, length: i32) { .send(alvr_sockets::ClientControlPacket::RequestIdr) .ok(); } + + unsafe { crate::notifyNewDecoder() }; } } } diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs index 057dbe05ea..004c093891 100644 --- a/alvr/client_core/src/lib.rs +++ b/alvr/client_core/src/lib.rs @@ -184,7 +184,6 @@ pub unsafe extern "C" fn alvr_initialize( logging_backend::init_logging(); - createDecoder = Some(decoder::create_decoder); pushNal = Some(decoder::push_nal); // Make sure to reset config in case of version compat mismatch. diff --git a/alvr/server/cpp/alvr_server/ClientConnection.cpp b/alvr/server/cpp/alvr_server/ClientConnection.cpp index 7f69bc9788..d21dc6f3f1 100644 --- a/alvr/server/cpp/alvr_server/ClientConnection.cpp +++ b/alvr/server/cpp/alvr_server/ClientConnection.cpp @@ -8,6 +8,9 @@ #include "Utils.h" #include "Settings.h" +static const uint8_t NAL_TYPE_SPS = 7; +static const uint8_t H265_NAL_TYPE_VPS = 32; + ClientConnection::ClientConnection() { m_Statistics = std::make_shared(); @@ -18,6 +21,31 @@ ClientConnection::ClientConnection() { m_fecPercentage = INITIAL_FEC_PERCENTAGE; } +int findVPSSPS(const uint8_t *frameBuffer, int frameByteSize) { + int zeroes = 0; + int foundNals = 0; + for (int i = 0; i < frameByteSize; i++) { + if (frameBuffer[i] == 0) { + zeroes++; + } else if (frameBuffer[i] == 1) { + if (zeroes >= 2) { + foundNals++; + if (Settings::Instance().m_codec == ALVR_CODEC_H264 && foundNals >= 3) { + // Find end of SPS+PPS on H.264. + return i - 3; + } else if (Settings::Instance().m_codec == ALVR_CODEC_H265 && foundNals >= 4) { + // Find end of VPS+SPS+PPS on H.264. + return i - 3; + } + } + zeroes = 0; + } else { + zeroes = 0; + } + } + return -1; +} + void ClientConnection::FECSend(uint8_t *buf, int len, uint64_t targetTimestampNs, uint64_t videoFrameIndex) { int shardPackets = CalculateFECShardPackets(len, m_fecPercentage); @@ -105,6 +133,31 @@ void ClientConnection::SendVideo(uint8_t *buf, int len, uint64_t targetTimestamp // Report before the frame is packetized ReportEncoded(targetTimestampNs); + uint8_t NALType; + if (Settings::Instance().m_codec == ALVR_CODEC_H264) + NALType = buf[4] & 0x1F; + else + NALType = (buf[4] >> 1) & 0x3F; + + if ((Settings::Instance().m_codec == ALVR_CODEC_H264 && NALType == NAL_TYPE_SPS) || + (Settings::Instance().m_codec == ALVR_CODEC_H265 && NALType == H265_NAL_TYPE_VPS)) { + // This frame contains (VPS + )SPS + PPS + IDR on NVENC H.264 (H.265) stream. + // (VPS + )SPS + PPS has short size (8bytes + 28bytes in some environment), so we can + // assume SPS + PPS is contained in first fragment. + + int end = findVPSSPS(buf, len); + if (end == -1) { + // Invalid frame. + return; + } + + InitializeDecoder((const unsigned char *)buf, end); + + // move the cursor forward excluding config NALs + buf = &buf[end]; + len = len - end; + } + if (Settings::Instance().m_enableFec) { FECSend(buf, len, targetTimestampNs, mVideoFrameIndex); } else { diff --git a/alvr/server/cpp/alvr_server/alvr_server.cpp b/alvr/server/cpp/alvr_server/alvr_server.cpp index 942e2ccf7f..4afb4bf0e1 100644 --- a/alvr/server/cpp/alvr_server/alvr_server.cpp +++ b/alvr/server/cpp/alvr_server/alvr_server.cpp @@ -166,6 +166,7 @@ void (*LogInfo)(const char *stringPtr); void (*LogDebug)(const char *stringPtr); void (*LogPeriodically)(const char *tag, const char *stringPtr); void (*DriverReadyIdle)(bool setDefaultChaprone); +void (*InitializeDecoder)(const unsigned char *configBuffer, int len); void (*VideoSend)(VideoFrame header, unsigned char *buf, int len); void (*HapticsSend)(unsigned long long path, float duration_s, float frequency, float amplitude); void (*ShutdownRuntime)(); diff --git a/alvr/server/cpp/alvr_server/bindings.h b/alvr/server/cpp/alvr_server/bindings.h index ecdd1b1e2d..639d96b28a 100644 --- a/alvr/server/cpp/alvr_server/bindings.h +++ b/alvr/server/cpp/alvr_server/bindings.h @@ -111,6 +111,7 @@ extern "C" void (*LogInfo)(const char *stringPtr); extern "C" void (*LogDebug)(const char *stringPtr); extern "C" void (*LogPeriodically)(const char *tag, const char *stringPtr); extern "C" void (*DriverReadyIdle)(bool setDefaultChaprone); +extern "C" void (*InitializeDecoder)(const unsigned char *configBuffer, int len); extern "C" void (*VideoSend)(VideoFrame header, unsigned char *buf, int len); extern "C" void (*HapticsSend)(unsigned long long path, float duration_s, diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 59f264d944..d20209c035 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -2,9 +2,9 @@ use crate::{ buttons::BUTTON_PATH_FROM_ID, sockets::WelcomeSocket, statistics::StatisticsManager, tracking::TrackingManager, AlvrButtonType_BUTTON_TYPE_BINARY, AlvrButtonType_BUTTON_TYPE_SCALAR, AlvrButtonValue, AlvrButtonValue__bindgen_ty_1, - AlvrDeviceMotion, AlvrQuat, EyeFov, OculusHand, DISCONNECT_CLIENT_NOTIFIER, HAPTICS_SENDER, - IS_ALIVE, LAST_AVERAGE_TOTAL_LATENCY, RESTART_NOTIFIER, SERVER_DATA_MANAGER, - STATISTICS_MANAGER, VIDEO_SENDER, + AlvrDeviceMotion, AlvrQuat, EyeFov, OculusHand, CONTROL_CHANNEL_SENDER, + DISCONNECT_CLIENT_NOTIFIER, HAPTICS_SENDER, IS_ALIVE, LAST_AVERAGE_TOTAL_LATENCY, + RESTART_NOTIFIER, SERVER_DATA_MANAGER, STATISTICS_MANAGER, VIDEO_SENDER, }; use alvr_audio::{AudioDevice, AudioDeviceType}; use alvr_common::{ @@ -925,6 +925,20 @@ async fn connection_pipeline( } }; + let (control_channel_sender, mut control_channel_receiver) = tmpsc::unbounded_channel(); + *CONTROL_CHANNEL_SENDER.lock() = Some(control_channel_sender); + + let control_send_loop = { + let control_sender = Arc::clone(&control_sender); + async move { + while let Some(packet) = control_channel_receiver.recv().await { + control_sender.lock().await.send(&packet).await?; + } + + Ok(()) + } + }; + let control_loop = async move { loop { match control_receiver.recv().await { @@ -1022,6 +1036,7 @@ async fn connection_pipeline( // Leave these loops on the current task res = keepalive_loop => res, res = control_loop => res, + res = control_send_loop => res, _ = RESTART_NOTIFIER.notified() => { control_sender diff --git a/alvr/server/src/lib.rs b/alvr/server/src/lib.rs index 7d23fff8fc..6c82d7e213 100644 --- a/alvr/server/src/lib.rs +++ b/alvr/server/src/lib.rs @@ -30,7 +30,9 @@ use alvr_events::EventType; use alvr_filesystem::{self as afs, Layout}; use alvr_server_data::ServerDataManager; use alvr_session::{OpenvrPropValue, OpenvrPropertyKey}; -use alvr_sockets::{ClientListAction, GpuVendor, Haptics, VideoFrameHeaderPacket}; +use alvr_sockets::{ + ClientListAction, GpuVendor, Haptics, ServerControlPacket, VideoFrameHeaderPacket, +}; use statistics::StatisticsManager; use std::{ collections::HashMap, @@ -60,6 +62,8 @@ static WINDOW: Lazy>>> = Lazy::new(|| Mutex::new(No static LAST_AVERAGE_TOTAL_LATENCY: Lazy> = Lazy::new(|| Mutex::new(Duration::ZERO)); static STATISTICS_MANAGER: Lazy>> = Lazy::new(|| Mutex::new(None)); +static CONTROL_CHANNEL_SENDER: Lazy>>> = + Lazy::new(|| Mutex::new(None)); static VIDEO_SENDER: Lazy)>>>> = Lazy::new(|| Mutex::new(None)); static HAPTICS_SENDER: Lazy>>> = @@ -289,6 +293,20 @@ pub unsafe extern "C" fn HmdDriverFactory( } } + extern "C" fn initialize_decoder(buffer_ptr: *const u8, len: i32) { + if let Some(sender) = &*CONTROL_CHANNEL_SENDER.lock() { + let mut config_buffer = vec![0; len as usize]; + + unsafe { + ptr::copy_nonoverlapping(buffer_ptr, config_buffer.as_mut_ptr(), len as usize) + }; + + sender + .send(ServerControlPacket::InitializeDecoder { config_buffer }) + .ok(); + } + } + extern "C" fn video_send(header: VideoFrame, buffer_ptr: *mut u8, len: i32) { if let Some(sender) = &*VIDEO_SENDER.lock() { let header = VideoFrameHeaderPacket { @@ -388,6 +406,7 @@ pub unsafe extern "C" fn HmdDriverFactory( LogDebug = Some(log_debug); LogPeriodically = Some(log_periodically); DriverReadyIdle = Some(driver_ready_idle); + InitializeDecoder = Some(initialize_decoder); VideoSend = Some(video_send); HapticsSend = Some(haptics_send); ShutdownRuntime = Some(_shutdown_runtime); diff --git a/alvr/sockets/src/packets.rs b/alvr/sockets/src/packets.rs index abcfbcdef3..df4f016d4b 100644 --- a/alvr/sockets/src/packets.rs +++ b/alvr/sockets/src/packets.rs @@ -47,6 +47,7 @@ pub struct StreamConfigPacket { #[derive(Serialize, Deserialize)] pub enum ServerControlPacket { StartStream, + InitializeDecoder { config_buffer: Vec }, Restarting, KeepAlive, Reserved(String), diff --git a/android/app/build.gradle b/android/app/build.gradle index 16977d1210..43a03a49e6 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -9,8 +9,8 @@ android { applicationId "alvr.client.quest" minSdkVersion 26 targetSdkVersion 33 - versionCode 85 - versionName "20.0.0-dev00" + versionCode 86 + versionName "20.0.0-dev01" externalNativeBuild { cmake { cppFlags "-std=c++17 -fexceptions -frtti" diff --git a/packaging/deb/control b/packaging/deb/control index 021b2fd1fa..1ac36692b5 100644 --- a/packaging/deb/control +++ b/packaging/deb/control @@ -34,7 +34,7 @@ Vcs-Browser: https://github.com/alvr-org/ALVR Vcs-Git: https://github.com/alvr-org/ALVR.git Rules-Requires-Root: no Package: alvr -Version: 20.0.0-dev00 +Version: 20.0.0-dev01 Architecture: amd64 Recommends: steam Description: Stream VR games from your PC to your headset via Wi-Fi diff --git a/packaging/rpm/alvr.spec b/packaging/rpm/alvr.spec index 919c29723c..38d09ffdda 100644 --- a/packaging/rpm/alvr.spec +++ b/packaging/rpm/alvr.spec @@ -1,9 +1,9 @@ Name: alvr Version: 20.0.0 -Release: 0.0.1dev00 +Release: 0.0.1dev01 Summary: Stream VR games from your PC to your headset via Wi-Fi License: MIT -Source: https://github.com/alvr-org/ALVR/archive/refs/tags/v20.0.0-dev00.tar.gz +Source: https://github.com/alvr-org/ALVR/archive/refs/tags/v20.0.0-dev01.tar.gz URL: https://github.com/alvr-org/ALVR/ ExclusiveArch: x86_64 BuildRequires: alsa-lib-devel cairo-gobject-devel cargo clang-devel ffmpeg-devel gcc gcc-c++ cmake ImageMagick (jack-audio-connection-kit-devel or pipewire-jack-audio-connection-kit-devel) libunwind-devel openssl-devel rpmdevtools rust rust-atk-sys-devel rust-cairo-sys-rs-devel rust-gdk-sys-devel rust-glib-sys-devel rust-pango-sys-devel selinux-policy-devel vulkan-headers vulkan-loader-devel