Experiment support for decoding into YUV444P by SW FFMPEG decoders.

Application must call GlobalConfiguration::SetRangeExtensionEnabled(true)
to allow OWT switching to SW HEVC decoder.
By default range extension decoding of HEVC is disabled.

talk/owt/BUILD.gn

@@ -374,6 +374,8 @@ static_library("owt_sdk_base") {
       sources += [
         "sdk/base/win/d3d11_video_decoder.cc",
         "sdk/base/win/d3d11_video_decoder.h",
+        "sdk/base/win/ffmpeg_decoder_impl.cc",
+        "sdk/base/win/ffmpeg_decoder_impl.h",
       ]
     }
     defines += [ "OWT_USE_FFMPEG" ]

talk/owt/sdk/base/globalconfiguration.cc

@@ -12,6 +12,7 @@ bool GlobalConfiguration::hardware_acceleration_enabled_ = true;
 ID3D11Device* GlobalConfiguration::d3d11_decoding_device_ = nullptr;
 #endif
 bool GlobalConfiguration::flex_fec_enabled_ = false;
+bool GlobalConfiguration::range_extension_enabled_ = false;
 int GlobalConfiguration::link_mtu_ = 0; // not set;
 int GlobalConfiguration::min_port_ = 0; // not set;
 int GlobalConfiguration::max_port_ = 0; // not set;

talk/owt/sdk/base/peerconnectiondependencyfactory.cc

@@ -149,6 +149,9 @@ void PeerConnectionDependencyFactory::
   if (GlobalConfiguration::GetFlexFecEnabled()) {
     field_trial_ += "OWT-FlexFEC/Enabled/";
   }
+  if (GlobalConfiguration::GetRangeExtensionEnabled()) {
+    field_trial_ += "OWT-RangeExtension/Enabled/";
+  }
   int delay_bwe_weight = GlobalConfiguration::GetDelayBasedBweWeight();
   field_trial_ +=
       "OWT-DelayBweWeight/" + std::to_string(delay_bwe_weight) + "/";

talk/owt/sdk/base/webrtcvideorendererimpl.cc

@@ -9,6 +9,7 @@
 #include <d3d9.h>
 #include <dxva2api.h>
 #endif
+#include "rtc_base/logging.h"
 #include "talk/owt/sdk/base/nativehandlebuffer.h"
 #include "talk/owt/sdk/base/webrtcvideorendererimpl.h"
 #if defined(WEBRTC_WIN)
@@ -71,13 +72,18 @@ void WebrtcVideoRendererImpl::OnFrame(const webrtc::VideoFrame& frame) {
         (uint8_t*)render_ptr, resolution, VideoBufferType::kD3D11});
 
     renderer_.RenderFrame(std::move(video_buffer));
+    return;
 #else
     return;
 #endif
   }
+
+  // Non-native buffer. Only for I444 buffer and I444 renderer, we do a
+  // direct copy. Otherwise we convert to renderer type.
   VideoRendererType renderer_type = renderer_.Type();
   if (renderer_type != VideoRendererType::kI420 &&
-      renderer_type != VideoRendererType::kARGB)
+      renderer_type != VideoRendererType::kARGB &&
+      renderer_type != VideoRendererType::kI444)
     return;
   Resolution resolution(frame.width(), frame.height());
   if (renderer_type == VideoRendererType::kARGB) {
@@ -87,6 +93,28 @@ void WebrtcVideoRendererImpl::OnFrame(const webrtc::VideoFrame& frame) {
     std::unique_ptr<VideoBuffer> video_buffer(
         new VideoBuffer{buffer, resolution, VideoBufferType::kARGB});
     renderer_.RenderFrame(std::move(video_buffer));
+  } else if (renderer_type == VideoRendererType::kI444 &&
+             frame.video_frame_buffer()->type() ==
+                 webrtc::VideoFrameBuffer::Type::kI444) {
+    // Assume stride equals to width(might not stand here?)
+    uint8_t* buffer = new uint8_t[resolution.width * resolution.height * 3];
+    rtc::scoped_refptr<VideoFrameBuffer> frame_buffer =
+        frame.video_frame_buffer();
+    const webrtc::PlanarYuv8Buffer* planar_yuv_buffer =
+        reinterpret_cast<const webrtc::PlanarYuv8Buffer*>(
+            frame_buffer->GetI444());
+    size_t data_ptr = 0;
+    memcpy(buffer, planar_yuv_buffer->DataY(),
+           planar_yuv_buffer->StrideY() * planar_yuv_buffer->height());
+    data_ptr += planar_yuv_buffer->StrideY() * planar_yuv_buffer->height();
+    memcpy(buffer + data_ptr, planar_yuv_buffer->DataU(),
+           planar_yuv_buffer->StrideU() * planar_yuv_buffer->height());
+    data_ptr += planar_yuv_buffer->StrideU() * planar_yuv_buffer->height();
+    memcpy(buffer + data_ptr, planar_yuv_buffer->DataV(),
+           planar_yuv_buffer->StrideV() * planar_yuv_buffer->height());
+    std::unique_ptr<VideoBuffer> video_buffer(
+        new VideoBuffer{buffer, resolution, VideoBufferType::kI444});
+    renderer_.RenderFrame(std::move(video_buffer));
   } else {
     uint8_t* buffer = new uint8_t[resolution.width * resolution.height * 3 / 2];
     webrtc::ConvertFromI420(frame, webrtc::VideoType::kI420, 0,

talk/owt/sdk/base/win/d3d11_video_decoder.cc

@@ -310,11 +310,24 @@ int32_t D3D11VideoDecoder::Decode(const webrtc::EncodedImage& input_image,
       goto fail;
     }
 
+    if (!frame) {
+      RTC_LOG(LS_ERROR) << "Failed to decode current frame.";
+      goto fail;
+    }
+
+    int width = frame->width;
+    int height = frame->height;
+
+    VideoFrame* input_frame =
+        static_cast<VideoFrame*>(av_buffer_get_opaque(frame->buf[0]));
+    RTC_DCHECK(input_frame);
+
+    rtc::scoped_refptr<VideoFrameBuffer> frame_buffer =
+        input_frame->video_frame_buffer();
+
     // We get one frame from the decoder.
-    if (frame != nullptr && frame->format == hw_pix_fmt) {
+    if (frame->format != AV_PIX_FMT_YUV444P) {
       ID3D11Texture2D* texture = (ID3D11Texture2D*)frame->data[0];
-      int width = frame->width;
-      int height = frame->height;
       int index = (intptr_t)frame->data[1];
       D3D11_TEXTURE2D_DESC texture_desc;
 
@@ -380,6 +393,20 @@ int32_t D3D11VideoDecoder::Decode(const webrtc::EncodedImage& input_image,
 
       }
       av_frame_free(&frame);
+    } else {  // YUV444P. Which will be a software frame.
+      rtc::scoped_refptr<webrtc::VideoFrameBuffer> dst_buffer;
+      dst_buffer =
+          WrapI444Buffer(width, height, frame->data[0], frame->linesize[0],
+                         frame->data[1], frame->linesize[1], frame->data[2],
+                         frame->linesize[2], [frame_buffer] {});
+      webrtc::VideoFrame decoded_frame(dst_buffer, input_image.Timestamp(), 0,
+                                       webrtc::kVideoRotation_0);
+      decoded_frame.set_ntp_time_ms(input_image.ntp_time_ms_);
+      decoded_frame.set_timestamp(input_image.Timestamp());
+      if (decoded_image_callback_) {
+        decoded_image_callback_->Decoded(decoded_frame);
+      }
+      av_frame_free(&frame);
     }
   }
   return WEBRTC_VIDEO_CODEC_OK;

talk/owt/sdk/base/win/externalvideodecoderfactory.cc

@@ -13,17 +13,21 @@
 #include "talk/owt/sdk/base/codecutils.h"
 #ifdef OWT_USE_FFMPEG
 #include "talk/owt/sdk/base/win/d3d11_video_decoder.h"
+#include "talk/owt/sdk/base/win/ffmpeg_decoder_impl.h"
 #endif
 #ifdef OWT_USE_MSDK
 #include "talk/owt/sdk/base/win/msdkvideodecoder.h"
 #endif
+#include "system_wrappers/include/field_trial.h"
 #include "webrtc/rtc_base/checks.h"
 #include "webrtc/rtc_base/logging.h"
 
 namespace owt {
 namespace base {
 
 ExternalVideoDecoderFactory::ExternalVideoDecoderFactory(ID3D11Device* d3d11_device_external) {
+  range_extension_enabled_ =
+      webrtc::field_trial::IsEnabled("OWT-RangeExtension");
   supported_codec_types_.clear();
 
   bool is_vp8_hw_supported = false, is_vp9_hw_supported = false;
@@ -109,7 +113,11 @@ ExternalVideoDecoderFactory::CreateVideoDecoder(
   }
   if (vp8_hw || vp9_hw || h264_hw || h265_hw || av1_hw) {
 #if defined(OWT_USE_FFMPEG)
-    return owt::base::D3D11VideoDecoder::Create(cricket::VideoCodec(format));
+    if (range_extension_enabled_) {
+      return std::make_unique<FFMpegDecoderImpl>();
+    } else {
+      return owt::base::D3D11VideoDecoder::Create(cricket::VideoCodec(format));
+    }
 #endif
 #if defined(OWT_USE_MSDK)
     return owt::base::MSDKVideoDecoder::Create(cricket::VideoCodec(format));

talk/owt/sdk/base/win/externalvideodecoderfactory.h

@@ -35,6 +35,7 @@ class ExternalVideoDecoderFactory : public webrtc::VideoDecoderFactory {
  private:
   std::vector<webrtc::VideoCodecType> supported_codec_types_;
   ID3D11Device* external_device_ = nullptr;
+  bool range_extension_enabled_ = false;
 };
 }  // namespace base
 }  // namespace owt