content/renderer/pepper/pepper_media_stream_video_track_host.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/pepper/pepper_media_stream_video_track_host.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/rand_util.h"
  10 #include "base/strings/utf_string_conversions.h"
  11 #include "content/renderer/media/media_stream_video_track.h"
  12 #include "media/base/bind_to_current_loop.h"
  13 #include "media/base/yuv_convert.h"
  14 #include "ppapi/c/pp_errors.h"
  15 #include "ppapi/c/ppb_media_stream_video_track.h"
  16 #include "ppapi/c/ppb_video_frame.h"
  17 #include "ppapi/host/dispatch_host_message.h"
  18 #include "ppapi/host/host_message_context.h"
  19 #include "ppapi/proxy/ppapi_messages.h"
  20 #include "ppapi/shared_impl/media_stream_buffer.h"
  21
  22 // IS_ALIGNED is also defined in
  23 // third_party/webrtc/overrides/webrtc/base/basictypes.h
  24 // TODO(ronghuawu): Avoid undef.
  25 #undef IS_ALIGNED
  26 #include "third_party/libyuv/include/libyuv.h"
  27
  28 using media::VideoFrame;
  29 using ppapi::host::HostMessageContext;
  30 using ppapi::MediaStreamVideoTrackShared;
  31
  32 namespace {
  33
  34 const int32_t kDefaultNumberOfBuffers = 4;
  35 const int32_t kMaxNumberOfBuffers = 8;
  36 // Filter mode for scaling frames.
  37 const libyuv::FilterMode kFilterMode = libyuv::kFilterBox;
  38
  39 const char kPepperVideoSourceName[] = "PepperVideoSourceName";
  40
  41 // Default config for output mode.
  42 const int kDefaultOutputFrameRate = 30;
  43
  44 media::VideoPixelFormat ToPixelFormat(PP_VideoFrame_Format format) {
  45   switch (format) {
  46     case PP_VIDEOFRAME_FORMAT_YV12:
  47       return media::PIXEL_FORMAT_YV12;
  48     case PP_VIDEOFRAME_FORMAT_I420:
  49       return media::PIXEL_FORMAT_I420;
  50     default:
  51       DVLOG(1) << "Unsupported pixel format " << format;
  52       return media::PIXEL_FORMAT_UNKNOWN;
  53   }
  54 }
  55
  56 PP_VideoFrame_Format ToPpapiFormat(media::VideoPixelFormat format) {
  57   switch (format) {
  58     case media::PIXEL_FORMAT_YV12:
  59       return PP_VIDEOFRAME_FORMAT_YV12;
  60     case media::PIXEL_FORMAT_I420:
  61       return PP_VIDEOFRAME_FORMAT_I420;
  62     default:
  63       DVLOG(1) << "Unsupported pixel format " << format;
  64       return PP_VIDEOFRAME_FORMAT_UNKNOWN;
  65   }
  66 }
  67
  68 media::VideoPixelFormat FromPpapiFormat(PP_VideoFrame_Format format) {
  69   switch (format) {
  70     case PP_VIDEOFRAME_FORMAT_YV12:
  71       return media::PIXEL_FORMAT_YV12;
  72     case PP_VIDEOFRAME_FORMAT_I420:
  73       return media::PIXEL_FORMAT_I420;
  74     default:
  75       DVLOG(1) << "Unsupported pixel format " << format;
  76       return media::PIXEL_FORMAT_UNKNOWN;
  77   }
  78 }
  79
  80 // Compute size base on the size of frame received from MediaStreamVideoSink
  81 // and size specified by plugin.
  82 gfx::Size GetTargetSize(const gfx::Size& source, const gfx::Size& plugin) {
  83   return gfx::Size(plugin.width() ? plugin.width() : source.width(),
  84                    plugin.height() ? plugin.height() : source.height());
  85 }
  86
  87 // Compute format base on the format of frame received from MediaStreamVideoSink
  88 // and format specified by plugin.
  89 PP_VideoFrame_Format GetTargetFormat(PP_VideoFrame_Format source,
  90                                      PP_VideoFrame_Format plugin) {
  91   return plugin != PP_VIDEOFRAME_FORMAT_UNKNOWN ? plugin : source;
  92 }
  93
  94 void ConvertFromMediaVideoFrame(const scoped_refptr<media::VideoFrame>& src,
  95                                 PP_VideoFrame_Format dst_format,
  96                                 const gfx::Size& dst_size,
  97                                 uint8_t* dst) {
  98   CHECK(src->format() == media::PIXEL_FORMAT_YV12 ||
  99         src->format() == media::PIXEL_FORMAT_I420);
 100   if (dst_format == PP_VIDEOFRAME_FORMAT_BGRA) {
 101     if (src->visible_rect().size() == dst_size) {
 102       libyuv::I420ToARGB(src->visible_data(VideoFrame::kYPlane),
 103                          src->stride(VideoFrame::kYPlane),
 104                          src->visible_data(VideoFrame::kUPlane),
 105                          src->stride(VideoFrame::kUPlane),
 106                          src->visible_data(VideoFrame::kVPlane),
 107                          src->stride(VideoFrame::kVPlane),
 108                          dst,
 109                          dst_size.width() * 4,
 110                          dst_size.width(),
 111                          dst_size.height());
 112     } else {
 113       media::ScaleYUVToRGB32(src->visible_data(VideoFrame::kYPlane),
 114                              src->visible_data(VideoFrame::kUPlane),
 115                              src->visible_data(VideoFrame::kVPlane),
 116                              dst,
 117                              src->visible_rect().width(),
 118                              src->visible_rect().height(),
 119                              dst_size.width(),
 120                              dst_size.height(),
 121                              src->stride(VideoFrame::kYPlane),
 122                              src->stride(VideoFrame::kUPlane),
 123                              dst_size.width() * 4,
 124                              media::YV12,
 125                              media::ROTATE_0,
 126                              media::FILTER_BILINEAR);
 127     }
 128   } else if (dst_format == PP_VIDEOFRAME_FORMAT_YV12 ||
 129              dst_format == PP_VIDEOFRAME_FORMAT_I420) {
 130     static const size_t kPlanesOrder[][3] = {
 131         {VideoFrame::kYPlane, VideoFrame::kVPlane,
 132          VideoFrame::kUPlane},  // YV12
 133         {VideoFrame::kYPlane, VideoFrame::kUPlane,
 134          VideoFrame::kVPlane},  // I420
 135     };
 136     const int plane_order = (dst_format == PP_VIDEOFRAME_FORMAT_YV12) ? 0 : 1;
 137     int dst_width = dst_size.width();
 138     int dst_height = dst_size.height();
 139     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][0]),
 140                        src->stride(kPlanesOrder[plane_order][0]),
 141                        src->visible_rect().width(),
 142                        src->visible_rect().height(),
 143                        dst,
 144                        dst_width,
 145                        dst_width,
 146                        dst_height,
 147                        kFilterMode);
 148     dst += dst_width * dst_height;
 149     const int src_halfwidth = (src->visible_rect().width() + 1) >> 1;
 150     const int src_halfheight = (src->visible_rect().height() + 1) >> 1;
 151     const int dst_halfwidth = (dst_width + 1) >> 1;
 152     const int dst_halfheight = (dst_height + 1) >> 1;
 153     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][1]),
 154                        src->stride(kPlanesOrder[plane_order][1]),
 155                        src_halfwidth,
 156                        src_halfheight,
 157                        dst,
 158                        dst_halfwidth,
 159                        dst_halfwidth,
 160                        dst_halfheight,
 161                        kFilterMode);
 162     dst += dst_halfwidth * dst_halfheight;
 163     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][2]),
 164                        src->stride(kPlanesOrder[plane_order][2]),
 165                        src_halfwidth,
 166                        src_halfheight,
 167                        dst,
 168                        dst_halfwidth,
 169                        dst_halfwidth,
 170                        dst_halfheight,
 171                        kFilterMode);
 172   } else {
 173     NOTREACHED();
 174   }
 175 }
 176
 177 }  // namespace
 178
 179 namespace content {
 180
 181 // Internal class used for delivering video frames on the IO-thread to
 182 // the MediaStreamVideoSource implementation.
 183 class PepperMediaStreamVideoTrackHost::FrameDeliverer
 184     : public base::RefCountedThreadSafe<FrameDeliverer> {
 185  public:
 186   FrameDeliverer(scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
 187                  const VideoCaptureDeliverFrameCB& new_frame_callback);
 188
 189   void DeliverVideoFrame(const scoped_refptr<media::VideoFrame>& frame);
 190
 191  private:
 192   friend class base::RefCountedThreadSafe<FrameDeliverer>;
 193   virtual ~FrameDeliverer();
 194
 195   void DeliverFrameOnIO(const scoped_refptr<media::VideoFrame>& frame);
 196
 197   scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
 198   VideoCaptureDeliverFrameCB new_frame_callback_;
 199
 200   DISALLOW_COPY_AND_ASSIGN(FrameDeliverer);
 201 };
 202
 203 PepperMediaStreamVideoTrackHost::FrameDeliverer::FrameDeliverer(
 204     scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
 205     const VideoCaptureDeliverFrameCB& new_frame_callback)
 206     : io_task_runner_(io_task_runner), new_frame_callback_(new_frame_callback) {
 207 }
 208
 209 PepperMediaStreamVideoTrackHost::FrameDeliverer::~FrameDeliverer() {
 210 }
 211
 212 void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverVideoFrame(
 213     const scoped_refptr<media::VideoFrame>& frame) {
 214   io_task_runner_->PostTask(
 215       FROM_HERE, base::Bind(&FrameDeliverer::DeliverFrameOnIO, this, frame));
 216 }
 217
 218 void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverFrameOnIO(
 219      const scoped_refptr<media::VideoFrame>& frame) {
 220   DCHECK(io_task_runner_->BelongsToCurrentThread());
 221   // The time when this frame is generated is unknown so give a null value to
 222   // |estimated_capture_time|.
 223   new_frame_callback_.Run(frame, base::TimeTicks());
 224 }
 225
 226 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
 227     RendererPpapiHost* host,
 228     PP_Instance instance,
 229     PP_Resource resource,
 230     const blink::WebMediaStreamTrack& track)
 231     : PepperMediaStreamTrackHostBase(host, instance, resource),
 232       track_(track),
 233       connected_(false),
 234       number_of_buffers_(kDefaultNumberOfBuffers),
 235       source_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 236       plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 237       frame_data_size_(0),
 238       type_(kRead),
 239       output_started_(false),
 240       weak_factory_(this) {
 241   DCHECK(!track_.isNull());
 242 }
 243
 244 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
 245     RendererPpapiHost* host,
 246     PP_Instance instance,
 247     PP_Resource resource)
 248     : PepperMediaStreamTrackHostBase(host, instance, resource),
 249       connected_(false),
 250       number_of_buffers_(kDefaultNumberOfBuffers),
 251       source_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 252       plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 253       frame_data_size_(0),
 254       type_(kWrite),
 255       output_started_(false),
 256       weak_factory_(this) {
 257   InitBlinkTrack();
 258   DCHECK(!track_.isNull());
 259 }
 260
 261 bool PepperMediaStreamVideoTrackHost::IsMediaStreamVideoTrackHost() {
 262   return true;
 263 }
 264
 265 PepperMediaStreamVideoTrackHost::~PepperMediaStreamVideoTrackHost() {
 266   OnClose();
 267 }
 268
 269 void PepperMediaStreamVideoTrackHost::InitBuffers() {
 270   gfx::Size size = GetTargetSize(source_frame_size_, plugin_frame_size_);
 271   DCHECK(!size.IsEmpty());
 272
 273   PP_VideoFrame_Format format =
 274       GetTargetFormat(source_frame_format_, plugin_frame_format_);
 275   DCHECK_NE(format, PP_VIDEOFRAME_FORMAT_UNKNOWN);
 276
 277   if (format == PP_VIDEOFRAME_FORMAT_BGRA) {
 278     frame_data_size_ = size.width() * size.height() * 4;
 279   } else {
 280     frame_data_size_ =
 281         VideoFrame::AllocationSize(FromPpapiFormat(format), size);
 282   }
 283
 284   DCHECK_GT(frame_data_size_, 0U);
 285   int32_t buffer_size =
 286       sizeof(ppapi::MediaStreamBuffer::Video) + frame_data_size_;
 287   bool result = PepperMediaStreamTrackHostBase::InitBuffers(number_of_buffers_,
 288                                                             buffer_size,
 289                                                             type_);
 290   CHECK(result);
 291
 292   if (type_ == kWrite) {
 293     for (int32_t i = 0; i < buffer_manager()->number_of_buffers(); ++i) {
 294       ppapi::MediaStreamBuffer::Video* buffer =
 295           &(buffer_manager()->GetBufferPointer(i)->video);
 296       buffer->header.size = buffer_manager()->buffer_size();
 297       buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
 298       buffer->format = format;
 299       buffer->size.width = size.width();
 300       buffer->size.height = size.height();
 301       buffer->data_size = frame_data_size_;
 302     }
 303
 304     // Make all the frames avaiable to the plugin.
 305     std::vector<int32_t> indices = buffer_manager()->DequeueBuffers();
 306     SendEnqueueBuffersMessageToPlugin(indices);
 307   }
 308 }
 309
 310 void PepperMediaStreamVideoTrackHost::OnClose() {
 311   if (connected_) {
 312     MediaStreamVideoSink::RemoveFromVideoTrack(this, track_);
 313     weak_factory_.InvalidateWeakPtrs();
 314     connected_ = false;
 315   }
 316 }
 317
 318 int32_t PepperMediaStreamVideoTrackHost::OnHostMsgEnqueueBuffer(
 319     ppapi::host::HostMessageContext* context, int32_t index) {
 320   if (type_ == kRead) {
 321     return PepperMediaStreamTrackHostBase::OnHostMsgEnqueueBuffer(context,
 322                                                                   index);
 323   } else {
 324     return SendFrameToTrack(index);
 325   }
 326 }
 327
 328 int32_t PepperMediaStreamVideoTrackHost::SendFrameToTrack(int32_t index) {
 329   DCHECK_EQ(type_, kWrite);
 330
 331   if (output_started_) {
 332     // Sends the frame to blink video track.
 333     ppapi::MediaStreamBuffer::Video* pp_frame =
 334         &(buffer_manager()->GetBufferPointer(index)->video);
 335
 336     int32 y_stride = plugin_frame_size_.width();
 337     int32 uv_stride = (plugin_frame_size_.width() + 1) / 2;
 338     uint8* y_data = static_cast<uint8*>(pp_frame->data);
 339     // Default to I420
 340     uint8* u_data = y_data + plugin_frame_size_.GetArea();
 341     uint8* v_data = y_data + (plugin_frame_size_.GetArea() * 5 / 4);
 342     if (plugin_frame_format_ == PP_VIDEOFRAME_FORMAT_YV12) {
 343       // Swap u and v for YV12.
 344       uint8* tmp = u_data;
 345       u_data = v_data;
 346       v_data = tmp;
 347     }
 348
 349     int64 ts_ms = static_cast<int64>(pp_frame->timestamp *
 350                                      base::Time::kMillisecondsPerSecond);
 351     scoped_refptr<VideoFrame> frame = media::VideoFrame::WrapExternalYuvData(
 352         FromPpapiFormat(plugin_frame_format_),
 353         plugin_frame_size_,
 354         gfx::Rect(plugin_frame_size_),
 355         plugin_frame_size_,
 356         y_stride,
 357         uv_stride,
 358         uv_stride,
 359         y_data,
 360         u_data,
 361         v_data,
 362         base::TimeDelta::FromMilliseconds(ts_ms));
 363
 364     frame_deliverer_->DeliverVideoFrame(frame);
 365   }
 366
 367   // Makes the frame available again for plugin.
 368   SendEnqueueBufferMessageToPlugin(index);
 369   return PP_OK;
 370 }
 371
 372 void PepperMediaStreamVideoTrackHost::OnVideoFrame(
 373     const scoped_refptr<VideoFrame>& frame,
 374     base::TimeTicks estimated_capture_time) {
 375   DCHECK(frame.get());
 376   // TODO(penghuang): Check |frame->end_of_stream()| and close the track.
 377   PP_VideoFrame_Format ppformat = ToPpapiFormat(frame->format());
 378   if (ppformat == PP_VIDEOFRAME_FORMAT_UNKNOWN)
 379     return;
 380
 381   if (source_frame_size_.IsEmpty()) {
 382     source_frame_size_ = frame->visible_rect().size();
 383     source_frame_format_ = ppformat;
 384     InitBuffers();
 385   }
 386
 387   int32_t index = buffer_manager()->DequeueBuffer();
 388   // Drop frames if the underlying buffer is full.
 389   if (index < 0) {
 390     DVLOG(1) << "A frame is dropped.";
 391     return;
 392   }
 393
 394   CHECK_EQ(ppformat, source_frame_format_) << "Frame format is changed.";
 395
 396   gfx::Size size = GetTargetSize(source_frame_size_, plugin_frame_size_);
 397   ppformat =
 398       GetTargetFormat(source_frame_format_, plugin_frame_format_);
 399   ppapi::MediaStreamBuffer::Video* buffer =
 400       &(buffer_manager()->GetBufferPointer(index)->video);
 401   buffer->header.size = buffer_manager()->buffer_size();
 402   buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
 403   buffer->timestamp = frame->timestamp().InSecondsF();
 404   buffer->format = ppformat;
 405   buffer->size.width = size.width();
 406   buffer->size.height = size.height();
 407   buffer->data_size = frame_data_size_;
 408   ConvertFromMediaVideoFrame(frame, ppformat, size, buffer->data);
 409
 410   SendEnqueueBufferMessageToPlugin(index);
 411 }
 412
 413 void PepperMediaStreamVideoTrackHost::GetCurrentSupportedFormats(
 414     int max_requested_width, int max_requested_height,
 415     double max_requested_frame_rate,
 416     const VideoCaptureDeviceFormatsCB& callback) {
 417   if (type_ != kWrite) {
 418     DVLOG(1) << "GetCurrentSupportedFormats is only supported in output mode.";
 419     callback.Run(media::VideoCaptureFormats());
 420     return;
 421   }
 422
 423   media::VideoCaptureFormats formats;
 424   formats.push_back(
 425       media::VideoCaptureFormat(plugin_frame_size_,
 426                                 kDefaultOutputFrameRate,
 427                                 ToPixelFormat(plugin_frame_format_)));
 428   callback.Run(formats);
 429 }
 430
 431 void PepperMediaStreamVideoTrackHost::StartSourceImpl(
 432     const media::VideoCaptureFormat& format,
 433     const blink::WebMediaConstraints& constraints,
 434     const VideoCaptureDeliverFrameCB& frame_callback) {
 435   output_started_ = true;
 436   frame_deliverer_ = new FrameDeliverer(io_task_runner(), frame_callback);
 437 }
 438
 439 void PepperMediaStreamVideoTrackHost::StopSourceImpl() {
 440   output_started_ = false;
 441   frame_deliverer_ = NULL;
 442 }
 443
 444 void PepperMediaStreamVideoTrackHost::DidConnectPendingHostToResource() {
 445   if (!connected_) {
 446     MediaStreamVideoSink::AddToVideoTrack(
 447         this,
 448         media::BindToCurrentLoop(
 449             base::Bind(
 450                 &PepperMediaStreamVideoTrackHost::OnVideoFrame,
 451                 weak_factory_.GetWeakPtr())),
 452         track_);
 453     connected_ = true;
 454   }
 455 }
 456
 457 int32_t PepperMediaStreamVideoTrackHost::OnResourceMessageReceived(
 458     const IPC::Message& msg,
 459     HostMessageContext* context) {
 460   PPAPI_BEGIN_MESSAGE_MAP(PepperMediaStreamVideoTrackHost, msg)
 461     PPAPI_DISPATCH_HOST_RESOURCE_CALL(
 462         PpapiHostMsg_MediaStreamVideoTrack_Configure, OnHostMsgConfigure)
 463   PPAPI_END_MESSAGE_MAP()
 464   return PepperMediaStreamTrackHostBase::OnResourceMessageReceived(msg,
 465                                                                    context);
 466 }
 467
 468 int32_t PepperMediaStreamVideoTrackHost::OnHostMsgConfigure(
 469     HostMessageContext* context,
 470     const MediaStreamVideoTrackShared::Attributes& attributes) {
 471   CHECK(MediaStreamVideoTrackShared::VerifyAttributes(attributes));
 472
 473   bool changed = false;
 474   gfx::Size new_size(attributes.width, attributes.height);
 475   if (GetTargetSize(source_frame_size_, plugin_frame_size_) !=
 476       GetTargetSize(source_frame_size_, new_size)) {
 477     changed = true;
 478   }
 479   plugin_frame_size_ = new_size;
 480
 481   int32_t buffers = attributes.buffers
 482                         ? std::min(kMaxNumberOfBuffers, attributes.buffers)
 483                         : kDefaultNumberOfBuffers;
 484   if (buffers != number_of_buffers_)
 485     changed = true;
 486   number_of_buffers_ = buffers;
 487
 488   if (GetTargetFormat(source_frame_format_, plugin_frame_format_) !=
 489       GetTargetFormat(source_frame_format_, attributes.format)) {
 490     changed = true;
 491   }
 492   plugin_frame_format_ = attributes.format;
 493
 494   // If the first frame has been received, we will re-initialize buffers with
 495   // new settings. Otherwise, we will initialize buffer when we receive
 496   // the first frame, because plugin can only provide part of attributes
 497   // which are not enough to initialize buffers.
 498   if (changed && (type_ == kWrite || !source_frame_size_.IsEmpty()))
 499     InitBuffers();
 500
 501   // TODO(ronghuawu): Ask the owner of DOMMediaStreamTrackToResource why
 502   // source id instead of track id is used there.
 503   const std::string id = track_.source().id().utf8();
 504   context->reply_msg = PpapiPluginMsg_MediaStreamVideoTrack_ConfigureReply(id);
 505   return PP_OK;
 506 }
 507
 508 void PepperMediaStreamVideoTrackHost::InitBlinkTrack() {
 509   std::string source_id;
 510   base::Base64Encode(base::RandBytesAsString(64), &source_id);
 511   blink::WebMediaStreamSource webkit_source;
 512   webkit_source.initialize(base::UTF8ToUTF16(source_id),
 513                            blink::WebMediaStreamSource::TypeVideo,
 514                            base::UTF8ToUTF16(kPepperVideoSourceName),
 515                            false /* remote */, true /* readonly */);
 516   webkit_source.setExtraData(this);
 517
 518   const bool enabled = true;
 519   blink::WebMediaConstraints constraints;
 520   constraints.initialize();
 521   track_ = MediaStreamVideoTrack::CreateVideoTrack(
 522        this, constraints,
 523        base::Bind(
 524            &PepperMediaStreamVideoTrackHost::OnTrackStarted,
 525            base::Unretained(this)),
 526        enabled);
 527 }
 528
 529 void PepperMediaStreamVideoTrackHost::OnTrackStarted(
 530     MediaStreamSource* source,
 531     MediaStreamRequestResult result,
 532     const blink::WebString& result_name) {
 533   DVLOG(3) << "OnTrackStarted result: " << result;
 534 }
 535
 536 }  // namespace content