content/renderer/pepper/pepper_media_stream_video_track_host.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/pepper/pepper_media_stream_video_track_host.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/rand_util.h"
  10 #include "base/strings/utf_string_conversions.h"
  11 #include "content/renderer/media/media_stream_video_track.h"
  12 #include "media/base/bind_to_current_loop.h"
  13 #include "media/base/yuv_convert.h"
  14 #include "ppapi/c/pp_errors.h"
  15 #include "ppapi/c/ppb_media_stream_video_track.h"
  16 #include "ppapi/c/ppb_video_frame.h"
  17 #include "ppapi/host/dispatch_host_message.h"
  18 #include "ppapi/host/host_message_context.h"
  19 #include "ppapi/proxy/ppapi_messages.h"
  20 #include "ppapi/shared_impl/media_stream_buffer.h"
  21 #include "third_party/libyuv/include/libyuv.h"
  22
  23 using media::VideoFrame;
  24 using ppapi::host::HostMessageContext;
  25 using ppapi::MediaStreamVideoTrackShared;
  26
  27 namespace {
  28
  29 const int32_t kDefaultNumberOfBuffers = 4;
  30 const int32_t kMaxNumberOfBuffers = 8;
  31 // Filter mode for scaling frames.
  32 const libyuv::FilterMode kFilterMode = libyuv::kFilterBox;
  33
  34 const char kPepperVideoSourceName[] = "PepperVideoSourceName";
  35
  36 // Default config for output mode.
  37 const int kDefaultOutputFrameRate = 30;
  38
  39 media::VideoPixelFormat ToPixelFormat(PP_VideoFrame_Format format) {
  40   switch (format) {
  41     case PP_VIDEOFRAME_FORMAT_YV12:
  42       return media::PIXEL_FORMAT_YV12;
  43     case PP_VIDEOFRAME_FORMAT_I420:
  44       return media::PIXEL_FORMAT_I420;
  45     default:
  46       DVLOG(1) << "Unsupported pixel format " << format;
  47       return media::PIXEL_FORMAT_UNKNOWN;
  48   }
  49 }
  50
  51 PP_VideoFrame_Format ToPpapiFormat(media::VideoPixelFormat format) {
  52   switch (format) {
  53     case media::PIXEL_FORMAT_YV12:
  54       return PP_VIDEOFRAME_FORMAT_YV12;
  55     case media::PIXEL_FORMAT_I420:
  56       return PP_VIDEOFRAME_FORMAT_I420;
  57     default:
  58       DVLOG(1) << "Unsupported pixel format " << format;
  59       return PP_VIDEOFRAME_FORMAT_UNKNOWN;
  60   }
  61 }
  62
  63 media::VideoPixelFormat FromPpapiFormat(PP_VideoFrame_Format format) {
  64   switch (format) {
  65     case PP_VIDEOFRAME_FORMAT_YV12:
  66       return media::PIXEL_FORMAT_YV12;
  67     case PP_VIDEOFRAME_FORMAT_I420:
  68       return media::PIXEL_FORMAT_I420;
  69     default:
  70       DVLOG(1) << "Unsupported pixel format " << format;
  71       return media::PIXEL_FORMAT_UNKNOWN;
  72   }
  73 }
  74
  75 // Compute size base on the size of frame received from MediaStreamVideoSink
  76 // and size specified by plugin.
  77 gfx::Size GetTargetSize(const gfx::Size& source, const gfx::Size& plugin) {
  78   return gfx::Size(plugin.width() ? plugin.width() : source.width(),
  79                    plugin.height() ? plugin.height() : source.height());
  80 }
  81
  82 // Compute format base on the format of frame received from MediaStreamVideoSink
  83 // and format specified by plugin.
  84 PP_VideoFrame_Format GetTargetFormat(PP_VideoFrame_Format source,
  85                                      PP_VideoFrame_Format plugin) {
  86   return plugin != PP_VIDEOFRAME_FORMAT_UNKNOWN ? plugin : source;
  87 }
  88
  89 void ConvertFromMediaVideoFrame(const scoped_refptr<media::VideoFrame>& src,
  90                                 PP_VideoFrame_Format dst_format,
  91                                 const gfx::Size& dst_size,
  92                                 uint8_t* dst) {
  93   CHECK(src->format() == media::PIXEL_FORMAT_YV12 ||
  94         src->format() == media::PIXEL_FORMAT_I420);
  95   if (dst_format == PP_VIDEOFRAME_FORMAT_BGRA) {
  96     if (src->visible_rect().size() == dst_size) {
  97       libyuv::I420ToARGB(src->visible_data(VideoFrame::kYPlane),
  98                          src->stride(VideoFrame::kYPlane),
  99                          src->visible_data(VideoFrame::kUPlane),
 100                          src->stride(VideoFrame::kUPlane),
 101                          src->visible_data(VideoFrame::kVPlane),
 102                          src->stride(VideoFrame::kVPlane),
 103                          dst,
 104                          dst_size.width() * 4,
 105                          dst_size.width(),
 106                          dst_size.height());
 107     } else {
 108       media::ScaleYUVToRGB32(src->visible_data(VideoFrame::kYPlane),
 109                              src->visible_data(VideoFrame::kUPlane),
 110                              src->visible_data(VideoFrame::kVPlane),
 111                              dst,
 112                              src->visible_rect().width(),
 113                              src->visible_rect().height(),
 114                              dst_size.width(),
 115                              dst_size.height(),
 116                              src->stride(VideoFrame::kYPlane),
 117                              src->stride(VideoFrame::kUPlane),
 118                              dst_size.width() * 4,
 119                              media::YV12,
 120                              media::ROTATE_0,
 121                              media::FILTER_BILINEAR);
 122     }
 123   } else if (dst_format == PP_VIDEOFRAME_FORMAT_YV12 ||
 124              dst_format == PP_VIDEOFRAME_FORMAT_I420) {
 125     static const size_t kPlanesOrder[][3] = {
 126         {VideoFrame::kYPlane, VideoFrame::kVPlane,
 127          VideoFrame::kUPlane},  // YV12
 128         {VideoFrame::kYPlane, VideoFrame::kUPlane,
 129          VideoFrame::kVPlane},  // I420
 130     };
 131     const int plane_order = (dst_format == PP_VIDEOFRAME_FORMAT_YV12) ? 0 : 1;
 132     int dst_width = dst_size.width();
 133     int dst_height = dst_size.height();
 134     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][0]),
 135                        src->stride(kPlanesOrder[plane_order][0]),
 136                        src->visible_rect().width(),
 137                        src->visible_rect().height(),
 138                        dst,
 139                        dst_width,
 140                        dst_width,
 141                        dst_height,
 142                        kFilterMode);
 143     dst += dst_width * dst_height;
 144     const int src_halfwidth = (src->visible_rect().width() + 1) >> 1;
 145     const int src_halfheight = (src->visible_rect().height() + 1) >> 1;
 146     const int dst_halfwidth = (dst_width + 1) >> 1;
 147     const int dst_halfheight = (dst_height + 1) >> 1;
 148     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][1]),
 149                        src->stride(kPlanesOrder[plane_order][1]),
 150                        src_halfwidth,
 151                        src_halfheight,
 152                        dst,
 153                        dst_halfwidth,
 154                        dst_halfwidth,
 155                        dst_halfheight,
 156                        kFilterMode);
 157     dst += dst_halfwidth * dst_halfheight;
 158     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][2]),
 159                        src->stride(kPlanesOrder[plane_order][2]),
 160                        src_halfwidth,
 161                        src_halfheight,
 162                        dst,
 163                        dst_halfwidth,
 164                        dst_halfwidth,
 165                        dst_halfheight,
 166                        kFilterMode);
 167   } else {
 168     NOTREACHED();
 169   }
 170 }
 171
 172 }  // namespace
 173
 174 namespace content {
 175
 176 // Internal class used for delivering video frames on the IO-thread to
 177 // the MediaStreamVideoSource implementation.
 178 class PepperMediaStreamVideoTrackHost::FrameDeliverer
 179     : public base::RefCountedThreadSafe<FrameDeliverer> {
 180  public:
 181   FrameDeliverer(scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
 182                  const VideoCaptureDeliverFrameCB& new_frame_callback);
 183
 184   void DeliverVideoFrame(const scoped_refptr<media::VideoFrame>& frame);
 185
 186  private:
 187   friend class base::RefCountedThreadSafe<FrameDeliverer>;
 188   virtual ~FrameDeliverer();
 189
 190   void DeliverFrameOnIO(const scoped_refptr<media::VideoFrame>& frame);
 191
 192   scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
 193   VideoCaptureDeliverFrameCB new_frame_callback_;
 194
 195   DISALLOW_COPY_AND_ASSIGN(FrameDeliverer);
 196 };
 197
 198 PepperMediaStreamVideoTrackHost::FrameDeliverer::FrameDeliverer(
 199     scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
 200     const VideoCaptureDeliverFrameCB& new_frame_callback)
 201     : io_task_runner_(io_task_runner), new_frame_callback_(new_frame_callback) {
 202 }
 203
 204 PepperMediaStreamVideoTrackHost::FrameDeliverer::~FrameDeliverer() {
 205 }
 206
 207 void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverVideoFrame(
 208     const scoped_refptr<media::VideoFrame>& frame) {
 209   io_task_runner_->PostTask(
 210       FROM_HERE, base::Bind(&FrameDeliverer::DeliverFrameOnIO, this, frame));
 211 }
 212
 213 void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverFrameOnIO(
 214      const scoped_refptr<media::VideoFrame>& frame) {
 215   DCHECK(io_task_runner_->BelongsToCurrentThread());
 216   // The time when this frame is generated is unknown so give a null value to
 217   // |estimated_capture_time|.
 218   new_frame_callback_.Run(frame, base::TimeTicks());
 219 }
 220
 221 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
 222     RendererPpapiHost* host,
 223     PP_Instance instance,
 224     PP_Resource resource,
 225     const blink::WebMediaStreamTrack& track)
 226     : PepperMediaStreamTrackHostBase(host, instance, resource),
 227       track_(track),
 228       connected_(false),
 229       number_of_buffers_(kDefaultNumberOfBuffers),
 230       source_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 231       plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 232       frame_data_size_(0),
 233       type_(kRead),
 234       output_started_(false),
 235       weak_factory_(this) {
 236   DCHECK(!track_.isNull());
 237 }
 238
 239 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
 240     RendererPpapiHost* host,
 241     PP_Instance instance,
 242     PP_Resource resource)
 243     : PepperMediaStreamTrackHostBase(host, instance, resource),
 244       connected_(false),
 245       number_of_buffers_(kDefaultNumberOfBuffers),
 246       source_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 247       plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 248       frame_data_size_(0),
 249       type_(kWrite),
 250       output_started_(false),
 251       weak_factory_(this) {
 252   InitBlinkTrack();
 253   DCHECK(!track_.isNull());
 254 }
 255
 256 bool PepperMediaStreamVideoTrackHost::IsMediaStreamVideoTrackHost() {
 257   return true;
 258 }
 259
 260 PepperMediaStreamVideoTrackHost::~PepperMediaStreamVideoTrackHost() {
 261   OnClose();
 262 }
 263
 264 void PepperMediaStreamVideoTrackHost::InitBuffers() {
 265   gfx::Size size = GetTargetSize(source_frame_size_, plugin_frame_size_);
 266   DCHECK(!size.IsEmpty());
 267
 268   PP_VideoFrame_Format format =
 269       GetTargetFormat(source_frame_format_, plugin_frame_format_);
 270   DCHECK_NE(format, PP_VIDEOFRAME_FORMAT_UNKNOWN);
 271
 272   if (format == PP_VIDEOFRAME_FORMAT_BGRA) {
 273     frame_data_size_ = size.width() * size.height() * 4;
 274   } else {
 275     frame_data_size_ =
 276         VideoFrame::AllocationSize(FromPpapiFormat(format), size);
 277   }
 278
 279   DCHECK_GT(frame_data_size_, 0U);
 280   int32_t buffer_size =
 281       sizeof(ppapi::MediaStreamBuffer::Video) + frame_data_size_;
 282   bool result = PepperMediaStreamTrackHostBase::InitBuffers(number_of_buffers_,
 283                                                             buffer_size,
 284                                                             type_);
 285   CHECK(result);
 286
 287   if (type_ == kWrite) {
 288     for (int32_t i = 0; i < buffer_manager()->number_of_buffers(); ++i) {
 289       ppapi::MediaStreamBuffer::Video* buffer =
 290           &(buffer_manager()->GetBufferPointer(i)->video);
 291       buffer->header.size = buffer_manager()->buffer_size();
 292       buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
 293       buffer->format = format;
 294       buffer->size.width = size.width();
 295       buffer->size.height = size.height();
 296       buffer->data_size = frame_data_size_;
 297     }
 298
 299     // Make all the frames avaiable to the plugin.
 300     std::vector<int32_t> indices = buffer_manager()->DequeueBuffers();
 301     SendEnqueueBuffersMessageToPlugin(indices);
 302   }
 303 }
 304
 305 void PepperMediaStreamVideoTrackHost::OnClose() {
 306   if (connected_) {
 307     MediaStreamVideoSink::RemoveFromVideoTrack(this, track_);
 308     weak_factory_.InvalidateWeakPtrs();
 309     connected_ = false;
 310   }
 311 }
 312
 313 int32_t PepperMediaStreamVideoTrackHost::OnHostMsgEnqueueBuffer(
 314     ppapi::host::HostMessageContext* context, int32_t index) {
 315   if (type_ == kRead) {
 316     return PepperMediaStreamTrackHostBase::OnHostMsgEnqueueBuffer(context,
 317                                                                   index);
 318   } else {
 319     return SendFrameToTrack(index);
 320   }
 321 }
 322
 323 int32_t PepperMediaStreamVideoTrackHost::SendFrameToTrack(int32_t index) {
 324   DCHECK_EQ(type_, kWrite);
 325
 326   if (output_started_) {
 327     // Sends the frame to blink video track.
 328     ppapi::MediaStreamBuffer::Video* pp_frame =
 329         &(buffer_manager()->GetBufferPointer(index)->video);
 330
 331     int32 y_stride = plugin_frame_size_.width();
 332     int32 uv_stride = (plugin_frame_size_.width() + 1) / 2;
 333     uint8* y_data = static_cast<uint8*>(pp_frame->data);
 334     // Default to I420
 335     uint8* u_data = y_data + plugin_frame_size_.GetArea();
 336     uint8* v_data = y_data + (plugin_frame_size_.GetArea() * 5 / 4);
 337     if (plugin_frame_format_ == PP_VIDEOFRAME_FORMAT_YV12) {
 338       // Swap u and v for YV12.
 339       uint8* tmp = u_data;
 340       u_data = v_data;
 341       v_data = tmp;
 342     }
 343
 344     int64 ts_ms = static_cast<int64>(pp_frame->timestamp *
 345                                      base::Time::kMillisecondsPerSecond);
 346     scoped_refptr<VideoFrame> frame = media::VideoFrame::WrapExternalYuvData(
 347         FromPpapiFormat(plugin_frame_format_),
 348         plugin_frame_size_,
 349         gfx::Rect(plugin_frame_size_),
 350         plugin_frame_size_,
 351         y_stride,
 352         uv_stride,
 353         uv_stride,
 354         y_data,
 355         u_data,
 356         v_data,
 357         base::TimeDelta::FromMilliseconds(ts_ms));
 358
 359     frame_deliverer_->DeliverVideoFrame(frame);
 360   }
 361
 362   // Makes the frame available again for plugin.
 363   SendEnqueueBufferMessageToPlugin(index);
 364   return PP_OK;
 365 }
 366
 367 void PepperMediaStreamVideoTrackHost::OnVideoFrame(
 368     const scoped_refptr<VideoFrame>& frame,
 369     base::TimeTicks estimated_capture_time) {
 370   DCHECK(frame.get());
 371   // TODO(penghuang): Check |frame->end_of_stream()| and close the track.
 372   PP_VideoFrame_Format ppformat = ToPpapiFormat(frame->format());
 373   if (ppformat == PP_VIDEOFRAME_FORMAT_UNKNOWN)
 374     return;
 375
 376   if (source_frame_size_.IsEmpty()) {
 377     source_frame_size_ = frame->visible_rect().size();
 378     source_frame_format_ = ppformat;
 379     InitBuffers();
 380   }
 381
 382   int32_t index = buffer_manager()->DequeueBuffer();
 383   // Drop frames if the underlying buffer is full.
 384   if (index < 0) {
 385     DVLOG(1) << "A frame is dropped.";
 386     return;
 387   }
 388
 389   CHECK_EQ(ppformat, source_frame_format_) << "Frame format is changed.";
 390
 391   gfx::Size size = GetTargetSize(source_frame_size_, plugin_frame_size_);
 392   ppformat =
 393       GetTargetFormat(source_frame_format_, plugin_frame_format_);
 394   ppapi::MediaStreamBuffer::Video* buffer =
 395       &(buffer_manager()->GetBufferPointer(index)->video);
 396   buffer->header.size = buffer_manager()->buffer_size();
 397   buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
 398   buffer->timestamp = frame->timestamp().InSecondsF();
 399   buffer->format = ppformat;
 400   buffer->size.width = size.width();
 401   buffer->size.height = size.height();
 402   buffer->data_size = frame_data_size_;
 403   ConvertFromMediaVideoFrame(frame, ppformat, size, buffer->data);
 404
 405   SendEnqueueBufferMessageToPlugin(index);
 406 }
 407
 408 void PepperMediaStreamVideoTrackHost::GetCurrentSupportedFormats(
 409     int max_requested_width, int max_requested_height,
 410     double max_requested_frame_rate,
 411     const VideoCaptureDeviceFormatsCB& callback) {
 412   if (type_ != kWrite) {
 413     DVLOG(1) << "GetCurrentSupportedFormats is only supported in output mode.";
 414     callback.Run(media::VideoCaptureFormats());
 415     return;
 416   }
 417
 418   media::VideoCaptureFormats formats;
 419   formats.push_back(
 420       media::VideoCaptureFormat(plugin_frame_size_,
 421                                 kDefaultOutputFrameRate,
 422                                 ToPixelFormat(plugin_frame_format_)));
 423   callback.Run(formats);
 424 }
 425
 426 void PepperMediaStreamVideoTrackHost::StartSourceImpl(
 427     const media::VideoCaptureFormat& format,
 428     const blink::WebMediaConstraints& constraints,
 429     const VideoCaptureDeliverFrameCB& frame_callback) {
 430   output_started_ = true;
 431   frame_deliverer_ = new FrameDeliverer(io_task_runner(), frame_callback);
 432 }
 433
 434 void PepperMediaStreamVideoTrackHost::StopSourceImpl() {
 435   output_started_ = false;
 436   frame_deliverer_ = NULL;
 437 }
 438
 439 void PepperMediaStreamVideoTrackHost::DidConnectPendingHostToResource() {
 440   if (!connected_) {
 441     MediaStreamVideoSink::AddToVideoTrack(
 442         this,
 443         media::BindToCurrentLoop(
 444             base::Bind(
 445                 &PepperMediaStreamVideoTrackHost::OnVideoFrame,
 446                 weak_factory_.GetWeakPtr())),
 447         track_);
 448     connected_ = true;
 449   }
 450 }
 451
 452 int32_t PepperMediaStreamVideoTrackHost::OnResourceMessageReceived(
 453     const IPC::Message& msg,
 454     HostMessageContext* context) {
 455   PPAPI_BEGIN_MESSAGE_MAP(PepperMediaStreamVideoTrackHost, msg)
 456     PPAPI_DISPATCH_HOST_RESOURCE_CALL(
 457         PpapiHostMsg_MediaStreamVideoTrack_Configure, OnHostMsgConfigure)
 458   PPAPI_END_MESSAGE_MAP()
 459   return PepperMediaStreamTrackHostBase::OnResourceMessageReceived(msg,
 460                                                                    context);
 461 }
 462
 463 int32_t PepperMediaStreamVideoTrackHost::OnHostMsgConfigure(
 464     HostMessageContext* context,
 465     const MediaStreamVideoTrackShared::Attributes& attributes) {
 466   CHECK(MediaStreamVideoTrackShared::VerifyAttributes(attributes));
 467
 468   bool changed = false;
 469   gfx::Size new_size(attributes.width, attributes.height);
 470   if (GetTargetSize(source_frame_size_, plugin_frame_size_) !=
 471       GetTargetSize(source_frame_size_, new_size)) {
 472     changed = true;
 473   }
 474   plugin_frame_size_ = new_size;
 475
 476   int32_t buffers = attributes.buffers
 477                         ? std::min(kMaxNumberOfBuffers, attributes.buffers)
 478                         : kDefaultNumberOfBuffers;
 479   if (buffers != number_of_buffers_)
 480     changed = true;
 481   number_of_buffers_ = buffers;
 482
 483   if (GetTargetFormat(source_frame_format_, plugin_frame_format_) !=
 484       GetTargetFormat(source_frame_format_, attributes.format)) {
 485     changed = true;
 486   }
 487   plugin_frame_format_ = attributes.format;
 488
 489   // If the first frame has been received, we will re-initialize buffers with
 490   // new settings. Otherwise, we will initialize buffer when we receive
 491   // the first frame, because plugin can only provide part of attributes
 492   // which are not enough to initialize buffers.
 493   if (changed && (type_ == kWrite || !source_frame_size_.IsEmpty()))
 494     InitBuffers();
 495
 496   // TODO(ronghuawu): Ask the owner of DOMMediaStreamTrackToResource why
 497   // source id instead of track id is used there.
 498   const std::string id = track_.source().id().utf8();
 499   context->reply_msg = PpapiPluginMsg_MediaStreamVideoTrack_ConfigureReply(id);
 500   return PP_OK;
 501 }
 502
 503 void PepperMediaStreamVideoTrackHost::InitBlinkTrack() {
 504   std::string source_id;
 505   base::Base64Encode(base::RandBytesAsString(64), &source_id);
 506   blink::WebMediaStreamSource webkit_source;
 507   webkit_source.initialize(base::UTF8ToUTF16(source_id),
 508                            blink::WebMediaStreamSource::TypeVideo,
 509                            base::UTF8ToUTF16(kPepperVideoSourceName),
 510                            false /* remote */, true /* readonly */);
 511   webkit_source.setExtraData(this);
 512
 513   const bool enabled = true;
 514   blink::WebMediaConstraints constraints;
 515   constraints.initialize();
 516   track_ = MediaStreamVideoTrack::CreateVideoTrack(
 517        this, constraints,
 518        base::Bind(
 519            &PepperMediaStreamVideoTrackHost::OnTrackStarted,
 520            base::Unretained(this)),
 521        enabled);
 522 }
 523
 524 void PepperMediaStreamVideoTrackHost::OnTrackStarted(
 525     MediaStreamSource* source,
 526     MediaStreamRequestResult result,
 527     const blink::WebString& result_name) {
 528   DVLOG(3) << "OnTrackStarted result: " << result;
 529 }
 530
 531 }  // namespace content