content/renderer/pepper/pepper_media_stream_video_track_host.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/pepper/pepper_media_stream_video_track_host.h"
   6
   7 #include "base/base64.h"
   8 #include "base/logging.h"
   9 #include "base/rand_util.h"
  10 #include "base/strings/utf_string_conversions.h"
  11 #include "content/renderer/media/media_stream_video_track.h"
  12 #include "media/base/bind_to_current_loop.h"
  13 #include "media/base/yuv_convert.h"
  14 #include "ppapi/c/pp_errors.h"
  15 #include "ppapi/c/ppb_media_stream_video_track.h"
  16 #include "ppapi/c/ppb_video_frame.h"
  17 #include "ppapi/host/dispatch_host_message.h"
  18 #include "ppapi/host/host_message_context.h"
  19 #include "ppapi/proxy/ppapi_messages.h"
  20 #include "ppapi/shared_impl/media_stream_buffer.h"
  21
  22 // IS_ALIGNED is also defined in
  23 // third_party/webrtc/overrides/webrtc/base/basictypes.h
  24 // TODO(ronghuawu): Avoid undef.
  25 #undef IS_ALIGNED
  26 #include "third_party/libyuv/include/libyuv.h"
  27
  28 using media::VideoFrame;
  29 using ppapi::host::HostMessageContext;
  30 using ppapi::MediaStreamVideoTrackShared;
  31
  32 namespace {
  33
  34 const int32_t kDefaultNumberOfBuffers = 4;
  35 const int32_t kMaxNumberOfBuffers = 8;
  36 // Filter mode for scaling frames.
  37 const libyuv::FilterMode kFilterMode = libyuv::kFilterBox;
  38
  39 const char kPepperVideoSourceName[] = "PepperVideoSourceName";
  40
  41 // Default config for output mode.
  42 const int kDefaultOutputFrameRate = 30;
  43
  44 media::VideoPixelFormat ToPixelFormat(PP_VideoFrame_Format format) {
  45   switch (format) {
  46     case PP_VIDEOFRAME_FORMAT_YV12:
  47       return media::PIXEL_FORMAT_YV12;
  48     case PP_VIDEOFRAME_FORMAT_I420:
  49       return media::PIXEL_FORMAT_I420;
  50     default:
  51       DVLOG(1) << "Unsupported pixel format " << format;
  52       return media::PIXEL_FORMAT_UNKNOWN;
  53   }
  54 }
  55
  56 PP_VideoFrame_Format ToPpapiFormat(VideoFrame::Format format) {
  57   switch (format) {
  58     case VideoFrame::YV12:
  59       return PP_VIDEOFRAME_FORMAT_YV12;
  60     case VideoFrame::I420:
  61       return PP_VIDEOFRAME_FORMAT_I420;
  62     default:
  63       DVLOG(1) << "Unsupported pixel format " << format;
  64       return PP_VIDEOFRAME_FORMAT_UNKNOWN;
  65   }
  66 }
  67
  68 VideoFrame::Format FromPpapiFormat(PP_VideoFrame_Format format) {
  69   switch (format) {
  70     case PP_VIDEOFRAME_FORMAT_YV12:
  71       return VideoFrame::YV12;
  72     case PP_VIDEOFRAME_FORMAT_I420:
  73       return VideoFrame::I420;
  74     default:
  75       DVLOG(1) << "Unsupported pixel format " << format;
  76       return VideoFrame::UNKNOWN;
  77   }
  78 }
  79
  80 // Compute size base on the size of frame received from MediaStreamVideoSink
  81 // and size specified by plugin.
  82 gfx::Size GetTargetSize(const gfx::Size& source, const gfx::Size& plugin) {
  83   return gfx::Size(plugin.width() ? plugin.width() : source.width(),
  84                    plugin.height() ? plugin.height() : source.height());
  85 }
  86
  87 // Compute format base on the format of frame received from MediaStreamVideoSink
  88 // and format specified by plugin.
  89 PP_VideoFrame_Format GetTargetFormat(PP_VideoFrame_Format source,
  90                                      PP_VideoFrame_Format plugin) {
  91   return plugin != PP_VIDEOFRAME_FORMAT_UNKNOWN ? plugin : source;
  92 }
  93
  94 void ConvertFromMediaVideoFrame(const scoped_refptr<media::VideoFrame>& src,
  95                                 PP_VideoFrame_Format dst_format,
  96                                 const gfx::Size& dst_size,
  97                                 uint8_t* dst) {
  98   CHECK(src->format() == VideoFrame::YV12 || src->format() == VideoFrame::I420);
  99   if (dst_format == PP_VIDEOFRAME_FORMAT_BGRA) {
 100     if (src->visible_rect().size() == dst_size) {
 101       libyuv::I420ToARGB(src->visible_data(VideoFrame::kYPlane),
 102                          src->stride(VideoFrame::kYPlane),
 103                          src->visible_data(VideoFrame::kUPlane),
 104                          src->stride(VideoFrame::kUPlane),
 105                          src->visible_data(VideoFrame::kVPlane),
 106                          src->stride(VideoFrame::kVPlane),
 107                          dst,
 108                          dst_size.width() * 4,
 109                          dst_size.width(),
 110                          dst_size.height());
 111     } else {
 112       media::ScaleYUVToRGB32(src->visible_data(VideoFrame::kYPlane),
 113                              src->visible_data(VideoFrame::kUPlane),
 114                              src->visible_data(VideoFrame::kVPlane),
 115                              dst,
 116                              src->visible_rect().width(),
 117                              src->visible_rect().height(),
 118                              dst_size.width(),
 119                              dst_size.height(),
 120                              src->stride(VideoFrame::kYPlane),
 121                              src->stride(VideoFrame::kUPlane),
 122                              dst_size.width() * 4,
 123                              media::YV12,
 124                              media::ROTATE_0,
 125                              media::FILTER_BILINEAR);
 126     }
 127   } else if (dst_format == PP_VIDEOFRAME_FORMAT_YV12 ||
 128              dst_format == PP_VIDEOFRAME_FORMAT_I420) {
 129     static const size_t kPlanesOrder[][3] = {
 130         {VideoFrame::kYPlane, VideoFrame::kVPlane,
 131          VideoFrame::kUPlane},  // YV12
 132         {VideoFrame::kYPlane, VideoFrame::kUPlane,
 133          VideoFrame::kVPlane},  // I420
 134     };
 135     const int plane_order = (dst_format == PP_VIDEOFRAME_FORMAT_YV12) ? 0 : 1;
 136     int dst_width = dst_size.width();
 137     int dst_height = dst_size.height();
 138     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][0]),
 139                        src->stride(kPlanesOrder[plane_order][0]),
 140                        src->visible_rect().width(),
 141                        src->visible_rect().height(),
 142                        dst,
 143                        dst_width,
 144                        dst_width,
 145                        dst_height,
 146                        kFilterMode);
 147     dst += dst_width * dst_height;
 148     const int src_halfwidth = (src->visible_rect().width() + 1) >> 1;
 149     const int src_halfheight = (src->visible_rect().height() + 1) >> 1;
 150     const int dst_halfwidth = (dst_width + 1) >> 1;
 151     const int dst_halfheight = (dst_height + 1) >> 1;
 152     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][1]),
 153                        src->stride(kPlanesOrder[plane_order][1]),
 154                        src_halfwidth,
 155                        src_halfheight,
 156                        dst,
 157                        dst_halfwidth,
 158                        dst_halfwidth,
 159                        dst_halfheight,
 160                        kFilterMode);
 161     dst += dst_halfwidth * dst_halfheight;
 162     libyuv::ScalePlane(src->visible_data(kPlanesOrder[plane_order][2]),
 163                        src->stride(kPlanesOrder[plane_order][2]),
 164                        src_halfwidth,
 165                        src_halfheight,
 166                        dst,
 167                        dst_halfwidth,
 168                        dst_halfwidth,
 169                        dst_halfheight,
 170                        kFilterMode);
 171   } else {
 172     NOTREACHED();
 173   }
 174 }
 175
 176 }  // namespace
 177
 178 namespace content {
 179
 180 // Internal class used for delivering video frames on the IO-thread to
 181 // the MediaStreamVideoSource implementation.
 182 class PepperMediaStreamVideoTrackHost::FrameDeliverer
 183     : public base::RefCountedThreadSafe<FrameDeliverer> {
 184  public:
 185   FrameDeliverer(
 186       const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy,
 187       const VideoCaptureDeliverFrameCB& new_frame_callback);
 188
 189   void DeliverVideoFrame(const scoped_refptr<media::VideoFrame>& frame);
 190
 191  private:
 192   friend class base::RefCountedThreadSafe<FrameDeliverer>;
 193   virtual ~FrameDeliverer();
 194
 195   void DeliverFrameOnIO(const scoped_refptr<media::VideoFrame>& frame);
 196
 197   scoped_refptr<base::MessageLoopProxy> io_message_loop_;
 198   VideoCaptureDeliverFrameCB new_frame_callback_;
 199
 200   DISALLOW_COPY_AND_ASSIGN(FrameDeliverer);
 201 };
 202
 203 PepperMediaStreamVideoTrackHost::FrameDeliverer::FrameDeliverer(
 204     const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy,
 205     const VideoCaptureDeliverFrameCB& new_frame_callback)
 206     : io_message_loop_(io_message_loop_proxy),
 207       new_frame_callback_(new_frame_callback) {
 208 }
 209
 210 PepperMediaStreamVideoTrackHost::FrameDeliverer::~FrameDeliverer() {
 211 }
 212
 213 void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverVideoFrame(
 214     const scoped_refptr<media::VideoFrame>& frame) {
 215   io_message_loop_->PostTask(
 216       FROM_HERE,
 217       base::Bind(&FrameDeliverer::DeliverFrameOnIO, this, frame));
 218 }
 219
 220 void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverFrameOnIO(
 221      const scoped_refptr<media::VideoFrame>& frame) {
 222   DCHECK(io_message_loop_->BelongsToCurrentThread());
 223   // The time when this frame is generated is unknown so give a null value to
 224   // |estimated_capture_time|.
 225   new_frame_callback_.Run(frame, base::TimeTicks());
 226 }
 227
 228 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
 229     RendererPpapiHost* host,
 230     PP_Instance instance,
 231     PP_Resource resource,
 232     const blink::WebMediaStreamTrack& track)
 233     : PepperMediaStreamTrackHostBase(host, instance, resource),
 234       track_(track),
 235       connected_(false),
 236       number_of_buffers_(kDefaultNumberOfBuffers),
 237       source_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 238       plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 239       frame_data_size_(0),
 240       type_(kRead),
 241       output_started_(false),
 242       weak_factory_(this) {
 243   DCHECK(!track_.isNull());
 244 }
 245
 246 PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost(
 247     RendererPpapiHost* host,
 248     PP_Instance instance,
 249     PP_Resource resource)
 250     : PepperMediaStreamTrackHostBase(host, instance, resource),
 251       connected_(false),
 252       number_of_buffers_(kDefaultNumberOfBuffers),
 253       source_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 254       plugin_frame_format_(PP_VIDEOFRAME_FORMAT_UNKNOWN),
 255       frame_data_size_(0),
 256       type_(kWrite),
 257       output_started_(false),
 258       weak_factory_(this) {
 259   InitBlinkTrack();
 260   DCHECK(!track_.isNull());
 261 }
 262
 263 bool PepperMediaStreamVideoTrackHost::IsMediaStreamVideoTrackHost() {
 264   return true;
 265 }
 266
 267 PepperMediaStreamVideoTrackHost::~PepperMediaStreamVideoTrackHost() {
 268   OnClose();
 269 }
 270
 271 void PepperMediaStreamVideoTrackHost::InitBuffers() {
 272   gfx::Size size = GetTargetSize(source_frame_size_, plugin_frame_size_);
 273   DCHECK(!size.IsEmpty());
 274
 275   PP_VideoFrame_Format format =
 276       GetTargetFormat(source_frame_format_, plugin_frame_format_);
 277   DCHECK_NE(format, PP_VIDEOFRAME_FORMAT_UNKNOWN);
 278
 279   if (format == PP_VIDEOFRAME_FORMAT_BGRA) {
 280     frame_data_size_ = size.width() * size.height() * 4;
 281   } else {
 282     frame_data_size_ =
 283         VideoFrame::AllocationSize(FromPpapiFormat(format), size);
 284   }
 285
 286   DCHECK_GT(frame_data_size_, 0U);
 287   int32_t buffer_size =
 288       sizeof(ppapi::MediaStreamBuffer::Video) + frame_data_size_;
 289   bool result = PepperMediaStreamTrackHostBase::InitBuffers(number_of_buffers_,
 290                                                             buffer_size,
 291                                                             type_);
 292   CHECK(result);
 293
 294   if (type_ == kWrite) {
 295     for (int32_t i = 0; i < buffer_manager()->number_of_buffers(); ++i) {
 296       ppapi::MediaStreamBuffer::Video* buffer =
 297           &(buffer_manager()->GetBufferPointer(i)->video);
 298       buffer->header.size = buffer_manager()->buffer_size();
 299       buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
 300       buffer->format = format;
 301       buffer->size.width = size.width();
 302       buffer->size.height = size.height();
 303       buffer->data_size = frame_data_size_;
 304     }
 305
 306     // Make all the frames avaiable to the plugin.
 307     std::vector<int32_t> indices = buffer_manager()->DequeueBuffers();
 308     SendEnqueueBuffersMessageToPlugin(indices);
 309   }
 310 }
 311
 312 void PepperMediaStreamVideoTrackHost::OnClose() {
 313   if (connected_) {
 314     MediaStreamVideoSink::RemoveFromVideoTrack(this, track_);
 315     weak_factory_.InvalidateWeakPtrs();
 316     connected_ = false;
 317   }
 318 }
 319
 320 int32_t PepperMediaStreamVideoTrackHost::OnHostMsgEnqueueBuffer(
 321     ppapi::host::HostMessageContext* context, int32_t index) {
 322   if (type_ == kRead) {
 323     return PepperMediaStreamTrackHostBase::OnHostMsgEnqueueBuffer(context,
 324                                                                   index);
 325   } else {
 326     return SendFrameToTrack(index);
 327   }
 328 }
 329
 330 int32_t PepperMediaStreamVideoTrackHost::SendFrameToTrack(int32_t index) {
 331   DCHECK_EQ(type_, kWrite);
 332
 333   if (output_started_) {
 334     // Sends the frame to blink video track.
 335     ppapi::MediaStreamBuffer::Video* pp_frame =
 336         &(buffer_manager()->GetBufferPointer(index)->video);
 337
 338     int32 y_stride = plugin_frame_size_.width();
 339     int32 uv_stride = (plugin_frame_size_.width() + 1) / 2;
 340     uint8* y_data = static_cast<uint8*>(pp_frame->data);
 341     // Default to I420
 342     uint8* u_data = y_data + plugin_frame_size_.GetArea();
 343     uint8* v_data = y_data + (plugin_frame_size_.GetArea() * 5 / 4);
 344     if (plugin_frame_format_ == PP_VIDEOFRAME_FORMAT_YV12) {
 345       // Swap u and v for YV12.
 346       uint8* tmp = u_data;
 347       u_data = v_data;
 348       v_data = tmp;
 349     }
 350
 351     int64 ts_ms = static_cast<int64>(pp_frame->timestamp *
 352                                      base::Time::kMillisecondsPerSecond);
 353     scoped_refptr<VideoFrame> frame = media::VideoFrame::WrapExternalYuvData(
 354         FromPpapiFormat(plugin_frame_format_),
 355         plugin_frame_size_,
 356         gfx::Rect(plugin_frame_size_),
 357         plugin_frame_size_,
 358         y_stride,
 359         uv_stride,
 360         uv_stride,
 361         y_data,
 362         u_data,
 363         v_data,
 364         base::TimeDelta::FromMilliseconds(ts_ms),
 365         base::Closure());
 366
 367     frame_deliverer_->DeliverVideoFrame(frame);
 368   }
 369
 370   // Makes the frame available again for plugin.
 371   SendEnqueueBufferMessageToPlugin(index);
 372   return PP_OK;
 373 }
 374
 375 void PepperMediaStreamVideoTrackHost::OnVideoFrame(
 376     const scoped_refptr<VideoFrame>& frame,
 377     const base::TimeTicks& estimated_capture_time) {
 378   DCHECK(frame.get());
 379   // TODO(penghuang): Check |frame->end_of_stream()| and close the track.
 380   PP_VideoFrame_Format ppformat = ToPpapiFormat(frame->format());
 381   if (ppformat == PP_VIDEOFRAME_FORMAT_UNKNOWN)
 382     return;
 383
 384   if (source_frame_size_.IsEmpty()) {
 385     source_frame_size_ = frame->visible_rect().size();
 386     source_frame_format_ = ppformat;
 387     InitBuffers();
 388   }
 389
 390   int32_t index = buffer_manager()->DequeueBuffer();
 391   // Drop frames if the underlying buffer is full.
 392   if (index < 0) {
 393     DVLOG(1) << "A frame is dropped.";
 394     return;
 395   }
 396
 397   CHECK_EQ(ppformat, source_frame_format_) << "Frame format is changed.";
 398
 399   gfx::Size size = GetTargetSize(source_frame_size_, plugin_frame_size_);
 400   ppformat =
 401       GetTargetFormat(source_frame_format_, plugin_frame_format_);
 402   ppapi::MediaStreamBuffer::Video* buffer =
 403       &(buffer_manager()->GetBufferPointer(index)->video);
 404   buffer->header.size = buffer_manager()->buffer_size();
 405   buffer->header.type = ppapi::MediaStreamBuffer::TYPE_VIDEO;
 406   buffer->timestamp = frame->timestamp().InSecondsF();
 407   buffer->format = ppformat;
 408   buffer->size.width = size.width();
 409   buffer->size.height = size.height();
 410   buffer->data_size = frame_data_size_;
 411   ConvertFromMediaVideoFrame(frame, ppformat, size, buffer->data);
 412
 413   SendEnqueueBufferMessageToPlugin(index);
 414 }
 415
 416 void PepperMediaStreamVideoTrackHost::GetCurrentSupportedFormats(
 417     int max_requested_width, int max_requested_height,
 418     double max_requested_frame_rate,
 419     const VideoCaptureDeviceFormatsCB& callback) {
 420   if (type_ != kWrite) {
 421     DVLOG(1) << "GetCurrentSupportedFormats is only supported in output mode.";
 422     callback.Run(media::VideoCaptureFormats());
 423     return;
 424   }
 425
 426   media::VideoCaptureFormats formats;
 427   formats.push_back(
 428       media::VideoCaptureFormat(plugin_frame_size_,
 429                                 kDefaultOutputFrameRate,
 430                                 ToPixelFormat(plugin_frame_format_)));
 431   callback.Run(formats);
 432 }
 433
 434 void PepperMediaStreamVideoTrackHost::StartSourceImpl(
 435     const media::VideoCaptureFormat& format,
 436     const VideoCaptureDeliverFrameCB& frame_callback) {
 437   output_started_ = true;
 438   frame_deliverer_ = new FrameDeliverer(io_message_loop(), frame_callback);
 439 }
 440
 441 void PepperMediaStreamVideoTrackHost::StopSourceImpl() {
 442   output_started_ = false;
 443   frame_deliverer_ = NULL;
 444 }
 445
 446 void PepperMediaStreamVideoTrackHost::DidConnectPendingHostToResource() {
 447   if (!connected_) {
 448     MediaStreamVideoSink::AddToVideoTrack(
 449         this,
 450         media::BindToCurrentLoop(
 451             base::Bind(
 452                 &PepperMediaStreamVideoTrackHost::OnVideoFrame,
 453                 weak_factory_.GetWeakPtr())),
 454         track_);
 455     connected_ = true;
 456   }
 457 }
 458
 459 int32_t PepperMediaStreamVideoTrackHost::OnResourceMessageReceived(
 460     const IPC::Message& msg,
 461     HostMessageContext* context) {
 462   PPAPI_BEGIN_MESSAGE_MAP(PepperMediaStreamVideoTrackHost, msg)
 463     PPAPI_DISPATCH_HOST_RESOURCE_CALL(
 464         PpapiHostMsg_MediaStreamVideoTrack_Configure, OnHostMsgConfigure)
 465   PPAPI_END_MESSAGE_MAP()
 466   return PepperMediaStreamTrackHostBase::OnResourceMessageReceived(msg,
 467                                                                    context);
 468 }
 469
 470 int32_t PepperMediaStreamVideoTrackHost::OnHostMsgConfigure(
 471     HostMessageContext* context,
 472     const MediaStreamVideoTrackShared::Attributes& attributes) {
 473   CHECK(MediaStreamVideoTrackShared::VerifyAttributes(attributes));
 474
 475   bool changed = false;
 476   gfx::Size new_size(attributes.width, attributes.height);
 477   if (GetTargetSize(source_frame_size_, plugin_frame_size_) !=
 478       GetTargetSize(source_frame_size_, new_size)) {
 479     changed = true;
 480   }
 481   plugin_frame_size_ = new_size;
 482
 483   int32_t buffers = attributes.buffers
 484                         ? std::min(kMaxNumberOfBuffers, attributes.buffers)
 485                         : kDefaultNumberOfBuffers;
 486   if (buffers != number_of_buffers_)
 487     changed = true;
 488   number_of_buffers_ = buffers;
 489
 490   if (GetTargetFormat(source_frame_format_, plugin_frame_format_) !=
 491       GetTargetFormat(source_frame_format_, attributes.format)) {
 492     changed = true;
 493   }
 494   plugin_frame_format_ = attributes.format;
 495
 496   // If the first frame has been received, we will re-initialize buffers with
 497   // new settings. Otherwise, we will initialize buffer when we receive
 498   // the first frame, because plugin can only provide part of attributes
 499   // which are not enough to initialize buffers.
 500   if (changed && (type_ == kWrite || !source_frame_size_.IsEmpty()))
 501     InitBuffers();
 502
 503   // TODO(ronghuawu): Ask the owner of DOMMediaStreamTrackToResource why
 504   // source id instead of track id is used there.
 505   const std::string id = track_.source().id().utf8();
 506   context->reply_msg = PpapiPluginMsg_MediaStreamVideoTrack_ConfigureReply(id);
 507   return PP_OK;
 508 }
 509
 510 void PepperMediaStreamVideoTrackHost::InitBlinkTrack() {
 511   std::string source_id;
 512   base::Base64Encode(base::RandBytesAsString(64), &source_id);
 513   blink::WebMediaStreamSource webkit_source;
 514   webkit_source.initialize(base::UTF8ToUTF16(source_id),
 515                            blink::WebMediaStreamSource::TypeVideo,
 516                            base::UTF8ToUTF16(kPepperVideoSourceName),
 517                            false /* remote */, true /* readonly */);
 518   webkit_source.setExtraData(this);
 519
 520   const bool enabled = true;
 521   blink::WebMediaConstraints constraints;
 522   constraints.initialize();
 523   track_ = MediaStreamVideoTrack::CreateVideoTrack(
 524        this, constraints,
 525        base::Bind(
 526            &PepperMediaStreamVideoTrackHost::OnTrackStarted,
 527            base::Unretained(this)),
 528        enabled);
 529 }
 530
 531 void PepperMediaStreamVideoTrackHost::OnTrackStarted(
 532     MediaStreamSource* source,
 533     MediaStreamRequestResult result,
 534     const blink::WebString& result_name) {
 535   DVLOG(3) << "OnTrackStarted result: " << result;
 536 }
 537
 538 }  // namespace content