content/renderer/media/webmediaplayer_ms.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webmediaplayer_ms.h"
   6
   7 #include <limits>
   8
   9 #include "base/bind.h"
  10 #include "base/callback.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/metrics/histogram.h"
  13 #include "cc/blink/context_provider_web_context.h"
  14 #include "cc/blink/web_layer_impl.h"
  15 #include "cc/layers/video_layer.h"
  16 #include "content/public/renderer/media_stream_audio_renderer.h"
  17 #include "content/public/renderer/media_stream_renderer_factory.h"
  18 #include "content/public/renderer/render_view.h"
  19 #include "content/public/renderer/video_frame_provider.h"
  20 #include "content/renderer/render_frame_impl.h"
  21 #include "content/renderer/render_thread_impl.h"
  22 #include "gpu/blink/webgraphicscontext3d_impl.h"
  23 #include "media/base/media_log.h"
  24 #include "media/base/video_frame.h"
  25 #include "media/base/video_rotation.h"
  26 #include "media/base/video_util.h"
  27 #include "media/blink/webmediaplayer_delegate.h"
  28 #include "media/blink/webmediaplayer_util.h"
  29 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
  30 #include "third_party/WebKit/public/platform/WebRect.h"
  31 #include "third_party/WebKit/public/platform/WebSize.h"
  32 #include "third_party/WebKit/public/platform/WebURL.h"
  33 #include "third_party/WebKit/public/web/WebFrame.h"
  34 #include "third_party/WebKit/public/web/WebView.h"
  35 #include "third_party/skia/include/core/SkBitmap.h"
  36
  37 using blink::WebCanvas;
  38 using blink::WebMediaPlayer;
  39 using blink::WebRect;
  40 using blink::WebSize;
  41
  42 namespace content {
  43
  44 namespace {
  45
  46 // This function copies |frame| to a new YV12 media::VideoFrame.
  47 scoped_refptr<media::VideoFrame> CopyFrameToYV12(
  48     const scoped_refptr<media::VideoFrame>& frame,
  49     media::SkCanvasVideoRenderer* video_renderer) {
  50   scoped_refptr<media::VideoFrame> new_frame = media::VideoFrame::CreateFrame(
  51       media::PIXEL_FORMAT_YV12, frame->coded_size(), frame->visible_rect(),
  52       frame->natural_size(), frame->timestamp());
  53
  54   if (frame->HasTextures()) {
  55     DCHECK(frame->format() == media::PIXEL_FORMAT_ARGB ||
  56            frame->format() == media::PIXEL_FORMAT_XRGB);
  57     SkBitmap bitmap;
  58     bitmap.allocN32Pixels(frame->visible_rect().width(),
  59                           frame->visible_rect().height());
  60     SkCanvas canvas(bitmap);
  61
  62     cc::ContextProvider* const provider =
  63         RenderThreadImpl::current()->SharedMainThreadContextProvider().get();
  64     if (provider) {
  65       const media::Context3D context_3d =
  66           media::Context3D(provider->ContextGL(), provider->GrContext());
  67       DCHECK(context_3d.gl);
  68       video_renderer->Copy(frame.get(), &canvas, context_3d);
  69     } else {
  70       // GPU Process crashed.
  71       bitmap.eraseColor(SK_ColorTRANSPARENT);
  72     }
  73     media::CopyRGBToVideoFrame(reinterpret_cast<uint8*>(bitmap.getPixels()),
  74                                bitmap.rowBytes(),
  75                                frame->visible_rect(),
  76                                new_frame.get());
  77   } else {
  78     DCHECK(frame->IsMappable());
  79     DCHECK(frame->format() == media::PIXEL_FORMAT_YV12 ||
  80            frame->format() == media::PIXEL_FORMAT_I420);
  81     for (size_t i = 0; i < media::VideoFrame::NumPlanes(frame->format()); ++i) {
  82       media::CopyPlane(i, frame->data(i), frame->stride(i),
  83                        frame->rows(i), new_frame.get());
  84     }
  85   }
  86   return new_frame;
  87 }
  88
  89 }  // anonymous namespace
  90
  91 WebMediaPlayerMS::WebMediaPlayerMS(
  92     blink::WebFrame* frame,
  93     blink::WebMediaPlayerClient* client,
  94     base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
  95     media::MediaLog* media_log,
  96     scoped_ptr<MediaStreamRendererFactory> factory)
  97     : frame_(frame),
  98       network_state_(WebMediaPlayer::NetworkStateEmpty),
  99       ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
 100       buffered_(static_cast<size_t>(0)),
 101       volume_(1.0f),
 102       client_(client),
 103       delegate_(delegate),
 104       paused_(true),
 105       current_frame_used_(false),
 106       video_frame_provider_client_(NULL),
 107       received_first_frame_(false),
 108       total_frame_count_(0),
 109       dropped_frame_count_(0),
 110       media_log_(media_log),
 111       renderer_factory_(factory.Pass()) {
 112   DVLOG(1) << "WebMediaPlayerMS::ctor";
 113   media_log_->AddEvent(
 114       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
 115 }
 116
 117 WebMediaPlayerMS::~WebMediaPlayerMS() {
 118   DVLOG(1) << "WebMediaPlayerMS::dtor";
 119   DCHECK(thread_checker_.CalledOnValidThread());
 120
 121   SetVideoFrameProviderClient(NULL);
 122   GetClient()->setWebLayer(NULL);
 123
 124   if (video_frame_provider_.get())
 125     video_frame_provider_->Stop();
 126
 127   if (audio_renderer_.get())
 128     audio_renderer_->Stop();
 129
 130   media_log_->AddEvent(
 131       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
 132
 133   if (delegate_.get())
 134     delegate_->PlayerGone(this);
 135 }
 136
 137 void WebMediaPlayerMS::load(LoadType load_type,
 138                             const blink::WebURL& url,
 139                             CORSMode cors_mode) {
 140   DVLOG(1) << "WebMediaPlayerMS::load";
 141   DCHECK(thread_checker_.CalledOnValidThread());
 142
 143   // TODO(acolwell): Change this to DCHECK_EQ(load_type,
 144   // LoadTypeMediaStream) once Blink-side changes land.
 145   DCHECK_NE(load_type, LoadTypeMediaSource);
 146
 147   GURL gurl(url);
 148
 149   SetNetworkState(WebMediaPlayer::NetworkStateLoading);
 150   SetReadyState(WebMediaPlayer::ReadyStateHaveNothing);
 151   media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec()));
 152
 153   video_frame_provider_ = renderer_factory_->GetVideoFrameProvider(
 154       url,
 155       base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()),
 156       base::Bind(&WebMediaPlayerMS::OnFrameAvailable, AsWeakPtr()));
 157
 158   RenderFrame* frame = RenderFrame::FromWebFrame(frame_);
 159   audio_renderer_ = renderer_factory_->GetAudioRenderer(
 160     url,
 161     frame->GetRoutingID());
 162
 163   if (video_frame_provider_.get() || audio_renderer_.get()) {
 164     if (audio_renderer_.get()) {
 165       audio_renderer_->SetVolume(volume_);
 166       audio_renderer_->Start();
 167     }
 168
 169     if (video_frame_provider_.get()) {
 170       video_frame_provider_->Start();
 171     } else {
 172       // This is audio-only mode.
 173       DCHECK(audio_renderer_.get());
 174       SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
 175       SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
 176     }
 177   } else {
 178     SetNetworkState(WebMediaPlayer::NetworkStateNetworkError);
 179   }
 180 }
 181
 182 void WebMediaPlayerMS::play() {
 183   DVLOG(1) << "WebMediaPlayerMS::play";
 184   DCHECK(thread_checker_.CalledOnValidThread());
 185
 186   if (paused_) {
 187     if (video_frame_provider_.get())
 188       video_frame_provider_->Play();
 189
 190     if (audio_renderer_.get())
 191       audio_renderer_->Play();
 192
 193     if (delegate_.get())
 194       delegate_->DidPlay(this);
 195   }
 196
 197   paused_ = false;
 198
 199   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
 200 }
 201
 202 void WebMediaPlayerMS::pause() {
 203   DVLOG(1) << "WebMediaPlayerMS::pause";
 204   DCHECK(thread_checker_.CalledOnValidThread());
 205
 206   if (video_frame_provider_.get())
 207     video_frame_provider_->Pause();
 208
 209   if (!paused_) {
 210     if (audio_renderer_.get())
 211       audio_renderer_->Pause();
 212
 213     if (delegate_.get())
 214       delegate_->DidPause(this);
 215   }
 216
 217   paused_ = true;
 218
 219   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
 220
 221   if (!current_frame_.get())
 222     return;
 223
 224   // Copy the frame so that rendering can show the last received frame.
 225   // The original frame must not be referenced when the player is paused since
 226   // there might be a finite number of available buffers. E.g, video that
 227   // originates from a video camera.
 228   scoped_refptr<media::VideoFrame> new_frame =
 229       CopyFrameToYV12(current_frame_, &video_renderer_);
 230
 231   base::AutoLock auto_lock(current_frame_lock_);
 232   current_frame_ = new_frame;
 233 }
 234
 235 bool WebMediaPlayerMS::supportsSave() const {
 236   DCHECK(thread_checker_.CalledOnValidThread());
 237   return false;
 238 }
 239
 240 void WebMediaPlayerMS::seek(double seconds) {
 241   DCHECK(thread_checker_.CalledOnValidThread());
 242 }
 243
 244 void WebMediaPlayerMS::setRate(double rate) {
 245   DCHECK(thread_checker_.CalledOnValidThread());
 246 }
 247
 248 void WebMediaPlayerMS::setVolume(double volume) {
 249   DCHECK(thread_checker_.CalledOnValidThread());
 250   DVLOG(1) << "WebMediaPlayerMS::setVolume(volume=" << volume << ")";
 251   volume_ = volume;
 252   if (audio_renderer_.get())
 253     audio_renderer_->SetVolume(volume_);
 254 }
 255
 256 void WebMediaPlayerMS::setSinkId(const blink::WebString& device_id,
 257                                  media::WebSetSinkIdCB* web_callback) {
 258   DCHECK(thread_checker_.CalledOnValidThread());
 259   DVLOG(1) << __FUNCTION__;
 260   media::SwitchOutputDeviceCB callback =
 261       media::ConvertToSwitchOutputDeviceCB(web_callback);
 262   if (audio_renderer_.get()) {
 263     media::OutputDevice* output_device = audio_renderer_->GetOutputDevice();
 264     if (output_device) {
 265       std::string device_id_str(device_id.utf8());
 266       GURL security_origin(frame_->securityOrigin().toString().utf8());
 267       output_device->SwitchOutputDevice(device_id_str, security_origin,
 268                                         callback);
 269       return;
 270     }
 271   }
 272   callback.Run(media::SWITCH_OUTPUT_DEVICE_RESULT_ERROR_NOT_SUPPORTED);
 273 }
 274
 275 void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) {
 276   DCHECK(thread_checker_.CalledOnValidThread());
 277 }
 278
 279 bool WebMediaPlayerMS::hasVideo() const {
 280   DCHECK(thread_checker_.CalledOnValidThread());
 281   return (video_frame_provider_.get() != NULL);
 282 }
 283
 284 bool WebMediaPlayerMS::hasAudio() const {
 285   DCHECK(thread_checker_.CalledOnValidThread());
 286   return (audio_renderer_.get() != NULL);
 287 }
 288
 289 blink::WebSize WebMediaPlayerMS::naturalSize() const {
 290   DCHECK(thread_checker_.CalledOnValidThread());
 291
 292   gfx::Size size;
 293   if (current_frame_.get())
 294     size = current_frame_->natural_size();
 295   DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
 296   return blink::WebSize(size);
 297 }
 298
 299 bool WebMediaPlayerMS::paused() const {
 300   DCHECK(thread_checker_.CalledOnValidThread());
 301   return paused_;
 302 }
 303
 304 bool WebMediaPlayerMS::seeking() const {
 305   DCHECK(thread_checker_.CalledOnValidThread());
 306   return false;
 307 }
 308
 309 double WebMediaPlayerMS::duration() const {
 310   DCHECK(thread_checker_.CalledOnValidThread());
 311   return std::numeric_limits<double>::infinity();
 312 }
 313
 314 double WebMediaPlayerMS::currentTime() const {
 315   DCHECK(thread_checker_.CalledOnValidThread());
 316   if (current_time_.ToInternalValue() != 0) {
 317     return current_time_.InSecondsF();
 318   } else if (audio_renderer_.get()) {
 319     return audio_renderer_->GetCurrentRenderTime().InSecondsF();
 320   }
 321   return 0.0;
 322 }
 323
 324 WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
 325   DCHECK(thread_checker_.CalledOnValidThread());
 326   DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
 327   return network_state_;
 328 }
 329
 330 WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const {
 331   DCHECK(thread_checker_.CalledOnValidThread());
 332   DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_;
 333   return ready_state_;
 334 }
 335
 336 blink::WebTimeRanges WebMediaPlayerMS::buffered() const {
 337   DCHECK(thread_checker_.CalledOnValidThread());
 338   return buffered_;
 339 }
 340
 341 blink::WebTimeRanges WebMediaPlayerMS::seekable() const {
 342   DCHECK(thread_checker_.CalledOnValidThread());
 343   return blink::WebTimeRanges();
 344 }
 345
 346 bool WebMediaPlayerMS::didLoadingProgress() {
 347   DCHECK(thread_checker_.CalledOnValidThread());
 348   return true;
 349 }
 350
 351 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
 352                              const blink::WebRect& rect,
 353                              unsigned char alpha,
 354                              SkXfermode::Mode mode) {
 355   DVLOG(3) << "WebMediaPlayerMS::paint";
 356   DCHECK(thread_checker_.CalledOnValidThread());
 357
 358   media::Context3D context_3d;
 359   if (current_frame_.get() && current_frame_->HasTextures()) {
 360     cc::ContextProvider* provider =
 361         RenderThreadImpl::current()->SharedMainThreadContextProvider().get();
 362     // GPU Process crashed.
 363     if (!provider)
 364       return;
 365     context_3d = media::Context3D(provider->ContextGL(), provider->GrContext());
 366     DCHECK(context_3d.gl);
 367   }
 368   gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height);
 369   video_renderer_.Paint(current_frame_, canvas, dest_rect, alpha, mode,
 370                         media::VIDEO_ROTATION_0, context_3d);
 371
 372   {
 373     base::AutoLock auto_lock(current_frame_lock_);
 374     if (current_frame_.get())
 375       current_frame_used_ = true;
 376   }
 377 }
 378
 379 bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
 380   DCHECK(thread_checker_.CalledOnValidThread());
 381   return true;
 382 }
 383
 384 bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
 385   DCHECK(thread_checker_.CalledOnValidThread());
 386   return true;
 387 }
 388
 389 double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const {
 390   return media::ConvertSecondsToTimestamp(timeValue).InSecondsF();
 391 }
 392
 393 unsigned WebMediaPlayerMS::decodedFrameCount() const {
 394   DCHECK(thread_checker_.CalledOnValidThread());
 395   DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
 396   return total_frame_count_;
 397 }
 398
 399 unsigned WebMediaPlayerMS::droppedFrameCount() const {
 400   DCHECK(thread_checker_.CalledOnValidThread());
 401   DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
 402   return dropped_frame_count_;
 403 }
 404
 405 unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
 406   DCHECK(thread_checker_.CalledOnValidThread());
 407   NOTIMPLEMENTED();
 408   return 0;
 409 }
 410
 411 unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
 412   DCHECK(thread_checker_.CalledOnValidThread());
 413   NOTIMPLEMENTED();
 414   return 0;
 415 }
 416
 417 bool WebMediaPlayerMS::copyVideoTextureToPlatformTexture(
 418     blink::WebGraphicsContext3D* web_graphics_context,
 419     unsigned int texture,
 420     unsigned int internal_format,
 421     unsigned int type,
 422     bool premultiply_alpha,
 423     bool flip_y) {
 424   TRACE_EVENT0("media", "WebMediaPlayerMS:copyVideoTextureToPlatformTexture");
 425   DCHECK(thread_checker_.CalledOnValidThread());
 426
 427   scoped_refptr<media::VideoFrame> video_frame;
 428   {
 429     base::AutoLock auto_lock(current_frame_lock_);
 430     video_frame = current_frame_;
 431   }
 432
 433   if (!video_frame.get() || video_frame->HasTextures() ||
 434       media::VideoFrame::NumPlanes(video_frame->format()) != 1) {
 435     return false;
 436   }
 437
 438   // TODO(dshwang): need more elegant way to convert WebGraphicsContext3D to
 439   // GLES2Interface.
 440   gpu::gles2::GLES2Interface* gl =
 441       static_cast<gpu_blink::WebGraphicsContext3DImpl*>(web_graphics_context)
 442           ->GetGLInterface();
 443   media::SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
 444       gl, video_frame.get(), texture, internal_format, type, premultiply_alpha,
 445       flip_y);
 446   return true;
 447 }
 448
 449 void WebMediaPlayerMS::SetVideoFrameProviderClient(
 450     cc::VideoFrameProvider::Client* client) {
 451   // This is called from both the main renderer thread and the compositor
 452   // thread (when the main thread is blocked).
 453   if (video_frame_provider_client_)
 454     video_frame_provider_client_->StopUsingProvider();
 455   video_frame_provider_client_ = client;
 456 }
 457
 458 bool WebMediaPlayerMS::UpdateCurrentFrame(base::TimeTicks deadline_min,
 459                                           base::TimeTicks deadline_max) {
 460   // TODO(dalecurtis): This should make use of the deadline interval to ensure
 461   // the painted frame is correct for the given interval.
 462   NOTREACHED();
 463   return false;
 464 }
 465
 466 bool WebMediaPlayerMS::HasCurrentFrame() {
 467   base::AutoLock auto_lock(current_frame_lock_);
 468   return current_frame_;
 469 }
 470
 471 scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
 472   DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
 473   base::AutoLock auto_lock(current_frame_lock_);
 474   if (!current_frame_.get())
 475     return NULL;
 476   current_frame_used_ = true;
 477   return current_frame_;
 478 }
 479
 480 void WebMediaPlayerMS::PutCurrentFrame() {
 481   DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
 482 }
 483
 484 void WebMediaPlayerMS::OnFrameAvailable(
 485     const scoped_refptr<media::VideoFrame>& frame) {
 486   DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
 487   DCHECK(thread_checker_.CalledOnValidThread());
 488   ++total_frame_count_;
 489   if (!received_first_frame_) {
 490     received_first_frame_ = true;
 491     {
 492       base::AutoLock auto_lock(current_frame_lock_);
 493       DCHECK(!current_frame_used_);
 494       current_frame_ = frame;
 495     }
 496     SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
 497     SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
 498     GetClient()->sizeChanged();
 499
 500     if (video_frame_provider_.get()) {
 501       video_weblayer_.reset(new cc_blink::WebLayerImpl(
 502           cc::VideoLayer::Create(cc_blink::WebLayerImpl::LayerSettings(), this,
 503                                  media::VIDEO_ROTATION_0)));
 504       video_weblayer_->setOpaque(true);
 505       GetClient()->setWebLayer(video_weblayer_.get());
 506     }
 507   }
 508
 509   // Do not update |current_frame_| when paused.
 510   if (paused_)
 511     return;
 512
 513   bool size_changed = !current_frame_.get() ||
 514                       current_frame_->natural_size() != frame->natural_size();
 515
 516   {
 517     base::AutoLock auto_lock(current_frame_lock_);
 518     if (!current_frame_used_ && current_frame_.get())
 519       ++dropped_frame_count_;
 520     current_frame_ = frame;
 521     current_time_ = frame->timestamp();
 522     current_frame_used_ = false;
 523   }
 524
 525   if (size_changed)
 526     GetClient()->sizeChanged();
 527
 528   GetClient()->repaint();
 529 }
 530
 531 void WebMediaPlayerMS::RepaintInternal() {
 532   DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
 533   DCHECK(thread_checker_.CalledOnValidThread());
 534   GetClient()->repaint();
 535 }
 536
 537 void WebMediaPlayerMS::OnSourceError() {
 538   DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
 539   DCHECK(thread_checker_.CalledOnValidThread());
 540   SetNetworkState(WebMediaPlayer::NetworkStateFormatError);
 541   RepaintInternal();
 542 }
 543
 544 void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) {
 545   DCHECK(thread_checker_.CalledOnValidThread());
 546   network_state_ = state;
 547   // Always notify to ensure client has the latest value.
 548   GetClient()->networkStateChanged();
 549 }
 550
 551 void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) {
 552   DCHECK(thread_checker_.CalledOnValidThread());
 553   ready_state_ = state;
 554   // Always notify to ensure client has the latest value.
 555   GetClient()->readyStateChanged();
 556 }
 557
 558 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
 559   DCHECK(thread_checker_.CalledOnValidThread());
 560   DCHECK(client_);
 561   return client_;
 562 }
 563
 564 }  // namespace content