content/renderer/media/webmediaplayer_ms.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webmediaplayer_ms.h"
   6
   7 #include <limits>
   8
   9 #include "base/bind.h"
  10 #include "base/callback.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/metrics/histogram.h"
  13 #include "cc/blink/context_provider_web_context.h"
  14 #include "cc/blink/web_layer_impl.h"
  15 #include "cc/layers/video_layer.h"
  16 #include "content/public/renderer/media_stream_audio_renderer.h"
  17 #include "content/public/renderer/media_stream_renderer_factory.h"
  18 #include "content/public/renderer/render_view.h"
  19 #include "content/public/renderer/video_frame_provider.h"
  20 #include "content/renderer/render_frame_impl.h"
  21 #include "content/renderer/render_thread_impl.h"
  22 #include "gpu/blink/webgraphicscontext3d_impl.h"
  23 #include "media/base/media_log.h"
  24 #include "media/base/video_frame.h"
  25 #include "media/base/video_rotation.h"
  26 #include "media/base/video_util.h"
  27 #include "media/blink/webmediaplayer_delegate.h"
  28 #include "media/blink/webmediaplayer_util.h"
  29 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
  30 #include "third_party/WebKit/public/platform/WebRect.h"
  31 #include "third_party/WebKit/public/platform/WebSize.h"
  32 #include "third_party/WebKit/public/platform/WebURL.h"
  33 #include "third_party/WebKit/public/web/WebFrame.h"
  34 #include "third_party/WebKit/public/web/WebView.h"
  35 #include "third_party/skia/include/core/SkBitmap.h"
  36
  37 using blink::WebCanvas;
  38 using blink::WebMediaPlayer;
  39 using blink::WebRect;
  40 using blink::WebSize;
  41
  42 namespace content {
  43
  44 namespace {
  45
  46 // This function copies |frame| to a new YV12 media::VideoFrame.
  47 scoped_refptr<media::VideoFrame> CopyFrameToYV12(
  48     const scoped_refptr<media::VideoFrame>& frame,
  49     media::SkCanvasVideoRenderer* video_renderer) {
  50   scoped_refptr<media::VideoFrame> new_frame = media::VideoFrame::CreateFrame(
  51       media::PIXEL_FORMAT_YV12, frame->coded_size(), frame->visible_rect(),
  52       frame->natural_size(), frame->timestamp());
  53
  54   if (frame->HasTextures()) {
  55     DCHECK(frame->format() == media::PIXEL_FORMAT_ARGB ||
  56            frame->format() == media::PIXEL_FORMAT_XRGB);
  57     SkBitmap bitmap;
  58     bitmap.allocN32Pixels(frame->visible_rect().width(),
  59                           frame->visible_rect().height());
  60     SkCanvas canvas(bitmap);
  61
  62     cc::ContextProvider* const provider =
  63         RenderThreadImpl::current()->SharedMainThreadContextProvider().get();
  64     if (provider) {
  65       const media::Context3D context_3d =
  66           media::Context3D(provider->ContextGL(), provider->GrContext());
  67       DCHECK(context_3d.gl);
  68       video_renderer->Copy(frame.get(), &canvas, context_3d);
  69     } else {
  70       // GPU Process crashed.
  71       bitmap.eraseColor(SK_ColorTRANSPARENT);
  72     }
  73     media::CopyRGBToVideoFrame(reinterpret_cast<uint8*>(bitmap.getPixels()),
  74                                bitmap.rowBytes(),
  75                                frame->visible_rect(),
  76                                new_frame.get());
  77   } else {
  78     DCHECK(frame->IsMappable());
  79     DCHECK(frame->format() == media::PIXEL_FORMAT_YV12 ||
  80            frame->format() == media::PIXEL_FORMAT_I420);
  81     for (size_t i = 0; i < media::VideoFrame::NumPlanes(frame->format()); ++i) {
  82       media::CopyPlane(i, frame->data(i), frame->stride(i),
  83                        frame->rows(i), new_frame.get());
  84     }
  85   }
  86   return new_frame;
  87 }
  88
  89 }  // anonymous namespace
  90
  91 WebMediaPlayerMS::WebMediaPlayerMS(
  92     blink::WebFrame* frame,
  93     blink::WebMediaPlayerClient* client,
  94     base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
  95     media::MediaLog* media_log,
  96     scoped_ptr<MediaStreamRendererFactory> factory)
  97     : frame_(frame),
  98       network_state_(WebMediaPlayer::NetworkStateEmpty),
  99       ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
 100       buffered_(static_cast<size_t>(0)),
 101       volume_(1.0f),
 102       client_(client),
 103       delegate_(delegate),
 104       paused_(true),
 105       current_frame_used_(false),
 106       video_frame_provider_client_(NULL),
 107       received_first_frame_(false),
 108       total_frame_count_(0),
 109       dropped_frame_count_(0),
 110       media_log_(media_log),
 111       renderer_factory_(factory.Pass()) {
 112   DVLOG(1) << "WebMediaPlayerMS::ctor";
 113   media_log_->AddEvent(
 114       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
 115 }
 116
 117 WebMediaPlayerMS::~WebMediaPlayerMS() {
 118   DVLOG(1) << "WebMediaPlayerMS::dtor";
 119   DCHECK(thread_checker_.CalledOnValidThread());
 120
 121   SetVideoFrameProviderClient(NULL);
 122   GetClient()->setWebLayer(NULL);
 123
 124   if (video_frame_provider_.get())
 125     video_frame_provider_->Stop();
 126
 127   if (audio_renderer_.get())
 128     audio_renderer_->Stop();
 129
 130   media_log_->AddEvent(
 131       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
 132
 133   if (delegate_.get())
 134     delegate_->PlayerGone(this);
 135 }
 136
 137 void WebMediaPlayerMS::load(LoadType load_type,
 138                             const blink::WebURL& url,
 139                             CORSMode cors_mode) {
 140   DVLOG(1) << "WebMediaPlayerMS::load";
 141   DCHECK(thread_checker_.CalledOnValidThread());
 142
 143   // TODO(acolwell): Change this to DCHECK_EQ(load_type,
 144   // LoadTypeMediaStream) once Blink-side changes land.
 145   DCHECK_NE(load_type, LoadTypeMediaSource);
 146
 147   GURL gurl(url);
 148
 149   SetNetworkState(WebMediaPlayer::NetworkStateLoading);
 150   SetReadyState(WebMediaPlayer::ReadyStateHaveNothing);
 151   media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec()));
 152
 153   video_frame_provider_ = renderer_factory_->GetVideoFrameProvider(
 154       url,
 155       base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()),
 156       base::Bind(&WebMediaPlayerMS::OnFrameAvailable, AsWeakPtr()));
 157
 158   RenderFrame* frame = RenderFrame::FromWebFrame(frame_);
 159   audio_renderer_ = renderer_factory_->GetAudioRenderer(
 160     url,
 161     frame->GetRoutingID());
 162
 163   if (video_frame_provider_.get() || audio_renderer_.get()) {
 164     if (audio_renderer_.get()) {
 165       audio_renderer_->SetVolume(volume_);
 166       audio_renderer_->Start();
 167     }
 168
 169     if (video_frame_provider_.get()) {
 170       video_frame_provider_->Start();
 171     } else {
 172       // This is audio-only mode.
 173       DCHECK(audio_renderer_.get());
 174       SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
 175       SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
 176     }
 177   } else {
 178     SetNetworkState(WebMediaPlayer::NetworkStateNetworkError);
 179   }
 180 }
 181
 182 void WebMediaPlayerMS::play() {
 183   DVLOG(1) << "WebMediaPlayerMS::play";
 184   DCHECK(thread_checker_.CalledOnValidThread());
 185
 186   if (paused_) {
 187     if (video_frame_provider_.get())
 188       video_frame_provider_->Play();
 189
 190     if (audio_renderer_.get())
 191       audio_renderer_->Play();
 192
 193     if (delegate_.get())
 194       delegate_->DidPlay(this);
 195   }
 196
 197   paused_ = false;
 198
 199   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
 200 }
 201
 202 void WebMediaPlayerMS::pause() {
 203   DVLOG(1) << "WebMediaPlayerMS::pause";
 204   DCHECK(thread_checker_.CalledOnValidThread());
 205
 206   if (video_frame_provider_.get())
 207     video_frame_provider_->Pause();
 208
 209   if (!paused_) {
 210     if (audio_renderer_.get())
 211       audio_renderer_->Pause();
 212
 213     if (delegate_.get())
 214       delegate_->DidPause(this);
 215   }
 216
 217   paused_ = true;
 218
 219   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
 220
 221   if (!current_frame_.get())
 222     return;
 223
 224   // Copy the frame so that rendering can show the last received frame.
 225   // The original frame must not be referenced when the player is paused since
 226   // there might be a finite number of available buffers. E.g, video that
 227   // originates from a video camera.
 228   scoped_refptr<media::VideoFrame> new_frame =
 229       CopyFrameToYV12(current_frame_, &video_renderer_);
 230
 231   base::AutoLock auto_lock(current_frame_lock_);
 232   current_frame_ = new_frame;
 233 }
 234
 235 bool WebMediaPlayerMS::supportsSave() const {
 236   DCHECK(thread_checker_.CalledOnValidThread());
 237   return false;
 238 }
 239
 240 void WebMediaPlayerMS::seek(double seconds) {
 241   DCHECK(thread_checker_.CalledOnValidThread());
 242 }
 243
 244 void WebMediaPlayerMS::setRate(double rate) {
 245   DCHECK(thread_checker_.CalledOnValidThread());
 246 }
 247
 248 void WebMediaPlayerMS::setVolume(double volume) {
 249   DCHECK(thread_checker_.CalledOnValidThread());
 250   DVLOG(1) << "WebMediaPlayerMS::setVolume(volume=" << volume << ")";
 251   volume_ = volume;
 252   if (audio_renderer_.get())
 253     audio_renderer_->SetVolume(volume_);
 254 }
 255
 256 void WebMediaPlayerMS::setSinkId(const blink::WebString& device_id,
 257                                  media::WebSetSinkIdCB* web_callback) {
 258   DCHECK(thread_checker_.CalledOnValidThread());
 259   std::string device_id_str(device_id.utf8());
 260   GURL security_origin(frame_->securityOrigin().toString().utf8());
 261   DVLOG(1) << __FUNCTION__
 262            << "(" << device_id_str << ", " << security_origin << ")";
 263   media::SwitchOutputDeviceCB callback =
 264       media::ConvertToSwitchOutputDeviceCB(web_callback);
 265   if (audio_renderer_.get()) {
 266     audio_renderer_->SwitchOutputDevice(device_id_str, security_origin,
 267                                         callback);
 268   } else {
 269     callback.Run(media::SWITCH_OUTPUT_DEVICE_RESULT_ERROR_NOT_SUPPORTED);
 270   }
 271 }
 272
 273 void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) {
 274   DCHECK(thread_checker_.CalledOnValidThread());
 275 }
 276
 277 bool WebMediaPlayerMS::hasVideo() const {
 278   DCHECK(thread_checker_.CalledOnValidThread());
 279   return (video_frame_provider_.get() != NULL);
 280 }
 281
 282 bool WebMediaPlayerMS::hasAudio() const {
 283   DCHECK(thread_checker_.CalledOnValidThread());
 284   return (audio_renderer_.get() != NULL);
 285 }
 286
 287 blink::WebSize WebMediaPlayerMS::naturalSize() const {
 288   DCHECK(thread_checker_.CalledOnValidThread());
 289
 290   gfx::Size size;
 291   if (current_frame_.get())
 292     size = current_frame_->natural_size();
 293   DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
 294   return blink::WebSize(size);
 295 }
 296
 297 bool WebMediaPlayerMS::paused() const {
 298   DCHECK(thread_checker_.CalledOnValidThread());
 299   return paused_;
 300 }
 301
 302 bool WebMediaPlayerMS::seeking() const {
 303   DCHECK(thread_checker_.CalledOnValidThread());
 304   return false;
 305 }
 306
 307 double WebMediaPlayerMS::duration() const {
 308   DCHECK(thread_checker_.CalledOnValidThread());
 309   return std::numeric_limits<double>::infinity();
 310 }
 311
 312 double WebMediaPlayerMS::currentTime() const {
 313   DCHECK(thread_checker_.CalledOnValidThread());
 314   if (current_time_.ToInternalValue() != 0) {
 315     return current_time_.InSecondsF();
 316   } else if (audio_renderer_.get()) {
 317     return audio_renderer_->GetCurrentRenderTime().InSecondsF();
 318   }
 319   return 0.0;
 320 }
 321
 322 WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
 323   DCHECK(thread_checker_.CalledOnValidThread());
 324   DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
 325   return network_state_;
 326 }
 327
 328 WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const {
 329   DCHECK(thread_checker_.CalledOnValidThread());
 330   DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_;
 331   return ready_state_;
 332 }
 333
 334 blink::WebTimeRanges WebMediaPlayerMS::buffered() const {
 335   DCHECK(thread_checker_.CalledOnValidThread());
 336   return buffered_;
 337 }
 338
 339 blink::WebTimeRanges WebMediaPlayerMS::seekable() const {
 340   DCHECK(thread_checker_.CalledOnValidThread());
 341   return blink::WebTimeRanges();
 342 }
 343
 344 bool WebMediaPlayerMS::didLoadingProgress() {
 345   DCHECK(thread_checker_.CalledOnValidThread());
 346   return true;
 347 }
 348
 349 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
 350                              const blink::WebRect& rect,
 351                              unsigned char alpha,
 352                              SkXfermode::Mode mode) {
 353   DVLOG(3) << "WebMediaPlayerMS::paint";
 354   DCHECK(thread_checker_.CalledOnValidThread());
 355
 356   media::Context3D context_3d;
 357   if (current_frame_.get() && current_frame_->HasTextures()) {
 358     cc::ContextProvider* provider =
 359         RenderThreadImpl::current()->SharedMainThreadContextProvider().get();
 360     // GPU Process crashed.
 361     if (!provider)
 362       return;
 363     context_3d = media::Context3D(provider->ContextGL(), provider->GrContext());
 364     DCHECK(context_3d.gl);
 365   }
 366   gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height);
 367   video_renderer_.Paint(current_frame_, canvas, dest_rect, alpha, mode,
 368                         media::VIDEO_ROTATION_0, context_3d);
 369
 370   {
 371     base::AutoLock auto_lock(current_frame_lock_);
 372     if (current_frame_.get())
 373       current_frame_used_ = true;
 374   }
 375 }
 376
 377 bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
 378   DCHECK(thread_checker_.CalledOnValidThread());
 379   return true;
 380 }
 381
 382 bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
 383   DCHECK(thread_checker_.CalledOnValidThread());
 384   return true;
 385 }
 386
 387 double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const {
 388   return media::ConvertSecondsToTimestamp(timeValue).InSecondsF();
 389 }
 390
 391 unsigned WebMediaPlayerMS::decodedFrameCount() const {
 392   DCHECK(thread_checker_.CalledOnValidThread());
 393   DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
 394   return total_frame_count_;
 395 }
 396
 397 unsigned WebMediaPlayerMS::droppedFrameCount() const {
 398   DCHECK(thread_checker_.CalledOnValidThread());
 399   DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
 400   return dropped_frame_count_;
 401 }
 402
 403 unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
 404   DCHECK(thread_checker_.CalledOnValidThread());
 405   NOTIMPLEMENTED();
 406   return 0;
 407 }
 408
 409 unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
 410   DCHECK(thread_checker_.CalledOnValidThread());
 411   NOTIMPLEMENTED();
 412   return 0;
 413 }
 414
 415 bool WebMediaPlayerMS::copyVideoTextureToPlatformTexture(
 416     blink::WebGraphicsContext3D* web_graphics_context,
 417     unsigned int texture,
 418     unsigned int internal_format,
 419     unsigned int type,
 420     bool premultiply_alpha,
 421     bool flip_y) {
 422   TRACE_EVENT0("media", "WebMediaPlayerMS:copyVideoTextureToPlatformTexture");
 423   DCHECK(thread_checker_.CalledOnValidThread());
 424
 425   scoped_refptr<media::VideoFrame> video_frame;
 426   {
 427     base::AutoLock auto_lock(current_frame_lock_);
 428     video_frame = current_frame_;
 429   }
 430
 431   if (!video_frame.get() || video_frame->HasTextures() ||
 432       media::VideoFrame::NumPlanes(video_frame->format()) != 1) {
 433     return false;
 434   }
 435
 436   // TODO(dshwang): need more elegant way to convert WebGraphicsContext3D to
 437   // GLES2Interface.
 438   gpu::gles2::GLES2Interface* gl =
 439       static_cast<gpu_blink::WebGraphicsContext3DImpl*>(web_graphics_context)
 440           ->GetGLInterface();
 441   media::SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
 442       gl, video_frame.get(), texture, internal_format, type, premultiply_alpha,
 443       flip_y);
 444   return true;
 445 }
 446
 447 void WebMediaPlayerMS::SetVideoFrameProviderClient(
 448     cc::VideoFrameProvider::Client* client) {
 449   // This is called from both the main renderer thread and the compositor
 450   // thread (when the main thread is blocked).
 451   if (video_frame_provider_client_)
 452     video_frame_provider_client_->StopUsingProvider();
 453   video_frame_provider_client_ = client;
 454 }
 455
 456 bool WebMediaPlayerMS::UpdateCurrentFrame(base::TimeTicks deadline_min,
 457                                           base::TimeTicks deadline_max) {
 458   // TODO(dalecurtis): This should make use of the deadline interval to ensure
 459   // the painted frame is correct for the given interval.
 460   NOTREACHED();
 461   return false;
 462 }
 463
 464 bool WebMediaPlayerMS::HasCurrentFrame() {
 465   base::AutoLock auto_lock(current_frame_lock_);
 466   return current_frame_;
 467 }
 468
 469 scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
 470   DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
 471   base::AutoLock auto_lock(current_frame_lock_);
 472   if (!current_frame_.get())
 473     return NULL;
 474   current_frame_used_ = true;
 475   return current_frame_;
 476 }
 477
 478 void WebMediaPlayerMS::PutCurrentFrame() {
 479   DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
 480 }
 481
 482 void WebMediaPlayerMS::OnFrameAvailable(
 483     const scoped_refptr<media::VideoFrame>& frame) {
 484   DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
 485   DCHECK(thread_checker_.CalledOnValidThread());
 486   ++total_frame_count_;
 487   if (!received_first_frame_) {
 488     received_first_frame_ = true;
 489     {
 490       base::AutoLock auto_lock(current_frame_lock_);
 491       DCHECK(!current_frame_used_);
 492       current_frame_ = frame;
 493     }
 494     SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
 495     SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
 496     GetClient()->sizeChanged();
 497
 498     if (video_frame_provider_.get()) {
 499       video_weblayer_.reset(new cc_blink::WebLayerImpl(
 500           cc::VideoLayer::Create(cc_blink::WebLayerImpl::LayerSettings(), this,
 501                                  media::VIDEO_ROTATION_0)));
 502       video_weblayer_->setOpaque(true);
 503       GetClient()->setWebLayer(video_weblayer_.get());
 504     }
 505   }
 506
 507   // Do not update |current_frame_| when paused.
 508   if (paused_)
 509     return;
 510
 511   bool size_changed = !current_frame_.get() ||
 512                       current_frame_->natural_size() != frame->natural_size();
 513
 514   {
 515     base::AutoLock auto_lock(current_frame_lock_);
 516     if (!current_frame_used_ && current_frame_.get())
 517       ++dropped_frame_count_;
 518     current_frame_ = frame;
 519     current_time_ = frame->timestamp();
 520     current_frame_used_ = false;
 521   }
 522
 523   if (size_changed)
 524     GetClient()->sizeChanged();
 525
 526   GetClient()->repaint();
 527 }
 528
 529 void WebMediaPlayerMS::RepaintInternal() {
 530   DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
 531   DCHECK(thread_checker_.CalledOnValidThread());
 532   GetClient()->repaint();
 533 }
 534
 535 void WebMediaPlayerMS::OnSourceError() {
 536   DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
 537   DCHECK(thread_checker_.CalledOnValidThread());
 538   SetNetworkState(WebMediaPlayer::NetworkStateFormatError);
 539   RepaintInternal();
 540 }
 541
 542 void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) {
 543   DCHECK(thread_checker_.CalledOnValidThread());
 544   network_state_ = state;
 545   // Always notify to ensure client has the latest value.
 546   GetClient()->networkStateChanged();
 547 }
 548
 549 void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) {
 550   DCHECK(thread_checker_.CalledOnValidThread());
 551   ready_state_ = state;
 552   // Always notify to ensure client has the latest value.
 553   GetClient()->readyStateChanged();
 554 }
 555
 556 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
 557   DCHECK(thread_checker_.CalledOnValidThread());
 558   DCHECK(client_);
 559   return client_;
 560 }
 561
 562 }  // namespace content