content/renderer/media/webmediaplayer_ms.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webmediaplayer_ms.h"
   6
   7 #include <limits>
   8
   9 #include "base/bind.h"
  10 #include "base/callback.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/metrics/histogram.h"
  13 #include "cc/blink/context_provider_web_context.h"
  14 #include "cc/blink/web_layer_impl.h"
  15 #include "cc/layers/video_layer.h"
  16 #include "content/public/renderer/media_stream_audio_renderer.h"
  17 #include "content/public/renderer/media_stream_renderer_factory.h"
  18 #include "content/public/renderer/render_view.h"
  19 #include "content/public/renderer/video_frame_provider.h"
  20 #include "content/renderer/render_frame_impl.h"
  21 #include "content/renderer/render_thread_impl.h"
  22 #include "gpu/blink/webgraphicscontext3d_impl.h"
  23 #include "media/base/media_log.h"
  24 #include "media/base/video_frame.h"
  25 #include "media/base/video_rotation.h"
  26 #include "media/base/video_util.h"
  27 #include "media/blink/webmediaplayer_delegate.h"
  28 #include "media/blink/webmediaplayer_util.h"
  29 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
  30 #include "third_party/WebKit/public/platform/WebRect.h"
  31 #include "third_party/WebKit/public/platform/WebSize.h"
  32 #include "third_party/WebKit/public/platform/WebURL.h"
  33 #include "third_party/WebKit/public/web/WebFrame.h"
  34 #include "third_party/WebKit/public/web/WebView.h"
  35 #include "third_party/skia/include/core/SkBitmap.h"
  36
  37 using blink::WebCanvas;
  38 using blink::WebMediaPlayer;
  39 using blink::WebRect;
  40 using blink::WebSize;
  41
  42 namespace content {
  43
  44 namespace {
  45
  46 // This function copies |frame| to a new YV12 media::VideoFrame.
  47 scoped_refptr<media::VideoFrame> CopyFrameToYV12(
  48     const scoped_refptr<media::VideoFrame>& frame,
  49     media::SkCanvasVideoRenderer* video_renderer) {
  50   scoped_refptr<media::VideoFrame> new_frame =
  51       media::VideoFrame::CreateFrame(media::VideoFrame::YV12,
  52                                      frame->coded_size(),
  53                                      frame->visible_rect(),
  54                                      frame->natural_size(),
  55                                      frame->timestamp());
  56
  57   if (frame->HasTextures()) {
  58     DCHECK(frame->format() == media::VideoFrame::ARGB ||
  59            frame->format() == media::VideoFrame::XRGB);
  60     SkBitmap bitmap;
  61     bitmap.allocN32Pixels(frame->visible_rect().width(),
  62                           frame->visible_rect().height());
  63     SkCanvas canvas(bitmap);
  64
  65     cc::ContextProvider* const provider =
  66         RenderThreadImpl::current()->SharedMainThreadContextProvider().get();
  67     if (provider) {
  68       const media::Context3D context_3d =
  69           media::Context3D(provider->ContextGL(), provider->GrContext());
  70       DCHECK(context_3d.gl);
  71       video_renderer->Copy(frame.get(), &canvas, context_3d);
  72     } else {
  73       // GPU Process crashed.
  74       bitmap.eraseColor(SK_ColorTRANSPARENT);
  75     }
  76     media::CopyRGBToVideoFrame(reinterpret_cast<uint8*>(bitmap.getPixels()),
  77                                bitmap.rowBytes(),
  78                                frame->visible_rect(),
  79                                new_frame.get());
  80   } else {
  81     DCHECK(frame->IsMappable());
  82     DCHECK(frame->format() == media::VideoFrame::YV12 ||
  83            frame->format() == media::VideoFrame::I420);
  84     for (size_t i = 0; i < media::VideoFrame::NumPlanes(frame->format()); ++i) {
  85       media::CopyPlane(i, frame->data(i), frame->stride(i),
  86                        frame->rows(i), new_frame.get());
  87     }
  88   }
  89   return new_frame;
  90 }
  91
  92 }  // anonymous namespace
  93
  94 WebMediaPlayerMS::WebMediaPlayerMS(
  95     blink::WebFrame* frame,
  96     blink::WebMediaPlayerClient* client,
  97     base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
  98     media::MediaLog* media_log,
  99     scoped_ptr<MediaStreamRendererFactory> factory)
 100     : frame_(frame),
 101       network_state_(WebMediaPlayer::NetworkStateEmpty),
 102       ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
 103       buffered_(static_cast<size_t>(0)),
 104       volume_(1.0f),
 105       client_(client),
 106       delegate_(delegate),
 107       paused_(true),
 108       current_frame_used_(false),
 109       video_frame_provider_client_(NULL),
 110       received_first_frame_(false),
 111       total_frame_count_(0),
 112       dropped_frame_count_(0),
 113       media_log_(media_log),
 114       renderer_factory_(factory.Pass()) {
 115   DVLOG(1) << "WebMediaPlayerMS::ctor";
 116   media_log_->AddEvent(
 117       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
 118 }
 119
 120 WebMediaPlayerMS::~WebMediaPlayerMS() {
 121   DVLOG(1) << "WebMediaPlayerMS::dtor";
 122   DCHECK(thread_checker_.CalledOnValidThread());
 123
 124   SetVideoFrameProviderClient(NULL);
 125   GetClient()->setWebLayer(NULL);
 126
 127   if (video_frame_provider_.get())
 128     video_frame_provider_->Stop();
 129
 130   if (audio_renderer_.get())
 131     audio_renderer_->Stop();
 132
 133   media_log_->AddEvent(
 134       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
 135
 136   if (delegate_.get())
 137     delegate_->PlayerGone(this);
 138 }
 139
 140 void WebMediaPlayerMS::load(LoadType load_type,
 141                             const blink::WebURL& url,
 142                             CORSMode cors_mode) {
 143   DVLOG(1) << "WebMediaPlayerMS::load";
 144   DCHECK(thread_checker_.CalledOnValidThread());
 145
 146   // TODO(acolwell): Change this to DCHECK_EQ(load_type,
 147   // LoadTypeMediaStream) once Blink-side changes land.
 148   DCHECK_NE(load_type, LoadTypeMediaSource);
 149
 150   GURL gurl(url);
 151
 152   SetNetworkState(WebMediaPlayer::NetworkStateLoading);
 153   SetReadyState(WebMediaPlayer::ReadyStateHaveNothing);
 154   media_log_->AddEvent(media_log_->CreateLoadEvent(url.spec()));
 155
 156   video_frame_provider_ = renderer_factory_->GetVideoFrameProvider(
 157       url,
 158       base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr()),
 159       base::Bind(&WebMediaPlayerMS::OnFrameAvailable, AsWeakPtr()));
 160
 161   RenderFrame* frame = RenderFrame::FromWebFrame(frame_);
 162   audio_renderer_ = renderer_factory_->GetAudioRenderer(
 163     url,
 164     frame->GetRoutingID());
 165
 166   if (video_frame_provider_.get() || audio_renderer_.get()) {
 167     if (audio_renderer_.get()) {
 168       audio_renderer_->SetVolume(volume_);
 169       audio_renderer_->Start();
 170     }
 171
 172     if (video_frame_provider_.get()) {
 173       video_frame_provider_->Start();
 174     } else {
 175       // This is audio-only mode.
 176       DCHECK(audio_renderer_.get());
 177       SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
 178       SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
 179     }
 180   } else {
 181     SetNetworkState(WebMediaPlayer::NetworkStateNetworkError);
 182   }
 183 }
 184
 185 void WebMediaPlayerMS::play() {
 186   DVLOG(1) << "WebMediaPlayerMS::play";
 187   DCHECK(thread_checker_.CalledOnValidThread());
 188
 189   if (paused_) {
 190     if (video_frame_provider_.get())
 191       video_frame_provider_->Play();
 192
 193     if (audio_renderer_.get())
 194       audio_renderer_->Play();
 195
 196     if (delegate_.get())
 197       delegate_->DidPlay(this);
 198   }
 199
 200   paused_ = false;
 201
 202   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
 203 }
 204
 205 void WebMediaPlayerMS::pause() {
 206   DVLOG(1) << "WebMediaPlayerMS::pause";
 207   DCHECK(thread_checker_.CalledOnValidThread());
 208
 209   if (video_frame_provider_.get())
 210     video_frame_provider_->Pause();
 211
 212   if (!paused_) {
 213     if (audio_renderer_.get())
 214       audio_renderer_->Pause();
 215
 216     if (delegate_.get())
 217       delegate_->DidPause(this);
 218   }
 219
 220   paused_ = true;
 221
 222   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
 223
 224   if (!current_frame_.get())
 225     return;
 226
 227   // Copy the frame so that rendering can show the last received frame.
 228   // The original frame must not be referenced when the player is paused since
 229   // there might be a finite number of available buffers. E.g, video that
 230   // originates from a video camera.
 231   scoped_refptr<media::VideoFrame> new_frame =
 232       CopyFrameToYV12(current_frame_, &video_renderer_);
 233
 234   base::AutoLock auto_lock(current_frame_lock_);
 235   current_frame_ = new_frame;
 236 }
 237
 238 bool WebMediaPlayerMS::supportsSave() const {
 239   DCHECK(thread_checker_.CalledOnValidThread());
 240   return false;
 241 }
 242
 243 void WebMediaPlayerMS::seek(double seconds) {
 244   DCHECK(thread_checker_.CalledOnValidThread());
 245 }
 246
 247 void WebMediaPlayerMS::setRate(double rate) {
 248   DCHECK(thread_checker_.CalledOnValidThread());
 249 }
 250
 251 void WebMediaPlayerMS::setVolume(double volume) {
 252   DCHECK(thread_checker_.CalledOnValidThread());
 253   DVLOG(1) << "WebMediaPlayerMS::setVolume(volume=" << volume << ")";
 254   volume_ = volume;
 255   if (audio_renderer_.get())
 256     audio_renderer_->SetVolume(volume_);
 257 }
 258
 259 void WebMediaPlayerMS::setSinkId(const blink::WebString& device_id,
 260                                  media::WebSetSinkIdCB* web_callback) {
 261   DCHECK(thread_checker_.CalledOnValidThread());
 262   std::string device_id_str(device_id.utf8());
 263   GURL security_origin(frame_->securityOrigin().toString().utf8());
 264   DVLOG(1) << __FUNCTION__
 265            << "(" << device_id_str << ", " << security_origin << ")";
 266   media::SwitchOutputDeviceCB callback =
 267       media::ConvertToSwitchOutputDeviceCB(web_callback);
 268   if (audio_renderer_.get()) {
 269     audio_renderer_->SwitchOutputDevice(device_id_str, security_origin,
 270                                         callback);
 271   } else {
 272     callback.Run(media::SWITCH_OUTPUT_DEVICE_RESULT_ERROR_NOT_SUPPORTED);
 273   }
 274 }
 275
 276 void WebMediaPlayerMS::setPreload(WebMediaPlayer::Preload preload) {
 277   DCHECK(thread_checker_.CalledOnValidThread());
 278 }
 279
 280 bool WebMediaPlayerMS::hasVideo() const {
 281   DCHECK(thread_checker_.CalledOnValidThread());
 282   return (video_frame_provider_.get() != NULL);
 283 }
 284
 285 bool WebMediaPlayerMS::hasAudio() const {
 286   DCHECK(thread_checker_.CalledOnValidThread());
 287   return (audio_renderer_.get() != NULL);
 288 }
 289
 290 blink::WebSize WebMediaPlayerMS::naturalSize() const {
 291   DCHECK(thread_checker_.CalledOnValidThread());
 292
 293   gfx::Size size;
 294   if (current_frame_.get())
 295     size = current_frame_->natural_size();
 296   DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
 297   return blink::WebSize(size);
 298 }
 299
 300 bool WebMediaPlayerMS::paused() const {
 301   DCHECK(thread_checker_.CalledOnValidThread());
 302   return paused_;
 303 }
 304
 305 bool WebMediaPlayerMS::seeking() const {
 306   DCHECK(thread_checker_.CalledOnValidThread());
 307   return false;
 308 }
 309
 310 double WebMediaPlayerMS::duration() const {
 311   DCHECK(thread_checker_.CalledOnValidThread());
 312   return std::numeric_limits<double>::infinity();
 313 }
 314
 315 double WebMediaPlayerMS::currentTime() const {
 316   DCHECK(thread_checker_.CalledOnValidThread());
 317   if (current_time_.ToInternalValue() != 0) {
 318     return current_time_.InSecondsF();
 319   } else if (audio_renderer_.get()) {
 320     return audio_renderer_->GetCurrentRenderTime().InSecondsF();
 321   }
 322   return 0.0;
 323 }
 324
 325 WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
 326   DCHECK(thread_checker_.CalledOnValidThread());
 327   DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
 328   return network_state_;
 329 }
 330
 331 WebMediaPlayer::ReadyState WebMediaPlayerMS::readyState() const {
 332   DCHECK(thread_checker_.CalledOnValidThread());
 333   DVLOG(1) << "WebMediaPlayerMS::readyState, state:" << ready_state_;
 334   return ready_state_;
 335 }
 336
 337 blink::WebTimeRanges WebMediaPlayerMS::buffered() const {
 338   DCHECK(thread_checker_.CalledOnValidThread());
 339   return buffered_;
 340 }
 341
 342 blink::WebTimeRanges WebMediaPlayerMS::seekable() const {
 343   DCHECK(thread_checker_.CalledOnValidThread());
 344   return blink::WebTimeRanges();
 345 }
 346
 347 bool WebMediaPlayerMS::didLoadingProgress() {
 348   DCHECK(thread_checker_.CalledOnValidThread());
 349   return true;
 350 }
 351
 352 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
 353                              const blink::WebRect& rect,
 354                              unsigned char alpha,
 355                              SkXfermode::Mode mode) {
 356   DVLOG(3) << "WebMediaPlayerMS::paint";
 357   DCHECK(thread_checker_.CalledOnValidThread());
 358
 359   media::Context3D context_3d;
 360   if (current_frame_.get() && current_frame_->HasTextures()) {
 361     cc::ContextProvider* provider =
 362         RenderThreadImpl::current()->SharedMainThreadContextProvider().get();
 363     // GPU Process crashed.
 364     if (!provider)
 365       return;
 366     context_3d = media::Context3D(provider->ContextGL(), provider->GrContext());
 367     DCHECK(context_3d.gl);
 368   }
 369   gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height);
 370   video_renderer_.Paint(current_frame_, canvas, dest_rect, alpha, mode,
 371                         media::VIDEO_ROTATION_0, context_3d);
 372
 373   {
 374     base::AutoLock auto_lock(current_frame_lock_);
 375     if (current_frame_.get())
 376       current_frame_used_ = true;
 377   }
 378 }
 379
 380 bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
 381   DCHECK(thread_checker_.CalledOnValidThread());
 382   return true;
 383 }
 384
 385 bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
 386   DCHECK(thread_checker_.CalledOnValidThread());
 387   return true;
 388 }
 389
 390 double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const {
 391   return media::ConvertSecondsToTimestamp(timeValue).InSecondsF();
 392 }
 393
 394 unsigned WebMediaPlayerMS::decodedFrameCount() const {
 395   DCHECK(thread_checker_.CalledOnValidThread());
 396   DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
 397   return total_frame_count_;
 398 }
 399
 400 unsigned WebMediaPlayerMS::droppedFrameCount() const {
 401   DCHECK(thread_checker_.CalledOnValidThread());
 402   DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
 403   return dropped_frame_count_;
 404 }
 405
 406 unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
 407   DCHECK(thread_checker_.CalledOnValidThread());
 408   NOTIMPLEMENTED();
 409   return 0;
 410 }
 411
 412 unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
 413   DCHECK(thread_checker_.CalledOnValidThread());
 414   NOTIMPLEMENTED();
 415   return 0;
 416 }
 417
 418 bool WebMediaPlayerMS::copyVideoTextureToPlatformTexture(
 419     blink::WebGraphicsContext3D* web_graphics_context,
 420     unsigned int texture,
 421     unsigned int internal_format,
 422     unsigned int type,
 423     bool premultiply_alpha,
 424     bool flip_y) {
 425   TRACE_EVENT0("media", "WebMediaPlayerMS:copyVideoTextureToPlatformTexture");
 426   DCHECK(thread_checker_.CalledOnValidThread());
 427
 428   scoped_refptr<media::VideoFrame> video_frame;
 429   {
 430     base::AutoLock auto_lock(current_frame_lock_);
 431     video_frame = current_frame_;
 432   }
 433
 434   if (!video_frame.get() || video_frame->HasTextures() ||
 435       media::VideoFrame::NumPlanes(video_frame->format()) != 1) {
 436     return false;
 437   }
 438
 439   // TODO(dshwang): need more elegant way to convert WebGraphicsContext3D to
 440   // GLES2Interface.
 441   gpu::gles2::GLES2Interface* gl =
 442       static_cast<gpu_blink::WebGraphicsContext3DImpl*>(web_graphics_context)
 443           ->GetGLInterface();
 444   media::SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
 445       gl, video_frame.get(), texture, internal_format, type, premultiply_alpha,
 446       flip_y);
 447   return true;
 448 }
 449
 450 void WebMediaPlayerMS::SetVideoFrameProviderClient(
 451     cc::VideoFrameProvider::Client* client) {
 452   // This is called from both the main renderer thread and the compositor
 453   // thread (when the main thread is blocked).
 454   if (video_frame_provider_client_)
 455     video_frame_provider_client_->StopUsingProvider();
 456   video_frame_provider_client_ = client;
 457 }
 458
 459 bool WebMediaPlayerMS::UpdateCurrentFrame(base::TimeTicks deadline_min,
 460                                           base::TimeTicks deadline_max) {
 461   // TODO(dalecurtis): This should make use of the deadline interval to ensure
 462   // the painted frame is correct for the given interval.
 463   NOTREACHED();
 464   return false;
 465 }
 466
 467 bool WebMediaPlayerMS::HasCurrentFrame() {
 468   base::AutoLock auto_lock(current_frame_lock_);
 469   return current_frame_;
 470 }
 471
 472 scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
 473   DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
 474   base::AutoLock auto_lock(current_frame_lock_);
 475   if (!current_frame_.get())
 476     return NULL;
 477   current_frame_used_ = true;
 478   return current_frame_;
 479 }
 480
 481 void WebMediaPlayerMS::PutCurrentFrame() {
 482   DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
 483 }
 484
 485 void WebMediaPlayerMS::OnFrameAvailable(
 486     const scoped_refptr<media::VideoFrame>& frame) {
 487   DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
 488   DCHECK(thread_checker_.CalledOnValidThread());
 489   ++total_frame_count_;
 490   if (!received_first_frame_) {
 491     received_first_frame_ = true;
 492     {
 493       base::AutoLock auto_lock(current_frame_lock_);
 494       DCHECK(!current_frame_used_);
 495       current_frame_ = frame;
 496     }
 497     SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
 498     SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
 499     GetClient()->sizeChanged();
 500
 501     if (video_frame_provider_.get()) {
 502       video_weblayer_.reset(new cc_blink::WebLayerImpl(
 503           cc::VideoLayer::Create(cc_blink::WebLayerImpl::LayerSettings(), this,
 504                                  media::VIDEO_ROTATION_0)));
 505       video_weblayer_->setOpaque(true);
 506       GetClient()->setWebLayer(video_weblayer_.get());
 507     }
 508   }
 509
 510   // Do not update |current_frame_| when paused.
 511   if (paused_)
 512     return;
 513
 514   bool size_changed = !current_frame_.get() ||
 515                       current_frame_->natural_size() != frame->natural_size();
 516
 517   {
 518     base::AutoLock auto_lock(current_frame_lock_);
 519     if (!current_frame_used_ && current_frame_.get())
 520       ++dropped_frame_count_;
 521     current_frame_ = frame;
 522     current_time_ = frame->timestamp();
 523     current_frame_used_ = false;
 524   }
 525
 526   if (size_changed)
 527     GetClient()->sizeChanged();
 528
 529   GetClient()->repaint();
 530 }
 531
 532 void WebMediaPlayerMS::RepaintInternal() {
 533   DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
 534   DCHECK(thread_checker_.CalledOnValidThread());
 535   GetClient()->repaint();
 536 }
 537
 538 void WebMediaPlayerMS::OnSourceError() {
 539   DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
 540   DCHECK(thread_checker_.CalledOnValidThread());
 541   SetNetworkState(WebMediaPlayer::NetworkStateFormatError);
 542   RepaintInternal();
 543 }
 544
 545 void WebMediaPlayerMS::SetNetworkState(WebMediaPlayer::NetworkState state) {
 546   DCHECK(thread_checker_.CalledOnValidThread());
 547   network_state_ = state;
 548   // Always notify to ensure client has the latest value.
 549   GetClient()->networkStateChanged();
 550 }
 551
 552 void WebMediaPlayerMS::SetReadyState(WebMediaPlayer::ReadyState state) {
 553   DCHECK(thread_checker_.CalledOnValidThread());
 554   ready_state_ = state;
 555   // Always notify to ensure client has the latest value.
 556   GetClient()->readyStateChanged();
 557 }
 558
 559 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
 560   DCHECK(thread_checker_.CalledOnValidThread());
 561   DCHECK(client_);
 562   return client_;
 563 }
 564
 565 }  // namespace content