content/renderer/media/android/webmediaplayer_android.h

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef CONTENT_RENDERER_MEDIA_ANDROID_WEBMEDIAPLAYER_ANDROID_H_
   6 #define CONTENT_RENDERER_MEDIA_ANDROID_WEBMEDIAPLAYER_ANDROID_H_
   7
   8 #include <jni.h>
   9 #include <string>
  10 #include <vector>
  11
  12 #include "base/basictypes.h"
  13 #include "base/memory/ref_counted.h"
  14 #include "base/memory/scoped_ptr.h"
  15 #include "base/memory/weak_ptr.h"
  16 #include "base/threading/thread_checker.h"
  17 #include "base/time/default_tick_clock.h"
  18 #include "base/time/time.h"
  19 #include "cc/layers/video_frame_provider.h"
  20 #include "content/common/media/media_player_messages_enums_android.h"
  21 #include "content/public/renderer/render_frame_observer.h"
  22 #include "content/renderer/media/android/media_info_loader.h"
  23 #include "content/renderer/media/android/media_source_delegate.h"
  24 #include "content/renderer/media/android/stream_texture_factory.h"
  25 #include "gpu/command_buffer/common/mailbox.h"
  26 #include "media/base/android/media_player_android.h"
  27 #include "media/base/cdm_context.h"
  28 #include "media/base/demuxer_stream.h"
  29 #include "media/base/media_keys.h"
  30 #include "media/base/time_delta_interpolator.h"
  31 #include "media/blink/webmediaplayer_params.h"
  32 #include "media/blink/webmediaplayer_util.h"
  33 #include "media/cdm/proxy_decryptor.h"
  34 #include "third_party/WebKit/public/platform/WebGraphicsContext3D.h"
  35 #include "third_party/WebKit/public/platform/WebMediaPlayer.h"
  36 #include "third_party/WebKit/public/platform/WebSize.h"
  37 #include "third_party/WebKit/public/platform/WebURL.h"
  38 #include "third_party/skia/include/core/SkBitmap.h"
  39 #include "ui/gfx/geometry/rect_f.h"
  40
  41 namespace base {
  42 class SingleThreadTaskRunner;
  43 }
  44
  45 namespace blink {
  46 class WebContentDecryptionModule;
  47 class WebContentDecryptionModuleResult;
  48 class WebFrame;
  49 class WebMediaPlayerClient;
  50 class WebMediaPlayerEncryptedMediaClient;
  51 class WebURL;
  52 }
  53
  54 namespace cc_blink {
  55 class WebLayerImpl;
  56 }
  57
  58 namespace gpu {
  59 struct MailboxHolder;
  60 }
  61
  62 namespace media {
  63 class CdmContext;
  64 class CdmFactory;
  65 class MediaLog;
  66 class MediaPermission;
  67 class WebContentDecryptionModuleImpl;
  68 class WebMediaPlayerDelegate;
  69 }
  70
  71 namespace content {
  72
  73 class RendererCdmManager;
  74 class RendererMediaPlayerManager;
  75
  76 // This class implements blink::WebMediaPlayer by keeping the android
  77 // media player in the browser process. It listens to all the status changes
  78 // sent from the browser process and sends playback controls to the media
  79 // player.
  80 class WebMediaPlayerAndroid : public blink::WebMediaPlayer,
  81                               public cc::VideoFrameProvider,
  82                               public RenderFrameObserver,
  83                               public StreamTextureFactoryContextObserver {
  84  public:
  85   // Construct a WebMediaPlayerAndroid object. This class communicates with the
  86   // MediaPlayerAndroid object in the browser process through |proxy|.
  87   // TODO(qinmin): |frame| argument is used to determine whether the current
  88   // player can enter fullscreen. This logic should probably be moved into
  89   // blink, so that enterFullscreen() will not be called if another video is
  90   // already in fullscreen.
  91   WebMediaPlayerAndroid(
  92       blink::WebFrame* frame,
  93       blink::WebMediaPlayerClient* client,
  94       blink::WebMediaPlayerEncryptedMediaClient* encrypted_client,
  95       base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
  96       RendererMediaPlayerManager* player_manager,
  97       media::CdmFactory* cdm_factory,
  98       scoped_refptr<StreamTextureFactory> factory,
  99       const media::WebMediaPlayerParams& params);
 100   virtual ~WebMediaPlayerAndroid();
 101
 102   // blink::WebMediaPlayer implementation.
 103   virtual bool supportsOverlayFullscreenVideo();
 104   virtual void enterFullscreen();
 105
 106   // Resource loading.
 107   virtual void load(LoadType load_type,
 108                     const blink::WebURL& url,
 109                     CORSMode cors_mode);
 110
 111   // Playback controls.
 112   virtual void play();
 113   virtual void pause();
 114   virtual void seek(double seconds);
 115   virtual bool supportsSave() const;
 116   virtual void setRate(double rate);
 117   virtual void setVolume(double volume);
 118   virtual void setSinkId(const blink::WebString& device_id,
 119                          media::WebSetSinkIdCB* raw_web_callbacks);
 120   virtual void requestRemotePlayback();
 121   virtual void requestRemotePlaybackControl();
 122   virtual blink::WebTimeRanges buffered() const;
 123   virtual blink::WebTimeRanges seekable() const;
 124
 125   // Poster image, as defined in the <video> element.
 126   virtual void setPoster(const blink::WebURL& poster) override;
 127
 128   // Methods for painting.
 129   // FIXME: This path "only works" on Android. It is a workaround for the
 130   // issue that Skia could not handle Android's GL_TEXTURE_EXTERNAL_OES texture
 131   // internally. It should be removed and replaced by the normal paint path.
 132   // https://code.google.com/p/skia/issues/detail?id=1189
 133   virtual void paint(blink::WebCanvas* canvas,
 134                      const blink::WebRect& rect,
 135                      unsigned char alpha,
 136                      SkXfermode::Mode mode);
 137
 138   bool copyVideoTextureToPlatformTexture(
 139       blink::WebGraphicsContext3D* web_graphics_context,
 140       unsigned int texture,
 141       unsigned int internal_format,
 142       unsigned int type,
 143       bool premultiply_alpha,
 144       bool flip_y) override;
 145
 146   // True if the loaded media has a playable video/audio track.
 147   virtual bool hasVideo() const;
 148   virtual bool hasAudio() const;
 149
 150   virtual bool isRemote() const;
 151
 152   // Dimensions of the video.
 153   virtual blink::WebSize naturalSize() const;
 154
 155   // Getters of playback state.
 156   virtual bool paused() const;
 157   virtual bool seeking() const;
 158   virtual double duration() const;
 159   virtual double timelineOffset() const;
 160   virtual double currentTime() const;
 161
 162   virtual bool didLoadingProgress();
 163
 164   // Internal states of loading and network.
 165   virtual blink::WebMediaPlayer::NetworkState networkState() const;
 166   virtual blink::WebMediaPlayer::ReadyState readyState() const;
 167
 168   virtual bool hasSingleSecurityOrigin() const;
 169   virtual bool didPassCORSAccessCheck() const;
 170
 171   virtual double mediaTimeForTimeValue(double timeValue) const;
 172
 173   // Provide statistics.
 174   virtual unsigned decodedFrameCount() const;
 175   virtual unsigned droppedFrameCount() const;
 176   virtual unsigned audioDecodedByteCount() const;
 177   virtual unsigned videoDecodedByteCount() const;
 178
 179   // cc::VideoFrameProvider implementation. These methods are running on the
 180   // compositor thread.
 181   void SetVideoFrameProviderClient(
 182       cc::VideoFrameProvider::Client* client) override;
 183   bool UpdateCurrentFrame(base::TimeTicks deadline_min,
 184                           base::TimeTicks deadline_max) override;
 185   bool HasCurrentFrame() override;
 186   scoped_refptr<media::VideoFrame> GetCurrentFrame() override;
 187   void PutCurrentFrame() override;
 188
 189   // Media player callback handlers.
 190   void OnMediaMetadataChanged(base::TimeDelta duration, int width,
 191                               int height, bool success);
 192   void OnPlaybackComplete();
 193   void OnBufferingUpdate(int percentage);
 194   void OnSeekRequest(const base::TimeDelta& time_to_seek);
 195   void OnSeekComplete(const base::TimeDelta& current_time);
 196   void OnMediaError(int error_type);
 197   void OnVideoSizeChanged(int width, int height);
 198   void OnDurationChanged(const base::TimeDelta& duration);
 199
 200   // Called to update the current time.
 201   void OnTimeUpdate(base::TimeDelta current_timestamp,
 202                     base::TimeTicks current_time_ticks);
 203
 204   // Functions called when media player status changes.
 205   void OnConnectedToRemoteDevice(const std::string& remote_playback_message);
 206   void OnDisconnectedFromRemoteDevice();
 207   void OnDidExitFullscreen();
 208   void OnMediaPlayerPlay();
 209   void OnMediaPlayerPause();
 210   void OnRemoteRouteAvailabilityChanged(bool routes_available);
 211
 212   // StreamTextureFactoryContextObserver implementation.
 213   void ResetStreamTextureProxy() override;
 214
 215   // Called when the player is released.
 216   virtual void OnPlayerReleased();
 217
 218   // This function is called by the RendererMediaPlayerManager to pause the
 219   // video and release the media player and surface texture when we switch tabs.
 220   // However, the actual GlTexture is not released to keep the video screenshot.
 221   virtual void ReleaseMediaResources();
 222
 223   // RenderFrameObserver implementation.
 224   void OnDestruct() override;
 225
 226 #if defined(VIDEO_HOLE)
 227   // Calculate the boundary rectangle of the media player (i.e. location and
 228   // size of the video frame).
 229   // Returns true if the geometry has been changed since the last call.
 230   bool UpdateBoundaryRectangle();
 231
 232   const gfx::RectF GetBoundaryRectangle();
 233 #endif  // defined(VIDEO_HOLE)
 234
 235   virtual MediaKeyException generateKeyRequest(
 236       const blink::WebString& key_system,
 237       const unsigned char* init_data,
 238       unsigned init_data_length);
 239   virtual MediaKeyException addKey(
 240       const blink::WebString& key_system,
 241       const unsigned char* key,
 242       unsigned key_length,
 243       const unsigned char* init_data,
 244       unsigned init_data_length,
 245       const blink::WebString& session_id);
 246   virtual MediaKeyException cancelKeyRequest(
 247       const blink::WebString& key_system,
 248       const blink::WebString& session_id);
 249
 250   virtual void setContentDecryptionModule(
 251       blink::WebContentDecryptionModule* cdm,
 252       blink::WebContentDecryptionModuleResult result);
 253
 254   void OnKeyAdded(const std::string& session_id);
 255   void OnKeyError(const std::string& session_id,
 256                   media::MediaKeys::KeyError error_code,
 257                   uint32 system_code);
 258   void OnKeyMessage(const std::string& session_id,
 259                     const std::vector<uint8>& message,
 260                     const GURL& destination_url);
 261
 262   void OnMediaSourceOpened(blink::WebMediaSource* web_media_source);
 263
 264   void OnEncryptedMediaInitData(media::EmeInitDataType init_data_type,
 265                                 const std::vector<uint8>& init_data);
 266
 267   // Called when a decoder detects that the key needed to decrypt the stream
 268   // is not available.
 269   void OnWaitingForDecryptionKey();
 270
 271  protected:
 272   // Helper method to update the playing state.
 273   void UpdatePlayingState(bool is_playing_);
 274
 275   // Helper methods for posting task for setting states and update WebKit.
 276   void UpdateNetworkState(blink::WebMediaPlayer::NetworkState state);
 277   void UpdateReadyState(blink::WebMediaPlayer::ReadyState state);
 278   void TryCreateStreamTextureProxyIfNeeded();
 279   void DoCreateStreamTexture();
 280
 281   // Helper method to reestablish the surface texture peer for android
 282   // media player.
 283   void EstablishSurfaceTexturePeer();
 284
 285   // Requesting whether the surface texture peer needs to be reestablished.
 286   void SetNeedsEstablishPeer(bool needs_establish_peer);
 287
 288  private:
 289   void InitializePlayer(const GURL& url,
 290                         const GURL& first_party_for_cookies,
 291                         bool allowed_stored_credentials,
 292                         int demuxer_client_id);
 293   void Pause(bool is_media_related_action);
 294   void DrawRemotePlaybackText(const std::string& remote_playback_message);
 295   void ReallocateVideoFrame();
 296   void SetCurrentFrameInternal(scoped_refptr<media::VideoFrame>& frame);
 297   void RemoveSurfaceTextureAndProxy();
 298   void DidLoadMediaInfo(MediaInfoLoader::Status status,
 299                         const GURL& redirected_url,
 300                         const GURL& first_party_for_cookies,
 301                         bool allow_stored_credentials);
 302   bool IsKeySystemSupported(const std::string& key_system);
 303   bool IsLocalResource();
 304
 305   // Actually do the work for generateKeyRequest/addKey so they can easily
 306   // report results to UMA.
 307   MediaKeyException GenerateKeyRequestInternal(const std::string& key_system,
 308                                                const unsigned char* init_data,
 309                                                unsigned init_data_length);
 310   MediaKeyException AddKeyInternal(const std::string& key_system,
 311                                    const unsigned char* key,
 312                                    unsigned key_length,
 313                                    const unsigned char* init_data,
 314                                    unsigned init_data_length,
 315                                    const std::string& session_id);
 316   MediaKeyException CancelKeyRequestInternal(const std::string& key_system,
 317                                              const std::string& session_id);
 318
 319   // Called when |cdm_context| is ready.
 320   void OnCdmContextReady(media::CdmContext* cdm_context);
 321
 322   // Sets the CDM. Should only be called when |is_player_initialized_| is true
 323   // and a new non-null |cdm_context_| is available. Fires |cdm_attached_cb_|
 324   // with the result after the CDM is attached.
 325   void SetCdmInternal(const media::CdmAttachedCB& cdm_attached_cb);
 326
 327   // Requests that this object notifies when a decryptor is ready through the
 328   // |decryptor_ready_cb| provided.
 329   // If |decryptor_ready_cb| is null, the existing callback will be fired with
 330   // NULL immediately and reset.
 331   void SetDecryptorReadyCB(const media::DecryptorReadyCB& decryptor_ready_cb);
 332
 333   // Called when the ContentDecryptionModule has been attached to the
 334   // pipeline/decoders.
 335   void ContentDecryptionModuleAttached(
 336       blink::WebContentDecryptionModuleResult result,
 337       bool success);
 338
 339   bool IsHLSStream() const;
 340   // Report whether the loaded url, after following redirects, points to a HLS
 341   // playlist, and record the origin of the player.
 342   void ReportHLSMetrics() const;
 343
 344   // Called after |defer_load_cb_| has decided to allow the load. If
 345   // |defer_load_cb_| is null this is called immediately.
 346   void DoLoad(LoadType load_type, const blink::WebURL& url, CORSMode cors_mode);
 347
 348   blink::WebFrame* const frame_;
 349
 350   blink::WebMediaPlayerClient* const client_;
 351   blink::WebMediaPlayerEncryptedMediaClient* const encrypted_client_;
 352
 353   // |delegate_| is used to notify the browser process of the player status, so
 354   // that the browser process can control screen locks.
 355   // TODO(qinmin): Currently android mediaplayer takes care of the screen
 356   // lock. So this is only used for media source. Will apply this to regular
 357   // media tag once http://crbug.com/247892 is fixed.
 358   base::WeakPtr<media::WebMediaPlayerDelegate> delegate_;
 359
 360   // Callback responsible for determining if loading of media should be deferred
 361   // for external reasons; called during load().
 362   media::WebMediaPlayerParams::DeferLoadCB defer_load_cb_;
 363
 364   // Save the list of buffered time ranges.
 365   blink::WebTimeRanges buffered_;
 366
 367   // Size of the video.
 368   blink::WebSize natural_size_;
 369
 370   // Size that has been sent to StreamTexture.
 371   blink::WebSize cached_stream_texture_size_;
 372
 373   // The video frame object used for rendering by the compositor.
 374   scoped_refptr<media::VideoFrame> current_frame_;
 375   base::Lock current_frame_lock_;
 376
 377   base::ThreadChecker main_thread_checker_;
 378
 379   // Message loop for media thread.
 380   const scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
 381
 382   // URL of the media file to be fetched.
 383   GURL url_;
 384
 385   // URL of the media file after |media_info_loader_| resolves all the
 386   // redirections.
 387   GURL redirected_url_;
 388
 389   // Media duration.
 390   base::TimeDelta duration_;
 391
 392   // Flag to remember if we have a trusted duration_ value provided by
 393   // MediaSourceDelegate notifying OnDurationChanged(). In this case, ignore
 394   // any subsequent duration value passed to OnMediaMetadataChange().
 395   bool ignore_metadata_duration_change_;
 396
 397   // Seek gets pending if another seek is in progress. Only last pending seek
 398   // will have effect.
 399   bool pending_seek_;
 400   base::TimeDelta pending_seek_time_;
 401
 402   // Internal seek state.
 403   bool seeking_;
 404   base::TimeDelta seek_time_;
 405
 406   // Whether loading has progressed since the last call to didLoadingProgress.
 407   bool did_loading_progress_;
 408
 409   // Manages this object and delegates player calls to the browser process.
 410   // Owned by RenderFrameImpl.
 411   RendererMediaPlayerManager* const player_manager_;
 412
 413   // TODO(xhwang): Remove |cdm_factory_| when prefixed EME is deprecated. See
 414   // http://crbug.com/249976
 415   media::CdmFactory* const cdm_factory_;
 416
 417   media::MediaPermission* media_permission_;
 418
 419   // Player ID assigned by the |player_manager_|.
 420   int player_id_;
 421
 422   // Current player states.
 423   blink::WebMediaPlayer::NetworkState network_state_;
 424   blink::WebMediaPlayer::ReadyState ready_state_;
 425
 426   // GL texture ID allocated to the video.
 427   unsigned int texture_id_;
 428
 429   // GL texture mailbox for texture_id_ to provide in the VideoFrame, and sync
 430   // point for when the mailbox was produced.
 431   gpu::Mailbox texture_mailbox_;
 432
 433   // Stream texture ID allocated to the video.
 434   unsigned int stream_id_;
 435
 436   // Whether the media player has been initialized.
 437   bool is_player_initialized_;
 438
 439   // Whether the media player is playing.
 440   bool is_playing_;
 441
 442   // Whether media player needs to re-establish the surface texture peer.
 443   bool needs_establish_peer_;
 444
 445   // Whether the video size info is available.
 446   bool has_size_info_;
 447
 448   const scoped_refptr<base::SingleThreadTaskRunner> compositor_task_runner_;
 449
 450   // Object for allocating stream textures.
 451   scoped_refptr<StreamTextureFactory> stream_texture_factory_;
 452
 453   // Object for calling back the compositor thread to repaint the video when a
 454   // frame available. It should be initialized on the compositor thread.
 455   // Accessed on main thread and on compositor thread when main thread is
 456   // blocked.
 457   ScopedStreamTextureProxy stream_texture_proxy_;
 458
 459   // Whether media player needs external surface.
 460   // Only used for the VIDEO_HOLE logic.
 461   bool needs_external_surface_;
 462
 463   // Whether the player is in fullscreen.
 464   bool is_fullscreen_;
 465
 466   // A pointer back to the compositor to inform it about state changes. This is
 467   // not NULL while the compositor is actively using this webmediaplayer.
 468   // Accessed on main thread and on compositor thread when main thread is
 469   // blocked.
 470   cc::VideoFrameProvider::Client* video_frame_provider_client_;
 471
 472   scoped_ptr<cc_blink::WebLayerImpl> video_weblayer_;
 473
 474 #if defined(VIDEO_HOLE)
 475   // A rectangle represents the geometry of video frame, when computed last
 476   // time.
 477   gfx::RectF last_computed_rect_;
 478
 479   // Whether to use the video overlay for all embedded video.
 480   // True only for testing.
 481   bool force_use_overlay_embedded_video_;
 482 #endif  // defined(VIDEO_HOLE)
 483
 484   MediaPlayerHostMsg_Initialize_Type player_type_;
 485
 486   // Whether the browser is currently connected to a remote media player.
 487   bool is_remote_;
 488
 489   scoped_refptr<media::MediaLog> media_log_;
 490
 491   scoped_ptr<MediaInfoLoader> info_loader_;
 492
 493   // The currently selected key system. Empty string means that no key system
 494   // has been selected.
 495   std::string current_key_system_;
 496
 497   // Temporary for EME v0.1. Not needed for unprefixed EME, and can be removed
 498   // when prefixed EME is removed.
 499   media::EmeInitDataType init_data_type_;
 500
 501   // Manages decryption keys and decrypts encrypted frames.
 502   scoped_ptr<media::ProxyDecryptor> proxy_decryptor_;
 503
 504   // Non-owned pointer to the CdmContext. Updated in the constructor,
 505   // generateKeyRequest() or setContentDecryptionModule().
 506   media::CdmContext* cdm_context_;
 507
 508   // This is only Used by Clear Key key system implementation, where a renderer
 509   // side CDM will be used. This is similar to WebMediaPlayerImpl. For other key
 510   // systems, a browser side CDM will be used and we set CDM by calling
 511   // player_manager_->SetCdm() directly.
 512   media::DecryptorReadyCB decryptor_ready_cb_;
 513
 514   SkBitmap bitmap_;
 515
 516   // Whether stored credentials are allowed to be passed to the server.
 517   bool allow_stored_credentials_;
 518
 519   // Whether the resource is local.
 520   bool is_local_resource_;
 521
 522   // base::TickClock used by |interpolator_|.
 523   base::DefaultTickClock default_tick_clock_;
 524
 525   // Tracks the most recent media time update and provides interpolated values
 526   // as playback progresses.
 527   media::TimeDeltaInterpolator interpolator_;
 528
 529   scoped_ptr<MediaSourceDelegate> media_source_delegate_;
 530
 531   // NOTE: Weak pointers must be invalidated before all other member variables.
 532   base::WeakPtrFactory<WebMediaPlayerAndroid> weak_factory_;
 533
 534   DISALLOW_COPY_AND_ASSIGN(WebMediaPlayerAndroid);
 535 };
 536
 537 }  // namespace content
 538
 539 #endif  // CONTENT_RENDERER_MEDIA_ANDROID_WEBMEDIAPLAYER_ANDROID_H_