media/base/android/media_codec_decoder.h

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef MEDIA_BASE_ANDROID_MEDIA_CODEC_DECODER_H_
   6 #define MEDIA_BASE_ANDROID_MEDIA_CODEC_DECODER_H_
   7
   8 #include "base/android/scoped_java_ref.h"
   9 #include "base/callback.h"
  10 #include "base/macros.h"
  11 #include "base/memory/ref_counted.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/synchronization/lock.h"
  15 #include "base/threading/thread.h"
  16 #include "base/time/time.h"
  17 #include "media/base/android/access_unit_queue.h"
  18 #include "media/base/android/demuxer_stream_player_params.h"
  19
  20 namespace media {
  21
  22 class MediaCodecBridge;
  23
  24 // The decoder for MediaCodecPlayer.
  25 // This class accepts the incoming data into AccessUnitQueue and works with
  26 // MediaCodecBridge for decoding and rendering the frames. The MediaCodecPlayer
  27 // has two decoder objects: audio and video.
  28 //
  29 // The decoder works on two threads. The data from demuxer comes on Media
  30 // thread. The commands from MediaCodecPlayer, such as Prefetch, Start,
  31 // RequestToStop also come on the Media thread. The operations with MediaCodec
  32 // buffers and rendering happen on a separate thread called Decoder thread.
  33 // This class creates, starts and stops it as necessary.
  34 //
  35 // Decoder's internal state machine goes through the following states:
  36 //
  37 //  [ Stopped ] <-------------------          (any state except Error)
  38 //       |                         |                   |
  39 //       | Prefetch                |--- internal ------|
  40 //       v                         |   transition      v
  41 //  [ Prefetching ]                |               [ Error ]
  42 //       |                         |
  43 //       | internal transition     |
  44 //       v                         |            Error recovery:
  45 //  [ Prefetched ]                 |
  46 //       |                         |        (any state including Error)
  47 //       | Configure and Start     |                   |
  48 //       v                         |                   | ReleaseDecoderResources
  49 //  [ Running ]                    |                   v
  50 //       |                         |              [ Stopped ]
  51 //       | RequestToStop           |
  52 //       v                         |
  53 //  [ Stopping ] -------------------
  54 //
  55 //
  56 //  [ Stopped ] --------------------
  57 //       ^                         |
  58 //       |       Flush             |
  59 //       ---------------------------
  60
  61 // Here is the workflow that is expected to be maintained by a caller, which is
  62 // MediaCodecPlayer currently.
  63 //
  64 //  [ Stopped ]
  65 //       |
  66 //       | Prefetch
  67 //       v
  68 //  [ Prefetching ]
  69 //       |
  70 //       | (Enough data received)
  71 //       v
  72 //  [ Prefetched ]
  73 //       |
  74 //       | <---------- SetDemuxerConfigs (*)
  75 //       |
  76 //       | <---------- SetVideoSurface (**)
  77 //       |
  78 //       | Configure --------------------------------------------+
  79 //       |                                                       |
  80 //       v                                                       v
  81 //  ( Config Succeeded )                               ( Key frame required )
  82 //       |                                                       |
  83 //       | Start                                                 |
  84 //       v                                                       |
  85 //   [ Running ] ------------------------------+                 |
  86 //       |                                     |                 |
  87 //       |                                     |                 |
  88 //       | RequestToStop                       | SyncStop        | SyncStop
  89 //       |                                     |                 |
  90 //   [ Stopping ]                              |                 |
  91 //       |                                     |                 |
  92 //       | ( Last frame rendered )             |                 |
  93 //       |                                     |                 |
  94 //       |                                     |                 |
  95 //       v                                     |                 |
  96 //   [ Stopped ] <-----------------------------+-----------------+
  97 //
  98 //
  99 // (*) Demuxer configs is a precondition to Configure(), but MediaCodecPlayer
 100 //     has stricter requirements and they are set before Prefetch().
 101 //
 102 // (**) VideoSurface is a precondition to video decoder Configure(), can be set
 103 //      any time before Configure().
 104
 105 class MediaCodecDecoder {
 106  public:
 107   // The result of MediaCodec configuration, used by MediaCodecPlayer.
 108   enum ConfigStatus {
 109     CONFIG_FAILURE = 0,
 110     CONFIG_OK,
 111     CONFIG_KEY_FRAME_REQUIRED,
 112   };
 113
 114   // The decoder reports current playback time to the MediaCodecPlayer.
 115   // For audio, the parameters designate the beginning and end of a time
 116   // interval. The beginning is the estimated time that is playing right now.
 117   // The end is the playback time of the last buffered data. During normal
 118   // playback the subsequent intervals overlap.
 119   // For video both values are PTS of the corresponding frame, i.e. the interval
 120   // has zero width.
 121   typedef base::Callback<void(base::TimeDelta, base::TimeDelta)>
 122       SetTimeCallback;
 123
 124   // MediaCodecDecoder constructor.
 125   // Parameters:
 126   //   media_task_runner:
 127   //     A task runner for the controlling thread. All public methods should be
 128   //     called on this thread, and callbacks are delivered on this thread.
 129   //     The MediaCodecPlayer uses a dedicated (Media) thread for this.
 130   //   external_request_data_cb:
 131   //     Called periodically as the amount of internally stored data decreases.
 132   //     The receiver should call OnDemuxerDataAvailable() with more data.
 133   //   starvation_cb:
 134   //     Called when starvation is detected. The decoder state does not change.
 135   //     The player is supposed to stop and then prefetch the decoder.
 136   //   stop_done_cb:
 137   //     Called when async stop request is completed.
 138   //   error_cb:
 139   //     Called when a MediaCodec error occurred. If this happens, a player has
 140   //     to either call ReleaseDecoderResources() or destroy the decoder object.
 141   //   decoder_thread_name:
 142   //     The thread name to be passed to decoder thread constructor.
 143   MediaCodecDecoder(
 144       const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
 145       const base::Closure& external_request_data_cb,
 146       const base::Closure& starvation_cb,
 147       const base::Closure& stop_done_cb,
 148       const base::Closure& error_cb,
 149       const char* decoder_thread_name);
 150   virtual ~MediaCodecDecoder();
 151
 152   virtual const char* class_name() const;
 153
 154   // MediaCodecDecoder exists through the whole lifetime of the player
 155   // to support dynamic addition and removal of the streams.
 156   // This method returns true if the current stream (audio or video)
 157   // is currently active.
 158   virtual bool HasStream() const = 0;
 159
 160   // Stores configuration for the use of upcoming Configure()
 161   virtual void SetDemuxerConfigs(const DemuxerConfigs& configs) = 0;
 162
 163   // Stops decoder thread, releases the MediaCodecBridge and other resources.
 164   virtual void ReleaseDecoderResources();
 165
 166   // Flushes the MediaCodec, after that resets the AccessUnitQueue and blocks
 167   // the input. Decoder thread should not be running.
 168   virtual void Flush();
 169
 170   // Releases MediaCodecBridge.
 171   void ReleaseMediaCodec();
 172
 173   // Returns corresponding conditions.
 174   bool IsPrefetchingOrPlaying() const;
 175   bool IsStopped() const;
 176   bool IsCompleted() const;
 177
 178   base::android::ScopedJavaLocalRef<jobject> GetMediaCrypto();
 179
 180   // Starts prefetching: accumulates enough data in AccessUnitQueue.
 181   // Decoder thread is not running.
 182   void Prefetch(const base::Closure& prefetch_done_cb);
 183
 184   // Configures MediaCodec.
 185   ConfigStatus Configure();
 186
 187   // Starts the decoder thread and resumes the playback.
 188   bool Start(base::TimeDelta current_time);
 189
 190   // Stops the playback process synchronously. This method stops the decoder
 191   // thread synchronously, and then releases all MediaCodec buffers.
 192   void SyncStop();
 193
 194   // Requests to stop the playback and returns.
 195   // Decoder will stop asynchronously after all the dequeued output buffers
 196   // are rendered.
 197   void RequestToStop();
 198
 199   // Notification posted when asynchronous stop is done or playback completed.
 200   void OnLastFrameRendered(bool completed);
 201
 202   // Puts the incoming data into AccessUnitQueue.
 203   void OnDemuxerDataAvailable(const DemuxerData& data);
 204
 205  protected:
 206   // Returns true if the new DemuxerConfigs requires MediaCodec
 207   // reconfiguration.
 208   virtual bool IsCodecReconfigureNeeded(const DemuxerConfigs& curr,
 209                                         const DemuxerConfigs& next) const = 0;
 210
 211   // Does the part of MediaCodecBridge configuration that is specific
 212   // to audio or video.
 213   virtual ConfigStatus ConfigureInternal() = 0;
 214
 215   // Associates PTS with device time so we can calculate delays.
 216   // We use delays for video decoder only.
 217   virtual void SynchronizePTSWithTime(base::TimeDelta current_time) {}
 218
 219   // Processes the change of the output format, varies by stream.
 220   virtual void OnOutputFormatChanged() = 0;
 221
 222   // Renders the decoded frame and releases output buffer, or posts
 223   // a delayed task to do it at a later time,
 224   virtual void Render(int buffer_index,
 225                       size_t size,
 226                       bool render_output,
 227                       base::TimeDelta pts,
 228                       bool eos_encountered) = 0;
 229
 230   // Returns the number of delayed task (we might have them for video).
 231   virtual int NumDelayedRenderTasks() const;
 232
 233   // Releases output buffers that are dequeued and not released yet
 234   // because their rendering is delayed (video).
 235   virtual void ReleaseDelayedBuffers() {}
 236
 237 #ifndef NDEBUG
 238   // For video, checks that access unit is the key frame or stand-alone EOS.
 239   virtual void VerifyUnitIsKeyFrame(const AccessUnit* unit) const {}
 240 #endif
 241
 242   // Helper methods.
 243
 244   // Notifies the decoder if the frame is the last one.
 245   void CheckLastFrame(bool eos_encountered, bool has_delayed_tasks);
 246
 247   // Protected data.
 248
 249   // Object for posting tasks on Media thread.
 250   scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
 251
 252   // Controls Android MediaCodec
 253   scoped_ptr<MediaCodecBridge> media_codec_bridge_;
 254
 255   // We call MediaCodecBridge on this thread for both
 256   // input and output buffers.
 257   base::Thread decoder_thread_;
 258
 259   // The queue of access units.
 260   AccessUnitQueue au_queue_;
 261
 262  private:
 263   enum DecoderState {
 264     kStopped = 0,
 265     kPrefetching,
 266     kPrefetched,
 267     kRunning,
 268     kStopping,
 269     kError,
 270   };
 271
 272   // Helper method that processes an error from MediaCodec.
 273   void OnCodecError();
 274
 275   // Requests data. Ensures there is no more than one request at a time.
 276   void RequestData();
 277
 278   // Prefetching callback that is posted to Media thread
 279   // in the kPrefetching state.
 280   void PrefetchNextChunk();
 281
 282   // The callback to do actual playback. Posted to Decoder thread
 283   // in the kRunning state.
 284   void ProcessNextFrame();
 285
 286   // Helper method for ProcessNextFrame.
 287   // Pushes one input buffer to the MediaCodec if the codec can accept it.
 288   // Returns false if there was MediaCodec error.
 289   bool EnqueueInputBuffer();
 290
 291   // Helper method for ProcessNextFrame.
 292   // Pulls all currently available output frames and renders them.
 293   // Returns false if there was MediaCodec error.
 294   bool DepleteOutputBufferQueue(bool* eos_encountered);
 295
 296   DecoderState GetState() const;
 297   void SetState(DecoderState state);
 298   const char* AsString(DecoderState state);
 299
 300   // Private Data.
 301
 302   // External data request callback that is passed to decoder.
 303   base::Closure external_request_data_cb_;
 304
 305   // These notifications are called on corresponding conditions.
 306   base::Closure prefetch_done_cb_;
 307   base::Closure starvation_cb_;
 308   base::Closure stop_done_cb_;
 309   base::Closure error_cb_;
 310
 311   // Data request callback that is posted by decoder internally.
 312   base::Closure request_data_cb_;
 313
 314   // Callback used to post OnCodecError method.
 315   base::Closure internal_error_cb_;
 316
 317   // Internal state.
 318   DecoderState state_;
 319   mutable base::Lock state_lock_;
 320
 321   // Flag is set when the EOS is enqueued into MediaCodec. Reset by Flush.
 322   bool eos_enqueued_;
 323
 324   // Flag is set when the EOS is received in MediaCodec output. Reset by Flush.
 325   bool completed_;
 326
 327   // Flag to ensure we post last frame notification once.
 328   bool last_frame_posted_;
 329
 330   // Indicates whether the data request is in progress.
 331   bool is_data_request_in_progress_;
 332
 333   // Indicates whether the incoming data should be ignored.
 334   bool is_incoming_data_invalid_;
 335
 336 #ifndef NDEBUG
 337   // When set, we check that the following video frame is the key frame.
 338   bool verify_next_frame_is_key_;
 339 #endif
 340
 341   // NOTE: Weak pointers must be invalidated before all other member variables.
 342   base::WeakPtrFactory<MediaCodecDecoder> weak_factory_;
 343
 344   DISALLOW_COPY_AND_ASSIGN(MediaCodecDecoder);
 345 };
 346
 347 }  // namespace media
 348
 349 #endif  // MEDIA_BASE_ANDROID_MEDIA_CODEC_DECODER_H_