1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/base/android/media_codec_decoder.h"
8 #include "base/bind_helpers.h"
9 #include "base/callback_helpers.h"
10 #include "base/logging.h"
11 #include "media/base/android/media_codec_bridge.h"
17 // Stop requesting new data in the kPrefetching state when the queue size
18 // reaches this limit.
19 const int kPrefetchLimit
= 8;
21 // Request new data in the kRunning state if the queue size is less than this.
22 const int kPlaybackLowLimit
= 4;
24 // Posting delay of the next frame processing, in milliseconds
25 const int kNextFrameDelay
= 1;
27 // Timeout for dequeuing an input buffer from MediaCodec in milliseconds.
28 const int kInputBufferTimeout
= 20;
30 // Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
31 const int kOutputBufferTimeout
= 20;
34 MediaCodecDecoder::MediaCodecDecoder(
35 const scoped_refptr
<base::SingleThreadTaskRunner
>& media_task_runner
,
36 const base::Closure
& external_request_data_cb
,
37 const base::Closure
& starvation_cb
,
38 const base::Closure
& stop_done_cb
,
39 const base::Closure
& error_cb
,
40 const char* decoder_thread_name
)
41 : media_task_runner_(media_task_runner
),
42 decoder_thread_(decoder_thread_name
),
43 needs_reconfigure_(false),
44 external_request_data_cb_(external_request_data_cb
),
45 starvation_cb_(starvation_cb
),
46 stop_done_cb_(stop_done_cb
),
51 last_frame_posted_(false),
52 is_data_request_in_progress_(false),
53 is_incoming_data_invalid_(false),
55 verify_next_frame_is_key_(false),
58 DCHECK(media_task_runner_
->BelongsToCurrentThread());
60 DVLOG(1) << "Decoder::Decoder() " << decoder_thread_name
;
63 base::Bind(&MediaCodecDecoder::OnCodecError
, weak_factory_
.GetWeakPtr());
65 base::Bind(&MediaCodecDecoder::RequestData
, weak_factory_
.GetWeakPtr());
68 MediaCodecDecoder::~MediaCodecDecoder() {
69 DCHECK(media_task_runner_
->BelongsToCurrentThread());
71 DVLOG(1) << "Decoder::~Decoder()";
73 // NB: ReleaseDecoderResources() is virtual
74 ReleaseDecoderResources();
77 const char* MediaCodecDecoder::class_name() const {
81 void MediaCodecDecoder::ReleaseDecoderResources() {
82 DCHECK(media_task_runner_
->BelongsToCurrentThread());
84 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
86 // Set [kInEmergencyStop| state to block already posted ProcessNextFrame().
87 SetState(kInEmergencyStop
);
89 decoder_thread_
.Stop(); // synchronous
92 media_codec_bridge_
.reset();
95 void MediaCodecDecoder::Flush() {
96 DCHECK(media_task_runner_
->BelongsToCurrentThread());
98 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
100 DCHECK_EQ(GetState(), kStopped
);
102 // Flush() is a part of the Seek request. Whenever we request a seek we need
103 // to invalidate the current data request.
104 if (is_data_request_in_progress_
)
105 is_incoming_data_invalid_
= true;
107 eos_enqueued_
= false;
112 // We check and reset |verify_next_frame_is_key_| on Decoder thread.
113 // This DCHECK ensures we won't need to lock this variable.
114 DCHECK(!decoder_thread_
.IsRunning());
116 // For video the first frame after flush must be key frame.
117 verify_next_frame_is_key_
= true;
120 if (media_codec_bridge_
) {
121 // MediaCodecBridge::Reset() performs MediaCodecBridge.flush()
122 MediaCodecStatus flush_status
= media_codec_bridge_
->Reset();
123 if (flush_status
!= MEDIA_CODEC_OK
) {
124 DVLOG(0) << class_name() << "::" << __FUNCTION__
125 << "MediaCodecBridge::Reset() failed";
126 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
131 void MediaCodecDecoder::ReleaseMediaCodec() {
132 DCHECK(media_task_runner_
->BelongsToCurrentThread());
134 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
136 media_codec_bridge_
.reset();
139 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
140 DCHECK(media_task_runner_
->BelongsToCurrentThread());
142 base::AutoLock
lock(state_lock_
);
143 return state_
== kPrefetching
|| state_
== kRunning
;
146 bool MediaCodecDecoder::IsStopped() const {
147 DCHECK(media_task_runner_
->BelongsToCurrentThread());
149 return GetState() == kStopped
;
152 bool MediaCodecDecoder::IsCompleted() const {
153 DCHECK(media_task_runner_
->BelongsToCurrentThread());
158 base::android::ScopedJavaLocalRef
<jobject
> MediaCodecDecoder::GetMediaCrypto() {
159 base::android::ScopedJavaLocalRef
<jobject
> media_crypto
;
161 // TODO(timav): implement DRM.
162 // drm_bridge_ is not implemented
164 // media_crypto = drm_bridge_->GetMediaCrypto();
168 void MediaCodecDecoder::Prefetch(const base::Closure
& prefetch_done_cb
) {
169 DCHECK(media_task_runner_
->BelongsToCurrentThread());
171 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
173 DCHECK(GetState() == kStopped
);
175 prefetch_done_cb_
= prefetch_done_cb
;
177 SetState(kPrefetching
);
181 MediaCodecDecoder::ConfigStatus
MediaCodecDecoder::Configure() {
182 DCHECK(media_task_runner_
->BelongsToCurrentThread());
184 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
186 if (GetState() == kError
) {
187 DVLOG(0) << class_name() << "::" << __FUNCTION__
<< ": wrong state kError";
188 return kConfigFailure
;
191 if (needs_reconfigure_
) {
192 DVLOG(1) << class_name() << "::" << __FUNCTION__
193 << ": needs reconfigure, deleting MediaCodec";
194 needs_reconfigure_
= false;
195 media_codec_bridge_
.reset();
197 // No need to release these buffers since the MediaCodec is deleted, just
198 // remove their indexes from |delayed_buffers_|.
200 ClearDelayedBuffers(false);
203 MediaCodecDecoder::ConfigStatus result
;
204 if (media_codec_bridge_
) {
205 DVLOG(1) << class_name() << "::" << __FUNCTION__
206 << ": reconfiguration is not required, ignoring";
209 result
= ConfigureInternal();
212 // We check and reset |verify_next_frame_is_key_| on Decoder thread.
213 // This DCHECK ensures we won't need to lock this variable.
214 DCHECK(!decoder_thread_
.IsRunning());
216 // For video the first frame after reconfiguration must be key frame.
217 if (result
== kConfigOk
)
218 verify_next_frame_is_key_
= true;
225 bool MediaCodecDecoder::Start(base::TimeDelta current_time
) {
226 DCHECK(media_task_runner_
->BelongsToCurrentThread());
228 DVLOG(1) << class_name() << "::" << __FUNCTION__
229 << " current_time:" << current_time
;
231 DecoderState state
= GetState();
232 if (state
== kRunning
) {
233 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": already started";
234 return true; // already started
237 if (state
!= kPrefetched
) {
238 DVLOG(0) << class_name() << "::" << __FUNCTION__
<< ": wrong state "
239 << AsString(state
) << ", ignoring";
243 if (!media_codec_bridge_
) {
244 DVLOG(0) << class_name() << "::" << __FUNCTION__
245 << ": not configured, ignoring";
249 DCHECK(!decoder_thread_
.IsRunning());
251 // We only synchronize video stream.
252 // When audio is present, the |current_time| is audio time.
253 SynchronizePTSWithTime(current_time
);
255 last_frame_posted_
= false;
257 // Start the decoder thread
258 if (!decoder_thread_
.Start()) {
259 DVLOG(1) << class_name() << "::" << __FUNCTION__
260 << ": cannot start decoder thread";
266 decoder_thread_
.task_runner()->PostTask(
268 base::Bind(&MediaCodecDecoder::ProcessNextFrame
, base::Unretained(this)));
273 void MediaCodecDecoder::SyncStop() {
274 DCHECK(media_task_runner_
->BelongsToCurrentThread());
276 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
278 if (GetState() == kError
) {
279 DVLOG(0) << class_name() << "::" << __FUNCTION__
280 << ": wrong state kError, ignoring";
284 // After this method returns, decoder thread will not be running.
286 // Set [kInEmergencyStop| state to block already posted ProcessNextFrame().
287 SetState(kInEmergencyStop
);
289 decoder_thread_
.Stop(); // synchronous
293 ClearDelayedBuffers(true); // release prior to clearing |delayed_buffers_|.
296 void MediaCodecDecoder::RequestToStop() {
297 DCHECK(media_task_runner_
->BelongsToCurrentThread());
299 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
301 DecoderState state
= GetState();
304 DVLOG(0) << class_name() << "::" << __FUNCTION__
305 << ": wrong state kError, ignoring";
315 // There is nothing to wait for, we can sent nofigication right away.
316 DCHECK(!decoder_thread_
.IsRunning());
318 media_task_runner_
->PostTask(FROM_HERE
, stop_done_cb_
);
326 void MediaCodecDecoder::OnLastFrameRendered(bool completed
) {
327 DCHECK(media_task_runner_
->BelongsToCurrentThread());
329 DVLOG(1) << class_name() << "::" << __FUNCTION__
330 << " completed:" << completed
;
332 decoder_thread_
.Stop(); // synchronous
335 completed_
= completed
;
337 media_task_runner_
->PostTask(FROM_HERE
, stop_done_cb_
);
340 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData
& data
) {
341 DCHECK(media_task_runner_
->BelongsToCurrentThread());
343 // If |data| contains an aborted data, the last AU will have kAborted status.
345 !data
.access_units
.empty() &&
346 data
.access_units
.back().status
== DemuxerStream::kAborted
;
349 const char* explain_if_skipped
=
350 is_incoming_data_invalid_
? " skipped as invalid"
351 : (aborted_data
? " skipped as aborted" : "");
353 for (const auto& unit
: data
.access_units
)
354 DVLOG(2) << class_name() << "::" << __FUNCTION__
<< explain_if_skipped
356 for (const auto& configs
: data
.demuxer_configs
)
357 DVLOG(2) << class_name() << "::" << __FUNCTION__
<< " configs: " << configs
;
360 if (!is_incoming_data_invalid_
&& !aborted_data
)
361 au_queue_
.PushBack(data
);
363 is_incoming_data_invalid_
= false;
364 is_data_request_in_progress_
= false;
366 // Do not request data if we got kAborted. There is no point to request the
367 // data after kAborted and before the OnDemuxerSeekDone.
368 if (GetState() == kPrefetching
&& !aborted_data
)
372 int MediaCodecDecoder::NumDelayedRenderTasks() const {
376 void MediaCodecDecoder::CheckLastFrame(bool eos_encountered
,
377 bool has_delayed_tasks
) {
378 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
380 bool last_frame_when_stopping
= GetState() == kStopping
&& !has_delayed_tasks
;
382 if (last_frame_when_stopping
|| eos_encountered
) {
383 media_task_runner_
->PostTask(
384 FROM_HERE
, base::Bind(&MediaCodecDecoder::OnLastFrameRendered
,
385 weak_factory_
.GetWeakPtr(), eos_encountered
));
386 last_frame_posted_
= true;
390 void MediaCodecDecoder::OnCodecError() {
391 DCHECK(media_task_runner_
->BelongsToCurrentThread());
393 // Ignore codec errors from the moment surface is changed till the
394 // |media_codec_bridge_| is deleted.
395 if (needs_reconfigure_
) {
396 DVLOG(1) << class_name() << "::" << __FUNCTION__
397 << ": needs reconfigure, ignoring";
405 void MediaCodecDecoder::RequestData() {
406 DCHECK(media_task_runner_
->BelongsToCurrentThread());
408 // Ensure one data request at a time.
409 if (!is_data_request_in_progress_
) {
410 is_data_request_in_progress_
= true;
411 external_request_data_cb_
.Run();
415 void MediaCodecDecoder::PrefetchNextChunk() {
416 DCHECK(media_task_runner_
->BelongsToCurrentThread());
418 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
420 AccessUnitQueue::Info au_info
= au_queue_
.GetInfo();
422 if (eos_enqueued_
|| au_info
.length
>= kPrefetchLimit
|| au_info
.has_eos
) {
423 // We are done prefetching
424 SetState(kPrefetched
);
425 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< " posting PrefetchDone";
426 media_task_runner_
->PostTask(FROM_HERE
,
427 base::ResetAndReturn(&prefetch_done_cb_
));
431 request_data_cb_
.Run();
434 void MediaCodecDecoder::ProcessNextFrame() {
435 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
437 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
439 DecoderState state
= GetState();
441 if (state
!= kRunning
&& state
!= kStopping
) {
442 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": not running";
446 if (state
== kStopping
) {
447 if (NumDelayedRenderTasks() == 0 && !last_frame_posted_
) {
448 DVLOG(1) << class_name() << "::" << __FUNCTION__
449 << ": kStopping, posting OnLastFrameRendered";
450 media_task_runner_
->PostTask(
451 FROM_HERE
, base::Bind(&MediaCodecDecoder::OnLastFrameRendered
,
452 weak_factory_
.GetWeakPtr(), false));
453 last_frame_posted_
= true;
456 // We can stop processing, the |au_queue_| and MediaCodec queues can freeze.
457 // We only need to let finish the delayed rendering tasks.
461 DCHECK(state
== kRunning
);
463 if (!EnqueueInputBuffer())
466 if (!DepleteOutputBufferQueue())
469 // We need a small delay if we want to stop this thread by
470 // decoder_thread_.Stop() reliably.
471 // The decoder thread message loop processes all pending
472 // (but not delayed) tasks before it can quit; without a delay
473 // the message loop might be forever processing the pendng tasks.
474 decoder_thread_
.task_runner()->PostDelayedTask(
476 base::Bind(&MediaCodecDecoder::ProcessNextFrame
, base::Unretained(this)),
477 base::TimeDelta::FromMilliseconds(kNextFrameDelay
));
480 // Returns false if we should stop decoding process. Right now
481 // it happens if we got MediaCodec error or detected starvation.
482 bool MediaCodecDecoder::EnqueueInputBuffer() {
483 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
485 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
488 DVLOG(1) << class_name() << "::" << __FUNCTION__
489 << ": eos_enqueued, returning";
490 return true; // Nothing to do
493 // Keep the number pending video frames low, ideally maintaining
494 // the same audio and video duration after stop request
495 if (NumDelayedRenderTasks() > 1) {
496 DVLOG(2) << class_name() << "::" << __FUNCTION__
<< ": # delayed buffers ("
497 << NumDelayedRenderTasks() << ") exceeds 1, returning";
498 return true; // Nothing to do
501 // Get the next frame from the queue and the queue info
503 AccessUnitQueue::Info au_info
= au_queue_
.GetInfo();
505 // Request the data from Demuxer
506 if (au_info
.length
<= kPlaybackLowLimit
&& !au_info
.has_eos
)
507 media_task_runner_
->PostTask(FROM_HERE
, request_data_cb_
);
509 // Get the next frame from the queue
511 if (!au_info
.length
) {
512 // Report starvation and return, Start() will be called again later.
513 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": starvation detected";
514 media_task_runner_
->PostTask(FROM_HERE
, starvation_cb_
);
518 if (au_info
.configs
) {
519 DVLOG(1) << class_name() << "::" << __FUNCTION__
520 << ": received new configs, not implemented";
521 // post an error for now?
522 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
526 // We are ready to enqueue the front unit.
529 if (verify_next_frame_is_key_
) {
530 verify_next_frame_is_key_
= false;
531 VerifyUnitIsKeyFrame(au_info
.front_unit
);
535 // Dequeue input buffer
537 base::TimeDelta timeout
=
538 base::TimeDelta::FromMilliseconds(kInputBufferTimeout
);
540 MediaCodecStatus status
=
541 media_codec_bridge_
->DequeueInputBuffer(timeout
, &index
);
543 DVLOG(2) << class_name() << ":: DequeueInputBuffer index:" << index
;
546 case MEDIA_CODEC_ERROR
:
547 DVLOG(0) << class_name() << "::" << __FUNCTION__
548 << ": MEDIA_CODEC_ERROR DequeueInputBuffer failed";
549 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
552 case MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER
:
560 DCHECK_EQ(status
, MEDIA_CODEC_OK
);
563 const AccessUnit
* unit
= au_info
.front_unit
;
566 if (unit
->is_end_of_stream
) {
567 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": QueueEOS";
568 media_codec_bridge_
->QueueEOS(index
);
569 eos_enqueued_
= true;
573 DVLOG(2) << class_name() << ":: QueueInputBuffer pts:" << unit
->timestamp
;
575 status
= media_codec_bridge_
->QueueInputBuffer(
576 index
, &unit
->data
[0], unit
->data
.size(), unit
->timestamp
);
578 if (status
== MEDIA_CODEC_ERROR
) {
579 DVLOG(0) << class_name() << "::" << __FUNCTION__
580 << ": MEDIA_CODEC_ERROR: QueueInputBuffer failed";
581 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
585 // Have successfully queued input buffer, go to next access unit.
590 // Returns false if there was MediaCodec error.
591 bool MediaCodecDecoder::DepleteOutputBufferQueue() {
592 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
594 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
596 int buffer_index
= 0;
600 MediaCodecStatus status
;
601 bool eos_encountered
= false;
603 base::TimeDelta timeout
=
604 base::TimeDelta::FromMilliseconds(kOutputBufferTimeout
);
606 // Extract all output buffers that are available.
607 // Usually there will be only one, but sometimes it is preceeded by
608 // MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED or MEDIA_CODEC_OUTPUT_FORMAT_CHANGED.
610 status
= media_codec_bridge_
->DequeueOutputBuffer(
611 timeout
, &buffer_index
, &offset
, &size
, &pts
, &eos_encountered
,
614 // Reset the timeout to 0 for the subsequent DequeueOutputBuffer() calls
615 // to quickly break the loop after we got all currently available buffers.
616 timeout
= base::TimeDelta::FromMilliseconds(0);
619 case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED
:
620 // Output buffers are replaced in MediaCodecBridge, nothing to do.
623 case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED
:
624 DVLOG(2) << class_name() << "::" << __FUNCTION__
625 << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
626 OnOutputFormatChanged();
630 // We got the decoded frame
631 Render(buffer_index
, size
, true, pts
, eos_encountered
);
634 case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER
:
638 case MEDIA_CODEC_ERROR
:
639 DVLOG(0) << class_name() << "::" << __FUNCTION__
640 << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
641 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
649 } while (status
!= MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER
&&
650 status
!= MEDIA_CODEC_ERROR
&& !eos_encountered
);
652 if (eos_encountered
) {
653 DVLOG(1) << class_name() << "::" << __FUNCTION__
654 << " EOS dequeued, stopping frame processing";
658 if (status
== MEDIA_CODEC_ERROR
) {
659 DVLOG(1) << class_name() << "::" << __FUNCTION__
660 << " MediaCodec error, stopping frame processing";
667 MediaCodecDecoder::DecoderState
MediaCodecDecoder::GetState() const {
668 base::AutoLock
lock(state_lock_
);
672 void MediaCodecDecoder::SetState(DecoderState state
) {
673 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< " " << AsString(state
);
675 base::AutoLock
lock(state_lock_
);
680 #define RETURN_STRING(x) \
684 const char* MediaCodecDecoder::AsString(DecoderState state
) {
686 RETURN_STRING(kStopped
);
687 RETURN_STRING(kPrefetching
);
688 RETURN_STRING(kPrefetched
);
689 RETURN_STRING(kRunning
);
690 RETURN_STRING(kStopping
);
691 RETURN_STRING(kInEmergencyStop
);
692 RETURN_STRING(kError
);
694 return "Unknown DecoderState";