1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/base/android/media_codec_decoder.h"
8 #include "base/bind_helpers.h"
9 #include "base/callback_helpers.h"
10 #include "base/logging.h"
11 #include "media/base/android/media_codec_bridge.h"
17 // Stop requesting new data in the kPrefetching state when the queue size
18 // reaches this limit.
19 const int kPrefetchLimit
= 8;
21 // Request new data in the kRunning state if the queue size is less than this.
22 const int kPlaybackLowLimit
= 4;
24 // Posting delay of the next frame processing, in milliseconds
25 const int kNextFrameDelay
= 1;
27 // Timeout for dequeuing an input buffer from MediaCodec in milliseconds.
28 const int kInputBufferTimeout
= 20;
30 // Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
31 const int kOutputBufferTimeout
= 20;
34 MediaCodecDecoder::MediaCodecDecoder(
35 const scoped_refptr
<base::SingleThreadTaskRunner
>& media_task_runner
,
36 const base::Closure
& external_request_data_cb
,
37 const base::Closure
& starvation_cb
,
38 const base::Closure
& stop_done_cb
,
39 const base::Closure
& error_cb
,
40 const char* decoder_thread_name
)
41 : media_task_runner_(media_task_runner
),
42 decoder_thread_(decoder_thread_name
),
43 needs_reconfigure_(false),
44 external_request_data_cb_(external_request_data_cb
),
45 starvation_cb_(starvation_cb
),
46 stop_done_cb_(stop_done_cb
),
51 last_frame_posted_(false),
52 is_data_request_in_progress_(false),
53 is_incoming_data_invalid_(false),
55 verify_next_frame_is_key_(false),
58 DCHECK(media_task_runner_
->BelongsToCurrentThread());
60 DVLOG(1) << "Decoder::Decoder() " << decoder_thread_name
;
63 base::Bind(&MediaCodecDecoder::OnCodecError
, weak_factory_
.GetWeakPtr());
65 base::Bind(&MediaCodecDecoder::RequestData
, weak_factory_
.GetWeakPtr());
68 MediaCodecDecoder::~MediaCodecDecoder() {
69 DCHECK(media_task_runner_
->BelongsToCurrentThread());
71 DVLOG(1) << "Decoder::~Decoder()";
73 // NB: ReleaseDecoderResources() is virtual
74 ReleaseDecoderResources();
77 const char* MediaCodecDecoder::class_name() const {
81 void MediaCodecDecoder::ReleaseDecoderResources() {
82 DCHECK(media_task_runner_
->BelongsToCurrentThread());
84 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
86 // Set [kInEmergencyStop| state to block already posted ProcessNextFrame().
87 SetState(kInEmergencyStop
);
89 decoder_thread_
.Stop(); // synchronous
92 media_codec_bridge_
.reset();
95 void MediaCodecDecoder::Flush() {
96 DCHECK(media_task_runner_
->BelongsToCurrentThread());
98 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
100 DCHECK_EQ(GetState(), kStopped
);
102 // Flush() is a part of the Seek request. Whenever we request a seek we need
103 // to invalidate the current data request.
104 if (is_data_request_in_progress_
)
105 is_incoming_data_invalid_
= true;
107 eos_enqueued_
= false;
112 // We check and reset |verify_next_frame_is_key_| on Decoder thread.
113 // This DCHECK ensures we won't need to lock this variable.
114 DCHECK(!decoder_thread_
.IsRunning());
116 // For video the first frame after flush must be key frame.
117 verify_next_frame_is_key_
= true;
120 if (media_codec_bridge_
) {
121 // MediaCodecBridge::Reset() performs MediaCodecBridge.flush()
122 MediaCodecStatus flush_status
= media_codec_bridge_
->Reset();
123 if (flush_status
!= MEDIA_CODEC_OK
) {
124 DVLOG(0) << class_name() << "::" << __FUNCTION__
125 << "MediaCodecBridge::Reset() failed";
126 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
131 void MediaCodecDecoder::ReleaseMediaCodec() {
132 DCHECK(media_task_runner_
->BelongsToCurrentThread());
134 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
136 media_codec_bridge_
.reset();
139 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
140 DCHECK(media_task_runner_
->BelongsToCurrentThread());
142 base::AutoLock
lock(state_lock_
);
143 return state_
== kPrefetching
|| state_
== kRunning
;
146 bool MediaCodecDecoder::IsStopped() const {
147 DCHECK(media_task_runner_
->BelongsToCurrentThread());
149 return GetState() == kStopped
;
152 bool MediaCodecDecoder::IsCompleted() const {
153 DCHECK(media_task_runner_
->BelongsToCurrentThread());
158 base::android::ScopedJavaLocalRef
<jobject
> MediaCodecDecoder::GetMediaCrypto() {
159 base::android::ScopedJavaLocalRef
<jobject
> media_crypto
;
161 // TODO(timav): implement DRM.
162 // drm_bridge_ is not implemented
164 // media_crypto = drm_bridge_->GetMediaCrypto();
168 void MediaCodecDecoder::Prefetch(const base::Closure
& prefetch_done_cb
) {
169 DCHECK(media_task_runner_
->BelongsToCurrentThread());
171 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
173 DCHECK(GetState() == kStopped
);
175 prefetch_done_cb_
= prefetch_done_cb
;
177 SetState(kPrefetching
);
181 MediaCodecDecoder::ConfigStatus
MediaCodecDecoder::Configure() {
182 DCHECK(media_task_runner_
->BelongsToCurrentThread());
184 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
186 if (GetState() == kError
) {
187 DVLOG(0) << class_name() << "::" << __FUNCTION__
<< ": wrong state kError";
188 return kConfigFailure
;
191 if (needs_reconfigure_
) {
192 DVLOG(1) << class_name() << "::" << __FUNCTION__
193 << ": needs reconfigure, deleting MediaCodec";
194 needs_reconfigure_
= false;
195 media_codec_bridge_
.reset();
197 // No need to release these buffers since the MediaCodec is deleted, just
198 // remove their indexes from |delayed_buffers_|.
200 ClearDelayedBuffers(false);
203 MediaCodecDecoder::ConfigStatus result
;
204 if (media_codec_bridge_
) {
205 DVLOG(1) << class_name() << "::" << __FUNCTION__
206 << ": reconfiguration is not required, ignoring";
209 result
= ConfigureInternal();
212 // We check and reset |verify_next_frame_is_key_| on Decoder thread.
213 // This DCHECK ensures we won't need to lock this variable.
214 DCHECK(!decoder_thread_
.IsRunning());
216 // For video the first frame after reconfiguration must be key frame.
217 if (result
== kConfigOk
)
218 verify_next_frame_is_key_
= true;
225 bool MediaCodecDecoder::Start(base::TimeDelta current_time
) {
226 DCHECK(media_task_runner_
->BelongsToCurrentThread());
228 DVLOG(1) << class_name() << "::" << __FUNCTION__
229 << " current_time:" << current_time
;
231 DecoderState state
= GetState();
232 if (state
== kRunning
) {
233 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": already started";
234 return true; // already started
237 if (state
!= kPrefetched
) {
238 DVLOG(0) << class_name() << "::" << __FUNCTION__
<< ": wrong state "
239 << AsString(state
) << ", ignoring";
243 if (!media_codec_bridge_
) {
244 DVLOG(0) << class_name() << "::" << __FUNCTION__
245 << ": not configured, ignoring";
249 DCHECK(!decoder_thread_
.IsRunning());
251 // We only synchronize video stream.
252 // When audio is present, the |current_time| is audio time.
253 SynchronizePTSWithTime(current_time
);
255 last_frame_posted_
= false;
257 // Start the decoder thread
258 if (!decoder_thread_
.Start()) {
259 DVLOG(0) << class_name() << "::" << __FUNCTION__
260 << ": cannot start decoder thread";
264 DVLOG(0) << class_name() << "::" << __FUNCTION__
<< " decoder thread started";
268 decoder_thread_
.task_runner()->PostTask(
270 base::Bind(&MediaCodecDecoder::ProcessNextFrame
, base::Unretained(this)));
275 void MediaCodecDecoder::SyncStop() {
276 DCHECK(media_task_runner_
->BelongsToCurrentThread());
278 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
280 if (GetState() == kError
) {
281 DVLOG(0) << class_name() << "::" << __FUNCTION__
282 << ": wrong state kError, ignoring";
286 // After this method returns, decoder thread will not be running.
288 // Set [kInEmergencyStop| state to block already posted ProcessNextFrame().
289 SetState(kInEmergencyStop
);
291 decoder_thread_
.Stop(); // synchronous
295 ClearDelayedBuffers(true); // release prior to clearing |delayed_buffers_|.
298 void MediaCodecDecoder::RequestToStop() {
299 DCHECK(media_task_runner_
->BelongsToCurrentThread());
301 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
303 DecoderState state
= GetState();
306 DVLOG(0) << class_name() << "::" << __FUNCTION__
307 << ": wrong state kError, ignoring";
317 // There is nothing to wait for, we can sent nofigication right away.
318 DCHECK(!decoder_thread_
.IsRunning());
320 media_task_runner_
->PostTask(FROM_HERE
, stop_done_cb_
);
328 void MediaCodecDecoder::OnLastFrameRendered(bool completed
) {
329 DCHECK(media_task_runner_
->BelongsToCurrentThread());
331 DVLOG(0) << class_name() << "::" << __FUNCTION__
332 << " completed:" << completed
;
334 decoder_thread_
.Stop(); // synchronous
337 completed_
= completed
;
339 media_task_runner_
->PostTask(FROM_HERE
, stop_done_cb_
);
342 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData
& data
) {
343 DCHECK(media_task_runner_
->BelongsToCurrentThread());
345 // If |data| contains an aborted data, the last AU will have kAborted status.
347 !data
.access_units
.empty() &&
348 data
.access_units
.back().status
== DemuxerStream::kAborted
;
351 const char* explain_if_skipped
=
352 is_incoming_data_invalid_
? " skipped as invalid"
353 : (aborted_data
? " skipped as aborted" : "");
355 for (const auto& unit
: data
.access_units
)
356 DVLOG(2) << class_name() << "::" << __FUNCTION__
<< explain_if_skipped
358 for (const auto& configs
: data
.demuxer_configs
)
359 DVLOG(2) << class_name() << "::" << __FUNCTION__
<< " configs: " << configs
;
362 if (!is_incoming_data_invalid_
&& !aborted_data
)
363 au_queue_
.PushBack(data
);
365 is_incoming_data_invalid_
= false;
366 is_data_request_in_progress_
= false;
368 // Do not request data if we got kAborted. There is no point to request the
369 // data after kAborted and before the OnDemuxerSeekDone.
370 if (GetState() == kPrefetching
&& !aborted_data
)
374 int MediaCodecDecoder::NumDelayedRenderTasks() const {
378 void MediaCodecDecoder::CheckLastFrame(bool eos_encountered
,
379 bool has_delayed_tasks
) {
380 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
382 bool last_frame_when_stopping
= GetState() == kStopping
&& !has_delayed_tasks
;
384 if (last_frame_when_stopping
|| eos_encountered
) {
385 media_task_runner_
->PostTask(
386 FROM_HERE
, base::Bind(&MediaCodecDecoder::OnLastFrameRendered
,
387 weak_factory_
.GetWeakPtr(), eos_encountered
));
388 last_frame_posted_
= true;
392 void MediaCodecDecoder::OnCodecError() {
393 DCHECK(media_task_runner_
->BelongsToCurrentThread());
395 // Ignore codec errors from the moment surface is changed till the
396 // |media_codec_bridge_| is deleted.
397 if (needs_reconfigure_
) {
398 DVLOG(1) << class_name() << "::" << __FUNCTION__
399 << ": needs reconfigure, ignoring";
407 void MediaCodecDecoder::RequestData() {
408 DCHECK(media_task_runner_
->BelongsToCurrentThread());
410 // Ensure one data request at a time.
411 if (!is_data_request_in_progress_
) {
412 is_data_request_in_progress_
= true;
413 external_request_data_cb_
.Run();
417 void MediaCodecDecoder::PrefetchNextChunk() {
418 DCHECK(media_task_runner_
->BelongsToCurrentThread());
420 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
422 AccessUnitQueue::Info au_info
= au_queue_
.GetInfo();
424 if (eos_enqueued_
|| au_info
.length
>= kPrefetchLimit
|| au_info
.has_eos
) {
425 // We are done prefetching
426 SetState(kPrefetched
);
427 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< " posting PrefetchDone";
428 media_task_runner_
->PostTask(FROM_HERE
,
429 base::ResetAndReturn(&prefetch_done_cb_
));
433 request_data_cb_
.Run();
436 void MediaCodecDecoder::ProcessNextFrame() {
437 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
439 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
441 DecoderState state
= GetState();
443 if (state
!= kRunning
&& state
!= kStopping
) {
444 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": not running";
448 if (state
== kStopping
) {
449 if (NumDelayedRenderTasks() == 0 && !last_frame_posted_
) {
450 DVLOG(1) << class_name() << "::" << __FUNCTION__
451 << ": kStopping, posting OnLastFrameRendered";
452 media_task_runner_
->PostTask(
453 FROM_HERE
, base::Bind(&MediaCodecDecoder::OnLastFrameRendered
,
454 weak_factory_
.GetWeakPtr(), false));
455 last_frame_posted_
= true;
458 // We can stop processing, the |au_queue_| and MediaCodec queues can freeze.
459 // We only need to let finish the delayed rendering tasks.
463 DCHECK(state
== kRunning
);
465 if (!EnqueueInputBuffer())
468 if (!DepleteOutputBufferQueue())
471 // We need a small delay if we want to stop this thread by
472 // decoder_thread_.Stop() reliably.
473 // The decoder thread message loop processes all pending
474 // (but not delayed) tasks before it can quit; without a delay
475 // the message loop might be forever processing the pendng tasks.
476 decoder_thread_
.task_runner()->PostDelayedTask(
478 base::Bind(&MediaCodecDecoder::ProcessNextFrame
, base::Unretained(this)),
479 base::TimeDelta::FromMilliseconds(kNextFrameDelay
));
482 // Returns false if we should stop decoding process. Right now
483 // it happens if we got MediaCodec error or detected starvation.
484 bool MediaCodecDecoder::EnqueueInputBuffer() {
485 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
487 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
490 DVLOG(1) << class_name() << "::" << __FUNCTION__
491 << ": eos_enqueued, returning";
492 return true; // Nothing to do
495 // Keep the number pending video frames low, ideally maintaining
496 // the same audio and video duration after stop request
497 if (NumDelayedRenderTasks() > 1) {
498 DVLOG(2) << class_name() << "::" << __FUNCTION__
<< ": # delayed buffers ("
499 << NumDelayedRenderTasks() << ") exceeds 1, returning";
500 return true; // Nothing to do
503 // Get the next frame from the queue and the queue info
505 AccessUnitQueue::Info au_info
= au_queue_
.GetInfo();
507 // Request the data from Demuxer
508 if (au_info
.length
<= kPlaybackLowLimit
&& !au_info
.has_eos
)
509 media_task_runner_
->PostTask(FROM_HERE
, request_data_cb_
);
511 // Get the next frame from the queue
513 if (!au_info
.length
) {
514 // Report starvation and return, Start() will be called again later.
515 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": starvation detected";
516 media_task_runner_
->PostTask(FROM_HERE
, starvation_cb_
);
520 if (au_info
.configs
) {
521 DVLOG(1) << class_name() << "::" << __FUNCTION__
522 << ": received new configs, not implemented";
523 // post an error for now?
524 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
528 // We are ready to enqueue the front unit.
531 if (verify_next_frame_is_key_
) {
532 verify_next_frame_is_key_
= false;
533 VerifyUnitIsKeyFrame(au_info
.front_unit
);
537 // Dequeue input buffer
539 base::TimeDelta timeout
=
540 base::TimeDelta::FromMilliseconds(kInputBufferTimeout
);
542 MediaCodecStatus status
=
543 media_codec_bridge_
->DequeueInputBuffer(timeout
, &index
);
545 DVLOG(2) << class_name() << ":: DequeueInputBuffer index:" << index
;
548 case MEDIA_CODEC_ERROR
:
549 DVLOG(0) << class_name() << "::" << __FUNCTION__
550 << ": MEDIA_CODEC_ERROR DequeueInputBuffer failed";
551 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
554 case MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER
:
556 << class_name() << "::" << __FUNCTION__
557 << ": DequeueInputBuffer returned MediaCodec.INFO_TRY_AGAIN_LATER.";
565 DCHECK_EQ(status
, MEDIA_CODEC_OK
);
568 const AccessUnit
* unit
= au_info
.front_unit
;
571 if (unit
->is_end_of_stream
) {
572 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": QueueEOS";
573 media_codec_bridge_
->QueueEOS(index
);
574 eos_enqueued_
= true;
578 DVLOG(2) << class_name() << ":: QueueInputBuffer pts:" << unit
->timestamp
;
580 status
= media_codec_bridge_
->QueueInputBuffer(
581 index
, &unit
->data
[0], unit
->data
.size(), unit
->timestamp
);
583 if (status
== MEDIA_CODEC_ERROR
) {
584 DVLOG(0) << class_name() << "::" << __FUNCTION__
585 << ": MEDIA_CODEC_ERROR: QueueInputBuffer failed";
586 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
590 // Have successfully queued input buffer, go to next access unit.
595 // Returns false if there was MediaCodec error.
596 bool MediaCodecDecoder::DepleteOutputBufferQueue() {
597 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
599 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
601 int buffer_index
= 0;
605 MediaCodecStatus status
;
606 bool eos_encountered
= false;
608 base::TimeDelta timeout
=
609 base::TimeDelta::FromMilliseconds(kOutputBufferTimeout
);
611 // Extract all output buffers that are available.
612 // Usually there will be only one, but sometimes it is preceeded by
613 // MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED or MEDIA_CODEC_OUTPUT_FORMAT_CHANGED.
615 status
= media_codec_bridge_
->DequeueOutputBuffer(
616 timeout
, &buffer_index
, &offset
, &size
, &pts
, &eos_encountered
,
619 // Reset the timeout to 0 for the subsequent DequeueOutputBuffer() calls
620 // to quickly break the loop after we got all currently available buffers.
621 timeout
= base::TimeDelta::FromMilliseconds(0);
624 case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED
:
625 // Output buffers are replaced in MediaCodecBridge, nothing to do.
628 case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED
:
629 DVLOG(2) << class_name() << "::" << __FUNCTION__
630 << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
631 OnOutputFormatChanged();
635 // We got the decoded frame
636 Render(buffer_index
, size
, true, pts
, eos_encountered
);
639 case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER
:
643 case MEDIA_CODEC_ERROR
:
644 DVLOG(0) << class_name() << "::" << __FUNCTION__
645 << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
646 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
654 } while (status
!= MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER
&&
655 status
!= MEDIA_CODEC_ERROR
&& !eos_encountered
);
657 if (eos_encountered
) {
658 DVLOG(1) << class_name() << "::" << __FUNCTION__
659 << " EOS dequeued, stopping frame processing";
663 if (status
== MEDIA_CODEC_ERROR
) {
664 DVLOG(0) << class_name() << "::" << __FUNCTION__
665 << " MediaCodec error, stopping frame processing";
672 MediaCodecDecoder::DecoderState
MediaCodecDecoder::GetState() const {
673 base::AutoLock
lock(state_lock_
);
677 void MediaCodecDecoder::SetState(DecoderState state
) {
678 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< " " << AsString(state
);
680 base::AutoLock
lock(state_lock_
);
685 #define RETURN_STRING(x) \
689 const char* MediaCodecDecoder::AsString(DecoderState state
) {
691 RETURN_STRING(kStopped
);
692 RETURN_STRING(kPrefetching
);
693 RETURN_STRING(kPrefetched
);
694 RETURN_STRING(kRunning
);
695 RETURN_STRING(kStopping
);
696 RETURN_STRING(kInEmergencyStop
);
697 RETURN_STRING(kError
);
699 return "Unknown DecoderState";