1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/base/android/media_codec_decoder.h"
8 #include "base/bind_helpers.h"
9 #include "base/callback_helpers.h"
10 #include "base/logging.h"
11 #include "media/base/android/media_codec_bridge.h"
17 // Stop requesting new data in the kPrefetching state when the queue size
18 // reaches this limit.
19 const int kPrefetchLimit
= 8;
21 // Request new data in the kRunning state if the queue size is less than this.
22 const int kPlaybackLowLimit
= 4;
24 // Posting delay of the next frame processing, in milliseconds
25 const int kNextFrameDelay
= 1;
27 // Timeout for dequeuing an input buffer from MediaCodec in milliseconds.
28 const int kInputBufferTimeout
= 20;
30 // Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
31 const int kOutputBufferTimeout
= 20;
34 MediaCodecDecoder::MediaCodecDecoder(
35 const scoped_refptr
<base::SingleThreadTaskRunner
>& media_task_runner
,
36 const base::Closure
& external_request_data_cb
,
37 const base::Closure
& starvation_cb
,
38 const base::Closure
& stop_done_cb
,
39 const base::Closure
& error_cb
,
40 const char* decoder_thread_name
)
41 : media_task_runner_(media_task_runner
),
42 decoder_thread_(decoder_thread_name
),
43 external_request_data_cb_(external_request_data_cb
),
44 starvation_cb_(starvation_cb
),
45 stop_done_cb_(stop_done_cb
),
50 last_frame_posted_(false),
51 is_data_request_in_progress_(false),
52 is_incoming_data_invalid_(false),
54 verify_next_frame_is_key_(false),
57 DCHECK(media_task_runner_
->BelongsToCurrentThread());
59 DVLOG(1) << "Decoder::Decoder() " << decoder_thread_name
;
62 base::Bind(&MediaCodecDecoder::OnCodecError
, weak_factory_
.GetWeakPtr());
64 base::Bind(&MediaCodecDecoder::RequestData
, weak_factory_
.GetWeakPtr());
67 MediaCodecDecoder::~MediaCodecDecoder() {
68 DCHECK(media_task_runner_
->BelongsToCurrentThread());
70 DVLOG(1) << "Decoder::~Decoder()";
72 // NB: ReleaseDecoderResources() is virtual
73 ReleaseDecoderResources();
76 const char* MediaCodecDecoder::class_name() const {
80 void MediaCodecDecoder::ReleaseDecoderResources() {
81 DCHECK(media_task_runner_
->BelongsToCurrentThread());
83 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
85 decoder_thread_
.Stop(); // synchronous
87 media_codec_bridge_
.reset();
90 void MediaCodecDecoder::Flush() {
91 DCHECK(media_task_runner_
->BelongsToCurrentThread());
93 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
95 DCHECK_EQ(GetState(), kStopped
);
97 // Flush() is a part of the Seek request. Whenever we request a seek we need
98 // to invalidate the current data request.
99 if (is_data_request_in_progress_
)
100 is_incoming_data_invalid_
= true;
102 eos_enqueued_
= false;
107 // We check and reset |verify_next_frame_is_key_| on Decoder thread.
108 // This DCHECK ensures we won't need to lock this variable.
109 DCHECK(!decoder_thread_
.IsRunning());
111 // For video the first frame after flush must be key frame.
112 verify_next_frame_is_key_
= true;
115 if (media_codec_bridge_
) {
116 // MediaCodecBridge::Reset() performs MediaCodecBridge.flush()
117 MediaCodecStatus flush_status
= media_codec_bridge_
->Reset();
118 if (flush_status
!= MEDIA_CODEC_OK
) {
119 DVLOG(0) << class_name() << "::" << __FUNCTION__
120 << "MediaCodecBridge::Reset() failed";
121 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
126 void MediaCodecDecoder::ReleaseMediaCodec() {
127 DCHECK(media_task_runner_
->BelongsToCurrentThread());
129 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
131 media_codec_bridge_
.reset();
134 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
135 DCHECK(media_task_runner_
->BelongsToCurrentThread());
137 base::AutoLock
lock(state_lock_
);
138 return state_
== kPrefetching
|| state_
== kRunning
;
141 bool MediaCodecDecoder::IsStopped() const {
142 DCHECK(media_task_runner_
->BelongsToCurrentThread());
144 return GetState() == kStopped
;
147 bool MediaCodecDecoder::IsCompleted() const {
148 DCHECK(media_task_runner_
->BelongsToCurrentThread());
153 base::android::ScopedJavaLocalRef
<jobject
> MediaCodecDecoder::GetMediaCrypto() {
154 base::android::ScopedJavaLocalRef
<jobject
> media_crypto
;
156 // TODO(timav): implement DRM.
157 // drm_bridge_ is not implemented
159 // media_crypto = drm_bridge_->GetMediaCrypto();
163 void MediaCodecDecoder::Prefetch(const base::Closure
& prefetch_done_cb
) {
164 DCHECK(media_task_runner_
->BelongsToCurrentThread());
166 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
168 DCHECK(GetState() == kStopped
);
170 prefetch_done_cb_
= prefetch_done_cb
;
172 SetState(kPrefetching
);
176 MediaCodecDecoder::ConfigStatus
MediaCodecDecoder::Configure() {
177 DCHECK(media_task_runner_
->BelongsToCurrentThread());
179 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
181 if (GetState() == kError
) {
182 DVLOG(0) << class_name() << "::" << __FUNCTION__
<< ": wrong state kError";
183 return CONFIG_FAILURE
;
186 MediaCodecDecoder::ConfigStatus result
;
187 if (media_codec_bridge_
) {
188 DVLOG(1) << class_name() << "::" << __FUNCTION__
189 << ": reconfiguration is not required, ignoring";
192 result
= ConfigureInternal();
195 // We check and reset |verify_next_frame_is_key_| on Decoder thread.
196 // This DCHECK ensures we won't need to lock this variable.
197 DCHECK(!decoder_thread_
.IsRunning());
199 // For video the first frame after reconfiguration must be key frame.
200 if (result
== CONFIG_OK
)
201 verify_next_frame_is_key_
= true;
208 bool MediaCodecDecoder::Start(base::TimeDelta current_time
) {
209 DCHECK(media_task_runner_
->BelongsToCurrentThread());
211 DVLOG(1) << class_name() << "::" << __FUNCTION__
212 << " current_time:" << current_time
;
214 DecoderState state
= GetState();
215 if (state
== kRunning
) {
216 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": already started";
217 return true; // already started
220 if (state
!= kPrefetched
) {
221 DVLOG(0) << class_name() << "::" << __FUNCTION__
<< ": wrong state "
222 << AsString(state
) << " ignoring";
226 if (!media_codec_bridge_
) {
227 DVLOG(0) << class_name() << "::" << __FUNCTION__
228 << ": not configured, ignoring";
232 DCHECK(!decoder_thread_
.IsRunning());
234 // We only synchronize video stream.
235 // When audio is present, the |current_time| is audio time.
236 SynchronizePTSWithTime(current_time
);
238 last_frame_posted_
= false;
240 // Start the decoder thread
241 if (!decoder_thread_
.Start()) {
242 DVLOG(1) << class_name() << "::" << __FUNCTION__
243 << ": cannot start decoder thread";
249 decoder_thread_
.task_runner()->PostTask(
251 base::Bind(&MediaCodecDecoder::ProcessNextFrame
, base::Unretained(this)));
256 void MediaCodecDecoder::SyncStop() {
257 DCHECK(media_task_runner_
->BelongsToCurrentThread());
259 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
261 if (GetState() == kError
) {
262 DVLOG(0) << class_name() << "::" << __FUNCTION__
263 << ": wrong state kError, ignoring";
267 // After this method returns, decoder thread will not be running.
269 decoder_thread_
.Stop(); // synchronous
272 // Shall we move |delayed_buffers_| from VideoDecoder to Decoder class?
273 ReleaseDelayedBuffers();
276 void MediaCodecDecoder::RequestToStop() {
277 DCHECK(media_task_runner_
->BelongsToCurrentThread());
279 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
281 DecoderState state
= GetState();
284 DVLOG(0) << class_name() << "::" << __FUNCTION__
285 << ": wrong state kError, ignoring";
295 // There is nothing to wait for, we can sent nofigication right away.
296 DCHECK(!decoder_thread_
.IsRunning());
298 media_task_runner_
->PostTask(FROM_HERE
, stop_done_cb_
);
306 void MediaCodecDecoder::OnLastFrameRendered(bool completed
) {
307 DCHECK(media_task_runner_
->BelongsToCurrentThread());
309 DVLOG(1) << class_name() << "::" << __FUNCTION__
310 << " completed:" << completed
;
312 decoder_thread_
.Stop(); // synchronous
314 completed_
= completed
;
316 media_task_runner_
->PostTask(FROM_HERE
, stop_done_cb_
);
319 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData
& data
) {
320 DCHECK(media_task_runner_
->BelongsToCurrentThread());
322 // If |data| contains an aborted data, the last AU will have kAborted status.
324 !data
.access_units
.empty() &&
325 data
.access_units
.back().status
== DemuxerStream::kAborted
;
328 const char* explain_if_skipped
=
329 is_incoming_data_invalid_
? " skipped as invalid"
330 : (aborted_data
? " skipped as aborted" : "");
332 for (const auto& unit
: data
.access_units
)
333 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< explain_if_skipped
335 for (const auto& configs
: data
.demuxer_configs
)
336 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< " configs: " << configs
;
339 if (!is_incoming_data_invalid_
&& !aborted_data
)
340 au_queue_
.PushBack(data
);
342 is_incoming_data_invalid_
= false;
343 is_data_request_in_progress_
= false;
345 // Do not request data if we got kAborted. There is no point to request the
346 // data after kAborted and before the OnDemuxerSeekDone.
347 if (state_
== kPrefetching
&& !aborted_data
)
351 int MediaCodecDecoder::NumDelayedRenderTasks() const {
355 void MediaCodecDecoder::CheckLastFrame(bool eos_encountered
,
356 bool has_delayed_tasks
) {
357 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
359 bool last_frame_when_stopping
= GetState() == kStopping
&& !has_delayed_tasks
;
361 if (last_frame_when_stopping
|| eos_encountered
) {
362 media_task_runner_
->PostTask(
363 FROM_HERE
, base::Bind(&MediaCodecDecoder::OnLastFrameRendered
,
364 weak_factory_
.GetWeakPtr(), eos_encountered
));
365 last_frame_posted_
= true;
369 void MediaCodecDecoder::OnCodecError() {
370 DCHECK(media_task_runner_
->BelongsToCurrentThread());
376 void MediaCodecDecoder::RequestData() {
377 DCHECK(media_task_runner_
->BelongsToCurrentThread());
379 // Ensure one data request at a time.
380 if (!is_data_request_in_progress_
) {
381 is_data_request_in_progress_
= true;
382 external_request_data_cb_
.Run();
386 void MediaCodecDecoder::PrefetchNextChunk() {
387 DCHECK(media_task_runner_
->BelongsToCurrentThread());
389 DVLOG(1) << class_name() << "::" << __FUNCTION__
;
391 AccessUnitQueue::Info au_info
= au_queue_
.GetInfo();
393 if (eos_enqueued_
|| au_info
.length
>= kPrefetchLimit
|| au_info
.has_eos
) {
394 // We are done prefetching
395 SetState(kPrefetched
);
396 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< " posting PrefetchDone";
397 media_task_runner_
->PostTask(FROM_HERE
,
398 base::ResetAndReturn(&prefetch_done_cb_
));
402 request_data_cb_
.Run();
405 void MediaCodecDecoder::ProcessNextFrame() {
406 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
408 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
410 DecoderState state
= GetState();
412 if (state
!= kRunning
&& state
!= kStopping
) {
413 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": not running";
417 if (state
== kStopping
) {
418 if (NumDelayedRenderTasks() == 0 && !last_frame_posted_
) {
419 DVLOG(1) << class_name() << "::" << __FUNCTION__
420 << ": kStopping, posting OnLastFrameRendered";
421 media_task_runner_
->PostTask(
422 FROM_HERE
, base::Bind(&MediaCodecDecoder::OnLastFrameRendered
,
423 weak_factory_
.GetWeakPtr(), false));
424 last_frame_posted_
= true;
427 // We can stop processing, the |au_queue_| and MediaCodec queues can freeze.
428 // We only need to let finish the delayed rendering tasks.
432 DCHECK(state
== kRunning
);
434 if (!EnqueueInputBuffer())
437 bool eos_encountered
= false;
438 if (!DepleteOutputBufferQueue(&eos_encountered
))
441 if (eos_encountered
) {
442 DVLOG(1) << class_name() << "::" << __FUNCTION__
443 << " EOS dequeued, stopping frame processing";
447 // We need a small delay if we want to stop this thread by
448 // decoder_thread_.Stop() reliably.
449 // The decoder thread message loop processes all pending
450 // (but not delayed) tasks before it can quit; without a delay
451 // the message loop might be forever processing the pendng tasks.
452 decoder_thread_
.task_runner()->PostDelayedTask(
454 base::Bind(&MediaCodecDecoder::ProcessNextFrame
, base::Unretained(this)),
455 base::TimeDelta::FromMilliseconds(kNextFrameDelay
));
458 // Returns false if we should stop decoding process. Right now
459 // it happens if we got MediaCodec error or detected starvation.
460 bool MediaCodecDecoder::EnqueueInputBuffer() {
461 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
463 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
466 DVLOG(1) << class_name() << "::" << __FUNCTION__
467 << ": eos_enqueued, returning";
468 return true; // Nothing to do
471 // Keep the number pending video frames low, ideally maintaining
472 // the same audio and video duration after stop request
473 if (NumDelayedRenderTasks() > 1) {
474 DVLOG(2) << class_name() << "::" << __FUNCTION__
<< ": # delayed buffers ("
475 << NumDelayedRenderTasks() << ") exceeds 1, returning";
476 return true; // Nothing to do
479 // Get the next frame from the queue and the queue info
481 AccessUnitQueue::Info au_info
= au_queue_
.GetInfo();
483 // Request the data from Demuxer
484 if (au_info
.length
<= kPlaybackLowLimit
&& !au_info
.has_eos
)
485 media_task_runner_
->PostTask(FROM_HERE
, request_data_cb_
);
487 // Get the next frame from the queue
489 if (!au_info
.length
) {
490 // Report starvation and return, Start() will be called again later.
491 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": starvation detected";
492 media_task_runner_
->PostTask(FROM_HERE
, starvation_cb_
);
496 if (au_info
.configs
) {
497 DVLOG(1) << class_name() << "::" << __FUNCTION__
498 << ": received new configs, not implemented";
499 // post an error for now?
500 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
504 // We are ready to enqueue the front unit.
507 if (verify_next_frame_is_key_
) {
508 verify_next_frame_is_key_
= false;
509 VerifyUnitIsKeyFrame(au_info
.front_unit
);
513 // Dequeue input buffer
515 base::TimeDelta timeout
=
516 base::TimeDelta::FromMilliseconds(kInputBufferTimeout
);
518 MediaCodecStatus status
=
519 media_codec_bridge_
->DequeueInputBuffer(timeout
, &index
);
521 DVLOG(2) << class_name() << ":: DequeueInputBuffer index:" << index
;
524 case MEDIA_CODEC_ERROR
:
525 DVLOG(0) << class_name() << "::" << __FUNCTION__
526 << ": MEDIA_CODEC_ERROR DequeueInputBuffer failed";
527 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
530 case MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER
:
538 DCHECK_EQ(status
, MEDIA_CODEC_OK
);
541 const AccessUnit
* unit
= au_info
.front_unit
;
544 if (unit
->is_end_of_stream
) {
545 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< ": QueueEOS";
546 media_codec_bridge_
->QueueEOS(index
);
547 eos_enqueued_
= true;
551 DVLOG(2) << class_name() << ":: QueueInputBuffer pts:" << unit
->timestamp
;
553 status
= media_codec_bridge_
->QueueInputBuffer(
554 index
, &unit
->data
[0], unit
->data
.size(), unit
->timestamp
);
556 if (status
== MEDIA_CODEC_ERROR
) {
557 DVLOG(0) << class_name() << "::" << __FUNCTION__
558 << ": MEDIA_CODEC_ERROR: QueueInputBuffer failed";
559 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
563 // Have successfully queued input buffer, go to next access unit.
568 // Returns false if there was MediaCodec error.
569 bool MediaCodecDecoder::DepleteOutputBufferQueue(bool* eos_encountered
) {
570 DCHECK(decoder_thread_
.task_runner()->BelongsToCurrentThread());
572 DVLOG(2) << class_name() << "::" << __FUNCTION__
;
574 int buffer_index
= 0;
578 MediaCodecStatus status
;
580 base::TimeDelta timeout
=
581 base::TimeDelta::FromMilliseconds(kOutputBufferTimeout
);
583 // Extract all output buffers that are available.
584 // Usually there will be only one, but sometimes it is preceeded by
585 // MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED or MEDIA_CODEC_OUTPUT_FORMAT_CHANGED.
587 status
= media_codec_bridge_
->DequeueOutputBuffer(
588 timeout
, &buffer_index
, &offset
, &size
, &pts
, eos_encountered
, nullptr);
590 // Reset the timeout to 0 for the subsequent DequeueOutputBuffer() calls
591 // to quickly break the loop after we got all currently available buffers.
592 timeout
= base::TimeDelta::FromMilliseconds(0);
595 case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED
:
596 // Output buffers are replaced in MediaCodecBridge, nothing to do.
599 case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED
:
600 DVLOG(2) << class_name() << "::" << __FUNCTION__
601 << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
602 OnOutputFormatChanged();
606 // We got the decoded frame
607 Render(buffer_index
, size
, true, pts
, *eos_encountered
);
610 case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER
:
614 case MEDIA_CODEC_ERROR
:
615 DVLOG(0) << class_name() << "::" << __FUNCTION__
616 << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
617 media_task_runner_
->PostTask(FROM_HERE
, internal_error_cb_
);
625 } while (status
!= MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER
&&
626 status
!= MEDIA_CODEC_ERROR
&& !*eos_encountered
);
628 return status
!= MEDIA_CODEC_ERROR
;
631 MediaCodecDecoder::DecoderState
MediaCodecDecoder::GetState() const {
632 base::AutoLock
lock(state_lock_
);
636 void MediaCodecDecoder::SetState(DecoderState state
) {
637 DVLOG(1) << class_name() << "::" << __FUNCTION__
<< " " << state
;
639 base::AutoLock
lock(state_lock_
);
644 #define RETURN_STRING(x) \
648 const char* MediaCodecDecoder::AsString(DecoderState state
) {
650 RETURN_STRING(kStopped
);
651 RETURN_STRING(kPrefetching
);
652 RETURN_STRING(kPrefetched
);
653 RETURN_STRING(kRunning
);
654 RETURN_STRING(kStopping
);
655 RETURN_STRING(kError
);
657 return "Unknown DecoderState";