1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/vpx_video_decoder.h"
11 #include "base/bind.h"
12 #include "base/callback_helpers.h"
13 #include "base/command_line.h"
14 #include "base/location.h"
15 #include "base/logging.h"
16 #include "base/metrics/histogram_macros.h"
17 #include "base/single_thread_task_runner.h"
18 #include "base/stl_util.h"
19 #include "base/strings/string_number_conversions.h"
20 #include "base/sys_byteorder.h"
21 #include "base/sys_info.h"
22 #include "base/trace_event/memory_allocator_dump.h"
23 #include "base/trace_event/memory_dump_manager.h"
24 #include "base/trace_event/memory_dump_provider.h"
25 #include "base/trace_event/process_memory_dump.h"
26 #include "base/trace_event/trace_event.h"
27 #include "media/base/bind_to_current_loop.h"
28 #include "media/base/decoder_buffer.h"
29 #include "media/base/limits.h"
30 #include "media/base/media_switches.h"
31 #include "media/base/pipeline.h"
32 #include "media/base/timestamp_constants.h"
33 #include "media/base/video_util.h"
35 // Include libvpx header files.
36 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
37 // backwards compatibility for legacy applications using the library.
38 #define VPX_CODEC_DISABLE_COMPAT 1
40 #include "third_party/libvpx_new/source/libvpx/vpx/vp8dx.h"
41 #include "third_party/libvpx_new/source/libvpx/vpx/vpx_decoder.h"
42 #include "third_party/libvpx_new/source/libvpx/vpx/vpx_frame_buffer.h"
47 // Always try to use three threads for video decoding. There is little reason
48 // not to since current day CPUs tend to be multi-core and we measured
49 // performance benefits on older machines such as P4s with hyperthreading.
50 static const int kDecodeThreads
= 2;
51 static const int kMaxDecodeThreads
= 16;
53 // Returns the number of threads.
54 static int GetThreadCount(const VideoDecoderConfig
& config
) {
55 // Refer to http://crbug.com/93932 for tsan suppressions on decoding.
56 int decode_threads
= kDecodeThreads
;
58 const base::CommandLine
* cmd_line
= base::CommandLine::ForCurrentProcess();
59 std::string
threads(cmd_line
->GetSwitchValueASCII(switches::kVideoThreads
));
60 if (threads
.empty() || !base::StringToInt(threads
, &decode_threads
)) {
61 if (config
.codec() == kCodecVP9
) {
62 // For VP9 decode when using the default thread count, increase the number
63 // of decode threads to equal the maximum number of tiles possible for
64 // higher resolution streams.
65 if (config
.coded_size().width() >= 2048)
67 else if (config
.coded_size().width() >= 1024)
71 decode_threads
= std::min(decode_threads
,
72 base::SysInfo::NumberOfProcessors());
73 return decode_threads
;
76 decode_threads
= std::max(decode_threads
, 0);
77 decode_threads
= std::min(decode_threads
, kMaxDecodeThreads
);
78 return decode_threads
;
81 class VpxVideoDecoder::MemoryPool
82 : public base::RefCountedThreadSafe
<VpxVideoDecoder::MemoryPool
>,
83 public base::trace_event::MemoryDumpProvider
{
87 // Callback that will be called by libvpx when it needs a frame buffer.
89 // |user_priv| Private data passed to libvpx (pointer to memory pool).
90 // |min_size| Minimum size needed by libvpx to decompress the next frame.
91 // |fb| Pointer to the frame buffer to update.
92 // Returns 0 on success. Returns < 0 on failure.
93 static int32
GetVP9FrameBuffer(void* user_priv
, size_t min_size
,
94 vpx_codec_frame_buffer
* fb
);
96 // Callback that will be called by libvpx when the frame buffer is no longer
97 // being used by libvpx. Parameters:
98 // |user_priv| Private data passed to libvpx (pointer to memory pool).
99 // |fb| Pointer to the frame buffer that's being released.
100 static int32
ReleaseVP9FrameBuffer(void *user_priv
,
101 vpx_codec_frame_buffer
*fb
);
103 // Generates a "no_longer_needed" closure that holds a reference
105 base::Closure
CreateFrameCallback(void* fb_priv_data
);
107 bool OnMemoryDump(const base::trace_event::MemoryDumpArgs
& args
,
108 base::trace_event::ProcessMemoryDump
* pmd
) override
;
110 int NumberOfFrameBuffersInUseByDecoder() const;
111 int NumberOfFrameBuffersInUseByDecoderAndVideoFrame() const;
114 friend class base::RefCountedThreadSafe
<VpxVideoDecoder::MemoryPool
>;
115 ~MemoryPool() override
;
117 // Reference counted frame buffers used for VP9 decoding. Reference counting
118 // is done manually because both chromium and libvpx has to release this
119 // before a buffer can be re-used.
120 struct VP9FrameBuffer
{
121 VP9FrameBuffer() : ref_cnt(0) {}
122 std::vector
<uint8
> data
;
126 // Gets the next available frame buffer for use by libvpx.
127 VP9FrameBuffer
* GetFreeFrameBuffer(size_t min_size
);
129 // Method that gets called when a VideoFrame that references this pool gets
131 void OnVideoFrameDestroyed(VP9FrameBuffer
* frame_buffer
);
133 // Frame buffers to be used by libvpx for VP9 Decoding.
134 std::vector
<VP9FrameBuffer
*> frame_buffers_
;
136 // Number of VP9FrameBuffer currently in use by the decoder.
137 int in_use_by_decoder_
= 0;
138 // Number of VP9FrameBuffer currently in use by the decoder and a video frame.
139 int in_use_by_decoder_and_video_frame_
= 0;
140 DISALLOW_COPY_AND_ASSIGN(MemoryPool
);
143 VpxVideoDecoder::MemoryPool::MemoryPool() {}
145 VpxVideoDecoder::MemoryPool::~MemoryPool() {
146 STLDeleteElements(&frame_buffers_
);
149 VpxVideoDecoder::MemoryPool::VP9FrameBuffer
*
150 VpxVideoDecoder::MemoryPool::GetFreeFrameBuffer(size_t min_size
) {
151 // Check if a free frame buffer exists.
153 for (; i
< frame_buffers_
.size(); ++i
) {
154 if (frame_buffers_
[i
]->ref_cnt
== 0)
158 if (i
== frame_buffers_
.size()) {
159 // Create a new frame buffer.
160 frame_buffers_
.push_back(new VP9FrameBuffer());
163 // Resize the frame buffer if necessary.
164 if (frame_buffers_
[i
]->data
.size() < min_size
)
165 frame_buffers_
[i
]->data
.resize(min_size
);
166 return frame_buffers_
[i
];
169 int32
VpxVideoDecoder::MemoryPool::GetVP9FrameBuffer(
170 void* user_priv
, size_t min_size
, vpx_codec_frame_buffer
* fb
) {
174 VpxVideoDecoder::MemoryPool
* memory_pool
=
175 static_cast<VpxVideoDecoder::MemoryPool
*>(user_priv
);
177 VP9FrameBuffer
* fb_to_use
= memory_pool
->GetFreeFrameBuffer(min_size
);
178 if (fb_to_use
== NULL
)
181 fb
->data
= &fb_to_use
->data
[0];
182 fb
->size
= fb_to_use
->data
.size();
183 ++fb_to_use
->ref_cnt
;
184 ++memory_pool
->in_use_by_decoder_
;
186 // Set the frame buffer's private data to point at the external frame buffer.
187 fb
->priv
= static_cast<void*>(fb_to_use
);
191 int32
VpxVideoDecoder::MemoryPool::ReleaseVP9FrameBuffer(
192 void *user_priv
, vpx_codec_frame_buffer
*fb
) {
195 VP9FrameBuffer
* frame_buffer
= static_cast<VP9FrameBuffer
*>(fb
->priv
);
196 --frame_buffer
->ref_cnt
;
198 VpxVideoDecoder::MemoryPool
* memory_pool
=
199 static_cast<VpxVideoDecoder::MemoryPool
*>(user_priv
);
200 --memory_pool
->in_use_by_decoder_
;
201 if (frame_buffer
->ref_cnt
)
202 --memory_pool
->in_use_by_decoder_and_video_frame_
;
206 base::Closure
VpxVideoDecoder::MemoryPool::CreateFrameCallback(
207 void* fb_priv_data
) {
208 VP9FrameBuffer
* frame_buffer
= static_cast<VP9FrameBuffer
*>(fb_priv_data
);
209 ++frame_buffer
->ref_cnt
;
210 if (frame_buffer
->ref_cnt
> 1)
211 ++in_use_by_decoder_and_video_frame_
;
212 return BindToCurrentLoop(
213 base::Bind(&MemoryPool::OnVideoFrameDestroyed
, this,
217 bool VpxVideoDecoder::MemoryPool::OnMemoryDump(
218 const base::trace_event::MemoryDumpArgs
& args
,
219 base::trace_event::ProcessMemoryDump
* pmd
) {
220 base::trace_event::MemoryAllocatorDump
* memory_dump
=
221 pmd
->CreateAllocatorDump("media/vpx/memory_pool");
222 base::trace_event::MemoryAllocatorDump
* used_memory_dump
=
223 pmd
->CreateAllocatorDump("media/vpx/memory_pool/used");
225 pmd
->AddSuballocation(memory_dump
->guid(),
226 base::trace_event::MemoryDumpManager::GetInstance()
227 ->system_allocator_pool_name());
228 size_t bytes_used
= 0;
229 size_t bytes_reserved
= 0;
230 for (const VP9FrameBuffer
* frame_buffer
: frame_buffers_
) {
231 if (frame_buffer
->ref_cnt
) {
232 bytes_used
+= frame_buffer
->data
.size();
234 bytes_reserved
+= frame_buffer
->data
.size();
237 memory_dump
->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize
,
238 base::trace_event::MemoryAllocatorDump::kUnitsBytes
,
240 used_memory_dump
->AddScalar(
241 base::trace_event::MemoryAllocatorDump::kNameSize
,
242 base::trace_event::MemoryAllocatorDump::kUnitsBytes
, bytes_used
);
247 int VpxVideoDecoder::MemoryPool::NumberOfFrameBuffersInUseByDecoder() const {
248 return in_use_by_decoder_
;
251 int VpxVideoDecoder::MemoryPool::
252 NumberOfFrameBuffersInUseByDecoderAndVideoFrame() const {
253 return in_use_by_decoder_and_video_frame_
;
256 void VpxVideoDecoder::MemoryPool::OnVideoFrameDestroyed(
257 VP9FrameBuffer
* frame_buffer
) {
258 --frame_buffer
->ref_cnt
;
259 if (frame_buffer
->ref_cnt
)
260 --in_use_by_decoder_and_video_frame_
;
263 VpxVideoDecoder::VpxVideoDecoder(
264 const scoped_refptr
<base::SingleThreadTaskRunner
>& task_runner
)
265 : task_runner_(task_runner
),
266 state_(kUninitialized
),
268 vpx_codec_alpha_(NULL
) {}
270 VpxVideoDecoder::~VpxVideoDecoder() {
271 DCHECK(task_runner_
->BelongsToCurrentThread());
275 std::string
VpxVideoDecoder::GetDisplayName() const {
276 return "VpxVideoDecoder";
279 void VpxVideoDecoder::Initialize(const VideoDecoderConfig
& config
,
281 const InitCB
& init_cb
,
282 const OutputCB
& output_cb
) {
283 DCHECK(task_runner_
->BelongsToCurrentThread());
284 DCHECK(config
.IsValidConfig());
285 DCHECK(!config
.is_encrypted());
286 DCHECK(decode_cb_
.is_null());
288 InitCB bound_init_cb
= BindToCurrentLoop(init_cb
);
290 if (!ConfigureDecoder(config
)) {
291 bound_init_cb
.Run(false);
298 output_cb_
= BindToCurrentLoop(output_cb
);
299 bound_init_cb
.Run(true);
302 static vpx_codec_ctx
* InitializeVpxContext(vpx_codec_ctx
* context
,
303 const VideoDecoderConfig
& config
) {
304 context
= new vpx_codec_ctx();
305 vpx_codec_dec_cfg_t vpx_config
= {0};
306 vpx_config
.w
= config
.coded_size().width();
307 vpx_config
.h
= config
.coded_size().height();
308 vpx_config
.threads
= GetThreadCount(config
);
310 vpx_codec_err_t status
= vpx_codec_dec_init(context
,
311 config
.codec() == kCodecVP9
?
316 if (status
!= VPX_CODEC_OK
) {
317 LOG(ERROR
) << "vpx_codec_dec_init failed, status=" << status
;
324 bool VpxVideoDecoder::ConfigureDecoder(const VideoDecoderConfig
& config
) {
325 if (config
.codec() != kCodecVP8
&& config
.codec() != kCodecVP9
)
328 #if !defined(DISABLE_FFMPEG_VIDEO_DECODERS)
329 // When FFmpegVideoDecoder is available it handles VP8 that doesn't have
330 // alpha, and VpxVideoDecoder will handle VP8 with alpha.
331 if (config
.codec() == kCodecVP8
&& config
.format() != PIXEL_FORMAT_YV12A
)
337 vpx_codec_
= InitializeVpxContext(vpx_codec_
, config
);
341 // We use our own buffers for VP9 so that there is no need to copy data after
343 if (config
.codec() == kCodecVP9
) {
344 memory_pool_
= new MemoryPool();
345 base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
346 memory_pool_
.get(), task_runner_
);
347 if (vpx_codec_set_frame_buffer_functions(vpx_codec_
,
348 &MemoryPool::GetVP9FrameBuffer
,
349 &MemoryPool::ReleaseVP9FrameBuffer
,
350 memory_pool_
.get())) {
351 LOG(ERROR
) << "Failed to configure external buffers.";
356 if (config
.format() == PIXEL_FORMAT_YV12A
) {
357 vpx_codec_alpha_
= InitializeVpxContext(vpx_codec_alpha_
, config
);
358 if (!vpx_codec_alpha_
)
365 void VpxVideoDecoder::CloseDecoder() {
367 vpx_codec_destroy(vpx_codec_
);
370 base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
374 if (vpx_codec_alpha_
) {
375 vpx_codec_destroy(vpx_codec_alpha_
);
376 delete vpx_codec_alpha_
;
377 vpx_codec_alpha_
= NULL
;
381 void VpxVideoDecoder::Decode(const scoped_refptr
<DecoderBuffer
>& buffer
,
382 const DecodeCB
& decode_cb
) {
383 DCHECK(task_runner_
->BelongsToCurrentThread());
384 DCHECK(!decode_cb
.is_null());
385 CHECK_NE(state_
, kUninitialized
);
386 CHECK(decode_cb_
.is_null()) << "Overlapping decodes are not supported.";
388 decode_cb_
= BindToCurrentLoop(decode_cb
);
390 if (state_
== kError
) {
391 base::ResetAndReturn(&decode_cb_
).Run(kDecodeError
);
395 // Return empty frames if decoding has finished.
396 if (state_
== kDecodeFinished
) {
397 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
401 DecodeBuffer(buffer
);
404 void VpxVideoDecoder::Reset(const base::Closure
& closure
) {
405 DCHECK(task_runner_
->BelongsToCurrentThread());
406 DCHECK(decode_cb_
.is_null());
409 task_runner_
->PostTask(FROM_HERE
, closure
);
412 void VpxVideoDecoder::DecodeBuffer(const scoped_refptr
<DecoderBuffer
>& buffer
) {
413 DCHECK(task_runner_
->BelongsToCurrentThread());
414 DCHECK_NE(state_
, kUninitialized
);
415 DCHECK_NE(state_
, kDecodeFinished
);
416 DCHECK_NE(state_
, kError
);
417 DCHECK(!decode_cb_
.is_null());
418 DCHECK(buffer
.get());
420 // Transition to kDecodeFinished on the first end of stream buffer.
421 if (state_
== kNormal
&& buffer
->end_of_stream()) {
422 state_
= kDecodeFinished
;
423 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
427 scoped_refptr
<VideoFrame
> video_frame
;
428 if (!VpxDecode(buffer
, &video_frame
)) {
430 base::ResetAndReturn(&decode_cb_
).Run(kDecodeError
);
434 if (video_frame
.get())
435 output_cb_
.Run(video_frame
);
437 // VideoDecoderShim expects that |decode_cb| is called only after
439 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
442 bool VpxVideoDecoder::VpxDecode(const scoped_refptr
<DecoderBuffer
>& buffer
,
443 scoped_refptr
<VideoFrame
>* video_frame
) {
445 DCHECK(!buffer
->end_of_stream());
447 // Pass |buffer| to libvpx.
448 int64 timestamp
= buffer
->timestamp().InMicroseconds();
449 void* user_priv
= reinterpret_cast<void*>(×tamp
);
452 TRACE_EVENT1("video", "vpx_codec_decode", "timestamp", timestamp
);
453 vpx_codec_err_t status
= vpx_codec_decode(vpx_codec_
,
458 if (status
!= VPX_CODEC_OK
) {
459 LOG(ERROR
) << "vpx_codec_decode() failed, status=" << status
;
464 // Gets pointer to decoded data.
465 vpx_codec_iter_t iter
= NULL
;
466 const vpx_image_t
* vpx_image
= vpx_codec_get_frame(vpx_codec_
, &iter
);
472 if (vpx_image
->user_priv
!= reinterpret_cast<void*>(×tamp
)) {
473 LOG(ERROR
) << "Invalid output timestamp.";
477 const vpx_image_t
* vpx_image_alpha
= NULL
;
478 if (vpx_codec_alpha_
&& buffer
->side_data_size() >= 8) {
479 // Pass alpha data to libvpx.
480 int64 timestamp_alpha
= buffer
->timestamp().InMicroseconds();
481 void* user_priv_alpha
= reinterpret_cast<void*>(×tamp_alpha
);
483 // First 8 bytes of side data is side_data_id in big endian.
484 const uint64 side_data_id
= base::NetToHost64(
485 *(reinterpret_cast<const uint64
*>(buffer
->side_data())));
486 if (side_data_id
== 1) {
488 TRACE_EVENT1("video", "vpx_codec_decode_alpha",
489 "timestamp_alpha", timestamp_alpha
);
490 vpx_codec_err_t status
= vpx_codec_decode(vpx_codec_alpha_
,
491 buffer
->side_data() + 8,
492 buffer
->side_data_size() - 8,
495 if (status
!= VPX_CODEC_OK
) {
496 LOG(ERROR
) << "vpx_codec_decode() failed on alpha, status=" << status
;
501 // Gets pointer to decoded data.
502 vpx_codec_iter_t iter_alpha
= NULL
;
503 vpx_image_alpha
= vpx_codec_get_frame(vpx_codec_alpha_
, &iter_alpha
);
504 if (!vpx_image_alpha
) {
509 if (vpx_image_alpha
->user_priv
!=
510 reinterpret_cast<void*>(×tamp_alpha
)) {
511 LOG(ERROR
) << "Invalid output timestamp on alpha.";
515 if (vpx_image_alpha
->d_h
!= vpx_image
->d_h
||
516 vpx_image_alpha
->d_w
!= vpx_image
->d_w
) {
517 LOG(ERROR
) << "The alpha plane dimensions are not the same as the "
524 CopyVpxImageTo(vpx_image
, vpx_image_alpha
, video_frame
);
525 (*video_frame
)->set_timestamp(base::TimeDelta::FromMicroseconds(timestamp
));
529 void VpxVideoDecoder::CopyVpxImageTo(const vpx_image
* vpx_image
,
530 const struct vpx_image
* vpx_image_alpha
,
531 scoped_refptr
<VideoFrame
>* video_frame
) {
533 CHECK(vpx_image
->fmt
== VPX_IMG_FMT_I420
||
534 vpx_image
->fmt
== VPX_IMG_FMT_YV12
||
535 vpx_image
->fmt
== VPX_IMG_FMT_I444
);
537 VideoPixelFormat codec_format
= PIXEL_FORMAT_YV12
;
538 int uv_rows
= (vpx_image
->d_h
+ 1) / 2;
540 if (vpx_image
->fmt
== VPX_IMG_FMT_I444
) {
541 CHECK(!vpx_codec_alpha_
);
542 codec_format
= PIXEL_FORMAT_YV24
;
543 uv_rows
= vpx_image
->d_h
;
544 } else if (vpx_codec_alpha_
) {
545 codec_format
= PIXEL_FORMAT_YV12A
;
548 // Default to the color space from the config, but if the bistream specifies
549 // one, prefer that instead.
550 ColorSpace color_space
= config_
.color_space();
551 if (vpx_image
->cs
== VPX_CS_BT_709
)
552 color_space
= COLOR_SPACE_HD_REC709
;
553 else if (vpx_image
->cs
== VPX_CS_BT_601
)
554 color_space
= COLOR_SPACE_SD_REC601
;
556 // The mixed |w|/|d_h| in |coded_size| is intentional. Setting the correct
557 // coded width is necessary to allow coalesced memory access, which may avoid
558 // frame copies. Setting the correct coded height however does not have any
559 // benefit, and only risk copying too much data.
560 const gfx::Size
coded_size(vpx_image
->w
, vpx_image
->d_h
);
561 const gfx::Size
visible_size(vpx_image
->d_w
, vpx_image
->d_h
);
563 if (!vpx_codec_alpha_
&& memory_pool_
.get()) {
564 *video_frame
= VideoFrame::WrapExternalYuvData(
566 coded_size
, gfx::Rect(visible_size
), config_
.natural_size(),
567 vpx_image
->stride
[VPX_PLANE_Y
],
568 vpx_image
->stride
[VPX_PLANE_U
],
569 vpx_image
->stride
[VPX_PLANE_V
],
570 vpx_image
->planes
[VPX_PLANE_Y
],
571 vpx_image
->planes
[VPX_PLANE_U
],
572 vpx_image
->planes
[VPX_PLANE_V
],
574 video_frame
->get()->AddDestructionObserver(
575 memory_pool_
->CreateFrameCallback(vpx_image
->fb_priv
));
576 video_frame
->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE
,
579 UMA_HISTOGRAM_COUNTS("Media.Vpx.VideoDecoderBuffersInUseByDecoder",
580 memory_pool_
->NumberOfFrameBuffersInUseByDecoder());
581 UMA_HISTOGRAM_COUNTS(
582 "Media.Vpx.VideoDecoderBuffersInUseByDecoderAndVideoFrame",
583 memory_pool_
->NumberOfFrameBuffersInUseByDecoderAndVideoFrame());
588 *video_frame
= frame_pool_
.CreateFrame(
591 gfx::Rect(visible_size
),
592 config_
.natural_size(),
594 video_frame
->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE
,
597 CopyYPlane(vpx_image
->planes
[VPX_PLANE_Y
],
598 vpx_image
->stride
[VPX_PLANE_Y
],
601 CopyUPlane(vpx_image
->planes
[VPX_PLANE_U
],
602 vpx_image
->stride
[VPX_PLANE_U
],
605 CopyVPlane(vpx_image
->planes
[VPX_PLANE_V
],
606 vpx_image
->stride
[VPX_PLANE_V
],
609 if (!vpx_codec_alpha_
)
611 if (!vpx_image_alpha
) {
613 vpx_image
->stride
[VPX_PLANE_Y
], vpx_image
->d_h
, video_frame
->get());
616 CopyAPlane(vpx_image_alpha
->planes
[VPX_PLANE_Y
],
617 vpx_image_alpha
->stride
[VPX_PLANE_Y
],
618 vpx_image_alpha
->d_h
,