1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/vpx_video_decoder.h"
11 #include "base/bind.h"
12 #include "base/callback_helpers.h"
13 #include "base/command_line.h"
14 #include "base/location.h"
15 #include "base/logging.h"
16 #include "base/single_thread_task_runner.h"
17 #include "base/stl_util.h"
18 #include "base/strings/string_number_conversions.h"
19 #include "base/sys_byteorder.h"
20 #include "base/sys_info.h"
21 #include "base/trace_event/memory_allocator_dump.h"
22 #include "base/trace_event/memory_dump_manager.h"
23 #include "base/trace_event/memory_dump_provider.h"
24 #include "base/trace_event/process_memory_dump.h"
25 #include "base/trace_event/trace_event.h"
26 #include "media/base/bind_to_current_loop.h"
27 #include "media/base/decoder_buffer.h"
28 #include "media/base/limits.h"
29 #include "media/base/media_switches.h"
30 #include "media/base/pipeline.h"
31 #include "media/base/video_util.h"
33 // Include libvpx header files.
34 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
35 // backwards compatibility for legacy applications using the library.
36 #define VPX_CODEC_DISABLE_COMPAT 1
38 #include "third_party/libvpx/source/libvpx/vpx/vpx_decoder.h"
39 #include "third_party/libvpx/source/libvpx/vpx/vpx_frame_buffer.h"
40 #include "third_party/libvpx/source/libvpx/vpx/vp8dx.h"
45 // Always try to use three threads for video decoding. There is little reason
46 // not to since current day CPUs tend to be multi-core and we measured
47 // performance benefits on older machines such as P4s with hyperthreading.
48 static const int kDecodeThreads
= 2;
49 static const int kMaxDecodeThreads
= 16;
51 // Returns the number of threads.
52 static int GetThreadCount(const VideoDecoderConfig
& config
) {
53 // Refer to http://crbug.com/93932 for tsan suppressions on decoding.
54 int decode_threads
= kDecodeThreads
;
56 const base::CommandLine
* cmd_line
= base::CommandLine::ForCurrentProcess();
57 std::string
threads(cmd_line
->GetSwitchValueASCII(switches::kVideoThreads
));
58 if (threads
.empty() || !base::StringToInt(threads
, &decode_threads
)) {
59 if (config
.codec() == kCodecVP9
) {
60 // For VP9 decode when using the default thread count, increase the number
61 // of decode threads to equal the maximum number of tiles possible for
62 // higher resolution streams.
63 if (config
.coded_size().width() >= 2048)
65 else if (config
.coded_size().width() >= 1024)
69 decode_threads
= std::min(decode_threads
,
70 base::SysInfo::NumberOfProcessors());
71 return decode_threads
;
74 decode_threads
= std::max(decode_threads
, 0);
75 decode_threads
= std::min(decode_threads
, kMaxDecodeThreads
);
76 return decode_threads
;
79 class VpxVideoDecoder::MemoryPool
80 : public base::RefCountedThreadSafe
<VpxVideoDecoder::MemoryPool
>,
81 public base::trace_event::MemoryDumpProvider
{
85 // Callback that will be called by libvpx when it needs a frame buffer.
87 // |user_priv| Private data passed to libvpx (pointer to memory pool).
88 // |min_size| Minimum size needed by libvpx to decompress the next frame.
89 // |fb| Pointer to the frame buffer to update.
90 // Returns 0 on success. Returns < 0 on failure.
91 static int32
GetVP9FrameBuffer(void* user_priv
, size_t min_size
,
92 vpx_codec_frame_buffer
* fb
);
94 // Callback that will be called by libvpx when the frame buffer is no longer
95 // being used by libvpx. Parameters:
96 // |user_priv| Private data passed to libvpx (pointer to memory pool).
97 // |fb| Pointer to the frame buffer that's being released.
98 static int32
ReleaseVP9FrameBuffer(void *user_priv
,
99 vpx_codec_frame_buffer
*fb
);
101 // Generates a "no_longer_needed" closure that holds a reference
103 base::Closure
CreateFrameCallback(void* fb_priv_data
);
105 bool OnMemoryDump(base::trace_event::ProcessMemoryDump
* pmd
) override
;
108 friend class base::RefCountedThreadSafe
<VpxVideoDecoder::MemoryPool
>;
109 ~MemoryPool() override
;
111 // Reference counted frame buffers used for VP9 decoding. Reference counting
112 // is done manually because both chromium and libvpx has to release this
113 // before a buffer can be re-used.
114 struct VP9FrameBuffer
{
115 VP9FrameBuffer() : ref_cnt(0) {}
116 std::vector
<uint8
> data
;
120 // Gets the next available frame buffer for use by libvpx.
121 VP9FrameBuffer
* GetFreeFrameBuffer(size_t min_size
);
123 // Method that gets called when a VideoFrame that references this pool gets
125 void OnVideoFrameDestroyed(VP9FrameBuffer
* frame_buffer
);
127 // Frame buffers to be used by libvpx for VP9 Decoding.
128 std::vector
<VP9FrameBuffer
*> frame_buffers_
;
130 DISALLOW_COPY_AND_ASSIGN(MemoryPool
);
133 VpxVideoDecoder::MemoryPool::MemoryPool() {}
135 VpxVideoDecoder::MemoryPool::~MemoryPool() {
136 STLDeleteElements(&frame_buffers_
);
139 VpxVideoDecoder::MemoryPool::VP9FrameBuffer
*
140 VpxVideoDecoder::MemoryPool::GetFreeFrameBuffer(size_t min_size
) {
141 // Check if a free frame buffer exists.
143 for (; i
< frame_buffers_
.size(); ++i
) {
144 if (frame_buffers_
[i
]->ref_cnt
== 0)
148 if (i
== frame_buffers_
.size()) {
149 // Create a new frame buffer.
150 frame_buffers_
.push_back(new VP9FrameBuffer());
153 // Resize the frame buffer if necessary.
154 if (frame_buffers_
[i
]->data
.size() < min_size
)
155 frame_buffers_
[i
]->data
.resize(min_size
);
156 return frame_buffers_
[i
];
159 int32
VpxVideoDecoder::MemoryPool::GetVP9FrameBuffer(
160 void* user_priv
, size_t min_size
, vpx_codec_frame_buffer
* fb
) {
164 VpxVideoDecoder::MemoryPool
* memory_pool
=
165 static_cast<VpxVideoDecoder::MemoryPool
*>(user_priv
);
167 VP9FrameBuffer
* fb_to_use
= memory_pool
->GetFreeFrameBuffer(min_size
);
168 if (fb_to_use
== NULL
)
171 fb
->data
= &fb_to_use
->data
[0];
172 fb
->size
= fb_to_use
->data
.size();
173 ++fb_to_use
->ref_cnt
;
175 // Set the frame buffer's private data to point at the external frame buffer.
176 fb
->priv
= static_cast<void*>(fb_to_use
);
180 int32
VpxVideoDecoder::MemoryPool::ReleaseVP9FrameBuffer(
181 void *user_priv
, vpx_codec_frame_buffer
*fb
) {
182 VP9FrameBuffer
* frame_buffer
= static_cast<VP9FrameBuffer
*>(fb
->priv
);
183 --frame_buffer
->ref_cnt
;
187 base::Closure
VpxVideoDecoder::MemoryPool::CreateFrameCallback(
188 void* fb_priv_data
) {
189 VP9FrameBuffer
* frame_buffer
= static_cast<VP9FrameBuffer
*>(fb_priv_data
);
190 ++frame_buffer
->ref_cnt
;
191 return BindToCurrentLoop(
192 base::Bind(&MemoryPool::OnVideoFrameDestroyed
, this,
196 bool VpxVideoDecoder::MemoryPool::OnMemoryDump(
197 base::trace_event::ProcessMemoryDump
* pmd
) {
198 base::trace_event::MemoryAllocatorDump
* memory_dump
=
199 pmd
->CreateAllocatorDump("media/vpx/memory_pool");
200 base::trace_event::MemoryAllocatorDump
* used_memory_dump
=
201 pmd
->CreateAllocatorDump("media/vpx/memory_pool/used");
203 pmd
->AddSuballocation(memory_dump
->guid(),
204 base::trace_event::MemoryDumpManager::GetInstance()
205 ->system_allocator_pool_name());
206 size_t bytes_used
= 0;
207 size_t bytes_reserved
= 0;
208 for (const VP9FrameBuffer
* frame_buffer
: frame_buffers_
) {
209 if (frame_buffer
->ref_cnt
) {
210 bytes_used
+= frame_buffer
->data
.size();
212 bytes_reserved
+= frame_buffer
->data
.size();
215 memory_dump
->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize
,
216 base::trace_event::MemoryAllocatorDump::kUnitsBytes
,
218 used_memory_dump
->AddScalar(
219 base::trace_event::MemoryAllocatorDump::kNameSize
,
220 base::trace_event::MemoryAllocatorDump::kUnitsBytes
, bytes_used
);
225 void VpxVideoDecoder::MemoryPool::OnVideoFrameDestroyed(
226 VP9FrameBuffer
* frame_buffer
) {
227 --frame_buffer
->ref_cnt
;
230 VpxVideoDecoder::VpxVideoDecoder(
231 const scoped_refptr
<base::SingleThreadTaskRunner
>& task_runner
)
232 : task_runner_(task_runner
),
233 state_(kUninitialized
),
235 vpx_codec_alpha_(NULL
) {}
237 VpxVideoDecoder::~VpxVideoDecoder() {
238 DCHECK(task_runner_
->BelongsToCurrentThread());
242 std::string
VpxVideoDecoder::GetDisplayName() const {
243 return "VpxVideoDecoder";
246 void VpxVideoDecoder::Initialize(const VideoDecoderConfig
& config
,
248 const InitCB
& init_cb
,
249 const OutputCB
& output_cb
) {
250 DCHECK(task_runner_
->BelongsToCurrentThread());
251 DCHECK(config
.IsValidConfig());
252 DCHECK(!config
.is_encrypted());
253 DCHECK(decode_cb_
.is_null());
255 InitCB bound_init_cb
= BindToCurrentLoop(init_cb
);
257 if (!ConfigureDecoder(config
)) {
258 bound_init_cb
.Run(false);
265 output_cb_
= BindToCurrentLoop(output_cb
);
266 bound_init_cb
.Run(true);
269 static vpx_codec_ctx
* InitializeVpxContext(vpx_codec_ctx
* context
,
270 const VideoDecoderConfig
& config
) {
271 context
= new vpx_codec_ctx();
272 vpx_codec_dec_cfg_t vpx_config
= {0};
273 vpx_config
.w
= config
.coded_size().width();
274 vpx_config
.h
= config
.coded_size().height();
275 vpx_config
.threads
= GetThreadCount(config
);
277 vpx_codec_err_t status
= vpx_codec_dec_init(context
,
278 config
.codec() == kCodecVP9
?
283 if (status
!= VPX_CODEC_OK
) {
284 LOG(ERROR
) << "vpx_codec_dec_init failed, status=" << status
;
291 bool VpxVideoDecoder::ConfigureDecoder(const VideoDecoderConfig
& config
) {
292 if (config
.codec() != kCodecVP8
&& config
.codec() != kCodecVP9
)
295 // In VP8 videos, only those with alpha are handled by VpxVideoDecoder. All
296 // other VP8 videos go to FFmpegVideoDecoder.
297 if (config
.codec() == kCodecVP8
&& config
.format() != PIXEL_FORMAT_YV12A
)
302 vpx_codec_
= InitializeVpxContext(vpx_codec_
, config
);
306 // We use our own buffers for VP9 so that there is no need to copy data after
308 if (config
.codec() == kCodecVP9
) {
309 memory_pool_
= new MemoryPool();
310 base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
311 memory_pool_
.get(), task_runner_
);
312 if (vpx_codec_set_frame_buffer_functions(vpx_codec_
,
313 &MemoryPool::GetVP9FrameBuffer
,
314 &MemoryPool::ReleaseVP9FrameBuffer
,
315 memory_pool_
.get())) {
316 LOG(ERROR
) << "Failed to configure external buffers.";
321 if (config
.format() == PIXEL_FORMAT_YV12A
) {
322 vpx_codec_alpha_
= InitializeVpxContext(vpx_codec_alpha_
, config
);
323 if (!vpx_codec_alpha_
)
330 void VpxVideoDecoder::CloseDecoder() {
332 vpx_codec_destroy(vpx_codec_
);
335 base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
339 if (vpx_codec_alpha_
) {
340 vpx_codec_destroy(vpx_codec_alpha_
);
341 delete vpx_codec_alpha_
;
342 vpx_codec_alpha_
= NULL
;
346 void VpxVideoDecoder::Decode(const scoped_refptr
<DecoderBuffer
>& buffer
,
347 const DecodeCB
& decode_cb
) {
348 DCHECK(task_runner_
->BelongsToCurrentThread());
349 DCHECK(!decode_cb
.is_null());
350 CHECK_NE(state_
, kUninitialized
);
351 CHECK(decode_cb_
.is_null()) << "Overlapping decodes are not supported.";
353 decode_cb_
= BindToCurrentLoop(decode_cb
);
355 if (state_
== kError
) {
356 base::ResetAndReturn(&decode_cb_
).Run(kDecodeError
);
360 // Return empty frames if decoding has finished.
361 if (state_
== kDecodeFinished
) {
362 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
366 DecodeBuffer(buffer
);
369 void VpxVideoDecoder::Reset(const base::Closure
& closure
) {
370 DCHECK(task_runner_
->BelongsToCurrentThread());
371 DCHECK(decode_cb_
.is_null());
374 task_runner_
->PostTask(FROM_HERE
, closure
);
377 void VpxVideoDecoder::DecodeBuffer(const scoped_refptr
<DecoderBuffer
>& buffer
) {
378 DCHECK(task_runner_
->BelongsToCurrentThread());
379 DCHECK_NE(state_
, kUninitialized
);
380 DCHECK_NE(state_
, kDecodeFinished
);
381 DCHECK_NE(state_
, kError
);
382 DCHECK(!decode_cb_
.is_null());
383 DCHECK(buffer
.get());
385 // Transition to kDecodeFinished on the first end of stream buffer.
386 if (state_
== kNormal
&& buffer
->end_of_stream()) {
387 state_
= kDecodeFinished
;
388 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
392 scoped_refptr
<VideoFrame
> video_frame
;
393 if (!VpxDecode(buffer
, &video_frame
)) {
395 base::ResetAndReturn(&decode_cb_
).Run(kDecodeError
);
399 if (video_frame
.get())
400 output_cb_
.Run(video_frame
);
402 // VideoDecoderShim expects that |decode_cb| is called only after
404 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
407 bool VpxVideoDecoder::VpxDecode(const scoped_refptr
<DecoderBuffer
>& buffer
,
408 scoped_refptr
<VideoFrame
>* video_frame
) {
410 DCHECK(!buffer
->end_of_stream());
412 // Pass |buffer| to libvpx.
413 int64 timestamp
= buffer
->timestamp().InMicroseconds();
414 void* user_priv
= reinterpret_cast<void*>(×tamp
);
417 TRACE_EVENT1("video", "vpx_codec_decode", "timestamp", timestamp
);
418 vpx_codec_err_t status
= vpx_codec_decode(vpx_codec_
,
423 if (status
!= VPX_CODEC_OK
) {
424 LOG(ERROR
) << "vpx_codec_decode() failed, status=" << status
;
429 // Gets pointer to decoded data.
430 vpx_codec_iter_t iter
= NULL
;
431 const vpx_image_t
* vpx_image
= vpx_codec_get_frame(vpx_codec_
, &iter
);
437 if (vpx_image
->user_priv
!= reinterpret_cast<void*>(×tamp
)) {
438 LOG(ERROR
) << "Invalid output timestamp.";
442 const vpx_image_t
* vpx_image_alpha
= NULL
;
443 if (vpx_codec_alpha_
&& buffer
->side_data_size() >= 8) {
444 // Pass alpha data to libvpx.
445 int64 timestamp_alpha
= buffer
->timestamp().InMicroseconds();
446 void* user_priv_alpha
= reinterpret_cast<void*>(×tamp_alpha
);
448 // First 8 bytes of side data is side_data_id in big endian.
449 const uint64 side_data_id
= base::NetToHost64(
450 *(reinterpret_cast<const uint64
*>(buffer
->side_data())));
451 if (side_data_id
== 1) {
453 TRACE_EVENT1("video", "vpx_codec_decode_alpha",
454 "timestamp_alpha", timestamp_alpha
);
455 vpx_codec_err_t status
= vpx_codec_decode(vpx_codec_alpha_
,
456 buffer
->side_data() + 8,
457 buffer
->side_data_size() - 8,
460 if (status
!= VPX_CODEC_OK
) {
461 LOG(ERROR
) << "vpx_codec_decode() failed on alpha, status=" << status
;
466 // Gets pointer to decoded data.
467 vpx_codec_iter_t iter_alpha
= NULL
;
468 vpx_image_alpha
= vpx_codec_get_frame(vpx_codec_alpha_
, &iter_alpha
);
469 if (!vpx_image_alpha
) {
474 if (vpx_image_alpha
->user_priv
!=
475 reinterpret_cast<void*>(×tamp_alpha
)) {
476 LOG(ERROR
) << "Invalid output timestamp on alpha.";
480 if (vpx_image_alpha
->d_h
!= vpx_image
->d_h
||
481 vpx_image_alpha
->d_w
!= vpx_image
->d_w
) {
482 LOG(ERROR
) << "The alpha plane dimensions are not the same as the "
489 CopyVpxImageTo(vpx_image
, vpx_image_alpha
, video_frame
);
490 (*video_frame
)->set_timestamp(base::TimeDelta::FromMicroseconds(timestamp
));
494 void VpxVideoDecoder::CopyVpxImageTo(const vpx_image
* vpx_image
,
495 const struct vpx_image
* vpx_image_alpha
,
496 scoped_refptr
<VideoFrame
>* video_frame
) {
498 CHECK(vpx_image
->fmt
== VPX_IMG_FMT_I420
||
499 vpx_image
->fmt
== VPX_IMG_FMT_YV12
||
500 vpx_image
->fmt
== VPX_IMG_FMT_I444
);
502 VideoPixelFormat codec_format
= PIXEL_FORMAT_YV12
;
503 int uv_rows
= (vpx_image
->d_h
+ 1) / 2;
505 if (vpx_image
->fmt
== VPX_IMG_FMT_I444
) {
506 CHECK(!vpx_codec_alpha_
);
507 codec_format
= PIXEL_FORMAT_YV24
;
508 uv_rows
= vpx_image
->d_h
;
509 } else if (vpx_codec_alpha_
) {
510 codec_format
= PIXEL_FORMAT_YV12A
;
513 // Default to the color space from the config, but if the bistream specifies
514 // one, prefer that instead.
515 ColorSpace color_space
= config_
.color_space();
516 if (vpx_image
->cs
== VPX_CS_BT_709
)
517 color_space
= COLOR_SPACE_HD_REC709
;
518 else if (vpx_image
->cs
== VPX_CS_BT_601
)
519 color_space
= COLOR_SPACE_SD_REC601
;
521 // The mixed |w|/|d_h| in |coded_size| is intentional. Setting the correct
522 // coded width is necessary to allow coalesced memory access, which may avoid
523 // frame copies. Setting the correct coded height however does not have any
524 // benefit, and only risk copying too much data.
525 const gfx::Size
coded_size(vpx_image
->w
, vpx_image
->d_h
);
526 const gfx::Size
visible_size(vpx_image
->d_w
, vpx_image
->d_h
);
528 if (!vpx_codec_alpha_
&& memory_pool_
.get()) {
529 *video_frame
= VideoFrame::WrapExternalYuvData(
531 coded_size
, gfx::Rect(visible_size
), config_
.natural_size(),
532 vpx_image
->stride
[VPX_PLANE_Y
],
533 vpx_image
->stride
[VPX_PLANE_U
],
534 vpx_image
->stride
[VPX_PLANE_V
],
535 vpx_image
->planes
[VPX_PLANE_Y
],
536 vpx_image
->planes
[VPX_PLANE_U
],
537 vpx_image
->planes
[VPX_PLANE_V
],
539 video_frame
->get()->AddDestructionObserver(
540 memory_pool_
->CreateFrameCallback(vpx_image
->fb_priv
));
541 video_frame
->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE
,
546 *video_frame
= frame_pool_
.CreateFrame(
549 gfx::Rect(visible_size
),
550 config_
.natural_size(),
552 video_frame
->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE
,
555 CopyYPlane(vpx_image
->planes
[VPX_PLANE_Y
],
556 vpx_image
->stride
[VPX_PLANE_Y
],
559 CopyUPlane(vpx_image
->planes
[VPX_PLANE_U
],
560 vpx_image
->stride
[VPX_PLANE_U
],
563 CopyVPlane(vpx_image
->planes
[VPX_PLANE_V
],
564 vpx_image
->stride
[VPX_PLANE_V
],
567 if (!vpx_codec_alpha_
)
569 if (!vpx_image_alpha
) {
571 vpx_image
->stride
[VPX_PLANE_Y
], vpx_image
->d_h
, video_frame
->get());
574 CopyAPlane(vpx_image_alpha
->planes
[VPX_PLANE_Y
],
575 vpx_image_alpha
->stride
[VPX_PLANE_Y
],
576 vpx_image_alpha
->d_h
,