1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/vpx_video_decoder.h"
11 #include "base/bind.h"
12 #include "base/callback_helpers.h"
13 #include "base/command_line.h"
14 #include "base/location.h"
15 #include "base/logging.h"
16 #include "base/single_thread_task_runner.h"
17 #include "base/stl_util.h"
18 #include "base/strings/string_number_conversions.h"
19 #include "base/sys_byteorder.h"
20 #include "base/sys_info.h"
21 #include "base/trace_event/memory_allocator_dump.h"
22 #include "base/trace_event/memory_dump_manager.h"
23 #include "base/trace_event/memory_dump_provider.h"
24 #include "base/trace_event/process_memory_dump.h"
25 #include "base/trace_event/trace_event.h"
26 #include "media/base/bind_to_current_loop.h"
27 #include "media/base/decoder_buffer.h"
28 #include "media/base/limits.h"
29 #include "media/base/media_switches.h"
30 #include "media/base/pipeline.h"
31 #include "media/base/timestamp_constants.h"
32 #include "media/base/video_util.h"
34 // Include libvpx header files.
35 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
36 // backwards compatibility for legacy applications using the library.
37 #define VPX_CODEC_DISABLE_COMPAT 1
39 #include "third_party/libvpx/source/libvpx/vpx/vpx_decoder.h"
40 #include "third_party/libvpx/source/libvpx/vpx/vpx_frame_buffer.h"
41 #include "third_party/libvpx/source/libvpx/vpx/vp8dx.h"
46 // Always try to use three threads for video decoding. There is little reason
47 // not to since current day CPUs tend to be multi-core and we measured
48 // performance benefits on older machines such as P4s with hyperthreading.
49 static const int kDecodeThreads
= 2;
50 static const int kMaxDecodeThreads
= 16;
52 // Returns the number of threads.
53 static int GetThreadCount(const VideoDecoderConfig
& config
) {
54 // Refer to http://crbug.com/93932 for tsan suppressions on decoding.
55 int decode_threads
= kDecodeThreads
;
57 const base::CommandLine
* cmd_line
= base::CommandLine::ForCurrentProcess();
58 std::string
threads(cmd_line
->GetSwitchValueASCII(switches::kVideoThreads
));
59 if (threads
.empty() || !base::StringToInt(threads
, &decode_threads
)) {
60 if (config
.codec() == kCodecVP9
) {
61 // For VP9 decode when using the default thread count, increase the number
62 // of decode threads to equal the maximum number of tiles possible for
63 // higher resolution streams.
64 if (config
.coded_size().width() >= 2048)
66 else if (config
.coded_size().width() >= 1024)
70 decode_threads
= std::min(decode_threads
,
71 base::SysInfo::NumberOfProcessors());
72 return decode_threads
;
75 decode_threads
= std::max(decode_threads
, 0);
76 decode_threads
= std::min(decode_threads
, kMaxDecodeThreads
);
77 return decode_threads
;
80 class VpxVideoDecoder::MemoryPool
81 : public base::RefCountedThreadSafe
<VpxVideoDecoder::MemoryPool
>,
82 public base::trace_event::MemoryDumpProvider
{
86 // Callback that will be called by libvpx when it needs a frame buffer.
88 // |user_priv| Private data passed to libvpx (pointer to memory pool).
89 // |min_size| Minimum size needed by libvpx to decompress the next frame.
90 // |fb| Pointer to the frame buffer to update.
91 // Returns 0 on success. Returns < 0 on failure.
92 static int32
GetVP9FrameBuffer(void* user_priv
, size_t min_size
,
93 vpx_codec_frame_buffer
* fb
);
95 // Callback that will be called by libvpx when the frame buffer is no longer
96 // being used by libvpx. Parameters:
97 // |user_priv| Private data passed to libvpx (pointer to memory pool).
98 // |fb| Pointer to the frame buffer that's being released.
99 static int32
ReleaseVP9FrameBuffer(void *user_priv
,
100 vpx_codec_frame_buffer
*fb
);
102 // Generates a "no_longer_needed" closure that holds a reference
104 base::Closure
CreateFrameCallback(void* fb_priv_data
);
106 bool OnMemoryDump(const base::trace_event::MemoryDumpArgs
& args
,
107 base::trace_event::ProcessMemoryDump
* pmd
) override
;
110 friend class base::RefCountedThreadSafe
<VpxVideoDecoder::MemoryPool
>;
111 ~MemoryPool() override
;
113 // Reference counted frame buffers used for VP9 decoding. Reference counting
114 // is done manually because both chromium and libvpx has to release this
115 // before a buffer can be re-used.
116 struct VP9FrameBuffer
{
117 VP9FrameBuffer() : ref_cnt(0) {}
118 std::vector
<uint8
> data
;
122 // Gets the next available frame buffer for use by libvpx.
123 VP9FrameBuffer
* GetFreeFrameBuffer(size_t min_size
);
125 // Method that gets called when a VideoFrame that references this pool gets
127 void OnVideoFrameDestroyed(VP9FrameBuffer
* frame_buffer
);
129 // Frame buffers to be used by libvpx for VP9 Decoding.
130 std::vector
<VP9FrameBuffer
*> frame_buffers_
;
132 DISALLOW_COPY_AND_ASSIGN(MemoryPool
);
135 VpxVideoDecoder::MemoryPool::MemoryPool() {}
137 VpxVideoDecoder::MemoryPool::~MemoryPool() {
138 STLDeleteElements(&frame_buffers_
);
141 VpxVideoDecoder::MemoryPool::VP9FrameBuffer
*
142 VpxVideoDecoder::MemoryPool::GetFreeFrameBuffer(size_t min_size
) {
143 // Check if a free frame buffer exists.
145 for (; i
< frame_buffers_
.size(); ++i
) {
146 if (frame_buffers_
[i
]->ref_cnt
== 0)
150 if (i
== frame_buffers_
.size()) {
151 // Create a new frame buffer.
152 frame_buffers_
.push_back(new VP9FrameBuffer());
155 // Resize the frame buffer if necessary.
156 if (frame_buffers_
[i
]->data
.size() < min_size
)
157 frame_buffers_
[i
]->data
.resize(min_size
);
158 return frame_buffers_
[i
];
161 int32
VpxVideoDecoder::MemoryPool::GetVP9FrameBuffer(
162 void* user_priv
, size_t min_size
, vpx_codec_frame_buffer
* fb
) {
166 VpxVideoDecoder::MemoryPool
* memory_pool
=
167 static_cast<VpxVideoDecoder::MemoryPool
*>(user_priv
);
169 VP9FrameBuffer
* fb_to_use
= memory_pool
->GetFreeFrameBuffer(min_size
);
170 if (fb_to_use
== NULL
)
173 fb
->data
= &fb_to_use
->data
[0];
174 fb
->size
= fb_to_use
->data
.size();
175 ++fb_to_use
->ref_cnt
;
177 // Set the frame buffer's private data to point at the external frame buffer.
178 fb
->priv
= static_cast<void*>(fb_to_use
);
182 int32
VpxVideoDecoder::MemoryPool::ReleaseVP9FrameBuffer(
183 void *user_priv
, vpx_codec_frame_buffer
*fb
) {
184 VP9FrameBuffer
* frame_buffer
= static_cast<VP9FrameBuffer
*>(fb
->priv
);
185 --frame_buffer
->ref_cnt
;
189 base::Closure
VpxVideoDecoder::MemoryPool::CreateFrameCallback(
190 void* fb_priv_data
) {
191 VP9FrameBuffer
* frame_buffer
= static_cast<VP9FrameBuffer
*>(fb_priv_data
);
192 ++frame_buffer
->ref_cnt
;
193 return BindToCurrentLoop(
194 base::Bind(&MemoryPool::OnVideoFrameDestroyed
, this,
198 bool VpxVideoDecoder::MemoryPool::OnMemoryDump(
199 const base::trace_event::MemoryDumpArgs
& args
,
200 base::trace_event::ProcessMemoryDump
* pmd
) {
201 base::trace_event::MemoryAllocatorDump
* memory_dump
=
202 pmd
->CreateAllocatorDump("media/vpx/memory_pool");
203 base::trace_event::MemoryAllocatorDump
* used_memory_dump
=
204 pmd
->CreateAllocatorDump("media/vpx/memory_pool/used");
206 pmd
->AddSuballocation(memory_dump
->guid(),
207 base::trace_event::MemoryDumpManager::GetInstance()
208 ->system_allocator_pool_name());
209 size_t bytes_used
= 0;
210 size_t bytes_reserved
= 0;
211 for (const VP9FrameBuffer
* frame_buffer
: frame_buffers_
) {
212 if (frame_buffer
->ref_cnt
) {
213 bytes_used
+= frame_buffer
->data
.size();
215 bytes_reserved
+= frame_buffer
->data
.size();
218 memory_dump
->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize
,
219 base::trace_event::MemoryAllocatorDump::kUnitsBytes
,
221 used_memory_dump
->AddScalar(
222 base::trace_event::MemoryAllocatorDump::kNameSize
,
223 base::trace_event::MemoryAllocatorDump::kUnitsBytes
, bytes_used
);
228 void VpxVideoDecoder::MemoryPool::OnVideoFrameDestroyed(
229 VP9FrameBuffer
* frame_buffer
) {
230 --frame_buffer
->ref_cnt
;
233 VpxVideoDecoder::VpxVideoDecoder(
234 const scoped_refptr
<base::SingleThreadTaskRunner
>& task_runner
)
235 : task_runner_(task_runner
),
236 state_(kUninitialized
),
238 vpx_codec_alpha_(NULL
) {}
240 VpxVideoDecoder::~VpxVideoDecoder() {
241 DCHECK(task_runner_
->BelongsToCurrentThread());
245 std::string
VpxVideoDecoder::GetDisplayName() const {
246 return "VpxVideoDecoder";
249 void VpxVideoDecoder::Initialize(const VideoDecoderConfig
& config
,
251 const InitCB
& init_cb
,
252 const OutputCB
& output_cb
) {
253 DCHECK(task_runner_
->BelongsToCurrentThread());
254 DCHECK(config
.IsValidConfig());
255 DCHECK(!config
.is_encrypted());
256 DCHECK(decode_cb_
.is_null());
258 InitCB bound_init_cb
= BindToCurrentLoop(init_cb
);
260 if (!ConfigureDecoder(config
)) {
261 bound_init_cb
.Run(false);
268 output_cb_
= BindToCurrentLoop(output_cb
);
269 bound_init_cb
.Run(true);
272 static vpx_codec_ctx
* InitializeVpxContext(vpx_codec_ctx
* context
,
273 const VideoDecoderConfig
& config
) {
274 context
= new vpx_codec_ctx();
275 vpx_codec_dec_cfg_t vpx_config
= {0};
276 vpx_config
.w
= config
.coded_size().width();
277 vpx_config
.h
= config
.coded_size().height();
278 vpx_config
.threads
= GetThreadCount(config
);
280 vpx_codec_err_t status
= vpx_codec_dec_init(context
,
281 config
.codec() == kCodecVP9
?
286 if (status
!= VPX_CODEC_OK
) {
287 LOG(ERROR
) << "vpx_codec_dec_init failed, status=" << status
;
294 bool VpxVideoDecoder::ConfigureDecoder(const VideoDecoderConfig
& config
) {
295 if (config
.codec() != kCodecVP8
&& config
.codec() != kCodecVP9
)
298 #if !defined(DISABLE_FFMPEG_VIDEO_DECODERS)
299 // When FFmpegVideoDecoder is available it handles VP8 that doesn't have
300 // alpha, and VpxVideoDecoder will handle VP8 with alpha.
301 if (config
.codec() == kCodecVP8
&& config
.format() != PIXEL_FORMAT_YV12A
)
307 vpx_codec_
= InitializeVpxContext(vpx_codec_
, config
);
311 // We use our own buffers for VP9 so that there is no need to copy data after
313 if (config
.codec() == kCodecVP9
) {
314 memory_pool_
= new MemoryPool();
315 base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
316 memory_pool_
.get(), task_runner_
);
317 if (vpx_codec_set_frame_buffer_functions(vpx_codec_
,
318 &MemoryPool::GetVP9FrameBuffer
,
319 &MemoryPool::ReleaseVP9FrameBuffer
,
320 memory_pool_
.get())) {
321 LOG(ERROR
) << "Failed to configure external buffers.";
326 if (config
.format() == PIXEL_FORMAT_YV12A
) {
327 vpx_codec_alpha_
= InitializeVpxContext(vpx_codec_alpha_
, config
);
328 if (!vpx_codec_alpha_
)
335 void VpxVideoDecoder::CloseDecoder() {
337 vpx_codec_destroy(vpx_codec_
);
340 base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
344 if (vpx_codec_alpha_
) {
345 vpx_codec_destroy(vpx_codec_alpha_
);
346 delete vpx_codec_alpha_
;
347 vpx_codec_alpha_
= NULL
;
351 void VpxVideoDecoder::Decode(const scoped_refptr
<DecoderBuffer
>& buffer
,
352 const DecodeCB
& decode_cb
) {
353 DCHECK(task_runner_
->BelongsToCurrentThread());
354 DCHECK(!decode_cb
.is_null());
355 CHECK_NE(state_
, kUninitialized
);
356 CHECK(decode_cb_
.is_null()) << "Overlapping decodes are not supported.";
358 decode_cb_
= BindToCurrentLoop(decode_cb
);
360 if (state_
== kError
) {
361 base::ResetAndReturn(&decode_cb_
).Run(kDecodeError
);
365 // Return empty frames if decoding has finished.
366 if (state_
== kDecodeFinished
) {
367 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
371 DecodeBuffer(buffer
);
374 void VpxVideoDecoder::Reset(const base::Closure
& closure
) {
375 DCHECK(task_runner_
->BelongsToCurrentThread());
376 DCHECK(decode_cb_
.is_null());
379 task_runner_
->PostTask(FROM_HERE
, closure
);
382 void VpxVideoDecoder::DecodeBuffer(const scoped_refptr
<DecoderBuffer
>& buffer
) {
383 DCHECK(task_runner_
->BelongsToCurrentThread());
384 DCHECK_NE(state_
, kUninitialized
);
385 DCHECK_NE(state_
, kDecodeFinished
);
386 DCHECK_NE(state_
, kError
);
387 DCHECK(!decode_cb_
.is_null());
388 DCHECK(buffer
.get());
390 // Transition to kDecodeFinished on the first end of stream buffer.
391 if (state_
== kNormal
&& buffer
->end_of_stream()) {
392 state_
= kDecodeFinished
;
393 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
397 scoped_refptr
<VideoFrame
> video_frame
;
398 if (!VpxDecode(buffer
, &video_frame
)) {
400 base::ResetAndReturn(&decode_cb_
).Run(kDecodeError
);
404 if (video_frame
.get())
405 output_cb_
.Run(video_frame
);
407 // VideoDecoderShim expects that |decode_cb| is called only after
409 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
412 bool VpxVideoDecoder::VpxDecode(const scoped_refptr
<DecoderBuffer
>& buffer
,
413 scoped_refptr
<VideoFrame
>* video_frame
) {
415 DCHECK(!buffer
->end_of_stream());
417 // Pass |buffer| to libvpx.
418 int64 timestamp
= buffer
->timestamp().InMicroseconds();
419 void* user_priv
= reinterpret_cast<void*>(×tamp
);
422 TRACE_EVENT1("video", "vpx_codec_decode", "timestamp", timestamp
);
423 vpx_codec_err_t status
= vpx_codec_decode(vpx_codec_
,
428 if (status
!= VPX_CODEC_OK
) {
429 LOG(ERROR
) << "vpx_codec_decode() failed, status=" << status
;
434 // Gets pointer to decoded data.
435 vpx_codec_iter_t iter
= NULL
;
436 const vpx_image_t
* vpx_image
= vpx_codec_get_frame(vpx_codec_
, &iter
);
442 if (vpx_image
->user_priv
!= reinterpret_cast<void*>(×tamp
)) {
443 LOG(ERROR
) << "Invalid output timestamp.";
447 const vpx_image_t
* vpx_image_alpha
= NULL
;
448 if (vpx_codec_alpha_
&& buffer
->side_data_size() >= 8) {
449 // Pass alpha data to libvpx.
450 int64 timestamp_alpha
= buffer
->timestamp().InMicroseconds();
451 void* user_priv_alpha
= reinterpret_cast<void*>(×tamp_alpha
);
453 // First 8 bytes of side data is side_data_id in big endian.
454 const uint64 side_data_id
= base::NetToHost64(
455 *(reinterpret_cast<const uint64
*>(buffer
->side_data())));
456 if (side_data_id
== 1) {
458 TRACE_EVENT1("video", "vpx_codec_decode_alpha",
459 "timestamp_alpha", timestamp_alpha
);
460 vpx_codec_err_t status
= vpx_codec_decode(vpx_codec_alpha_
,
461 buffer
->side_data() + 8,
462 buffer
->side_data_size() - 8,
465 if (status
!= VPX_CODEC_OK
) {
466 LOG(ERROR
) << "vpx_codec_decode() failed on alpha, status=" << status
;
471 // Gets pointer to decoded data.
472 vpx_codec_iter_t iter_alpha
= NULL
;
473 vpx_image_alpha
= vpx_codec_get_frame(vpx_codec_alpha_
, &iter_alpha
);
474 if (!vpx_image_alpha
) {
479 if (vpx_image_alpha
->user_priv
!=
480 reinterpret_cast<void*>(×tamp_alpha
)) {
481 LOG(ERROR
) << "Invalid output timestamp on alpha.";
485 if (vpx_image_alpha
->d_h
!= vpx_image
->d_h
||
486 vpx_image_alpha
->d_w
!= vpx_image
->d_w
) {
487 LOG(ERROR
) << "The alpha plane dimensions are not the same as the "
494 CopyVpxImageTo(vpx_image
, vpx_image_alpha
, video_frame
);
495 (*video_frame
)->set_timestamp(base::TimeDelta::FromMicroseconds(timestamp
));
499 void VpxVideoDecoder::CopyVpxImageTo(const vpx_image
* vpx_image
,
500 const struct vpx_image
* vpx_image_alpha
,
501 scoped_refptr
<VideoFrame
>* video_frame
) {
503 CHECK(vpx_image
->fmt
== VPX_IMG_FMT_I420
||
504 vpx_image
->fmt
== VPX_IMG_FMT_YV12
||
505 vpx_image
->fmt
== VPX_IMG_FMT_I444
);
507 VideoPixelFormat codec_format
= PIXEL_FORMAT_YV12
;
508 int uv_rows
= (vpx_image
->d_h
+ 1) / 2;
510 if (vpx_image
->fmt
== VPX_IMG_FMT_I444
) {
511 CHECK(!vpx_codec_alpha_
);
512 codec_format
= PIXEL_FORMAT_YV24
;
513 uv_rows
= vpx_image
->d_h
;
514 } else if (vpx_codec_alpha_
) {
515 codec_format
= PIXEL_FORMAT_YV12A
;
518 // Default to the color space from the config, but if the bistream specifies
519 // one, prefer that instead.
520 ColorSpace color_space
= config_
.color_space();
521 if (vpx_image
->cs
== VPX_CS_BT_709
)
522 color_space
= COLOR_SPACE_HD_REC709
;
523 else if (vpx_image
->cs
== VPX_CS_BT_601
)
524 color_space
= COLOR_SPACE_SD_REC601
;
526 // The mixed |w|/|d_h| in |coded_size| is intentional. Setting the correct
527 // coded width is necessary to allow coalesced memory access, which may avoid
528 // frame copies. Setting the correct coded height however does not have any
529 // benefit, and only risk copying too much data.
530 const gfx::Size
coded_size(vpx_image
->w
, vpx_image
->d_h
);
531 const gfx::Size
visible_size(vpx_image
->d_w
, vpx_image
->d_h
);
533 if (!vpx_codec_alpha_
&& memory_pool_
.get()) {
534 *video_frame
= VideoFrame::WrapExternalYuvData(
536 coded_size
, gfx::Rect(visible_size
), config_
.natural_size(),
537 vpx_image
->stride
[VPX_PLANE_Y
],
538 vpx_image
->stride
[VPX_PLANE_U
],
539 vpx_image
->stride
[VPX_PLANE_V
],
540 vpx_image
->planes
[VPX_PLANE_Y
],
541 vpx_image
->planes
[VPX_PLANE_U
],
542 vpx_image
->planes
[VPX_PLANE_V
],
544 video_frame
->get()->AddDestructionObserver(
545 memory_pool_
->CreateFrameCallback(vpx_image
->fb_priv
));
546 video_frame
->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE
,
551 *video_frame
= frame_pool_
.CreateFrame(
554 gfx::Rect(visible_size
),
555 config_
.natural_size(),
557 video_frame
->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE
,
560 CopyYPlane(vpx_image
->planes
[VPX_PLANE_Y
],
561 vpx_image
->stride
[VPX_PLANE_Y
],
564 CopyUPlane(vpx_image
->planes
[VPX_PLANE_U
],
565 vpx_image
->stride
[VPX_PLANE_U
],
568 CopyVPlane(vpx_image
->planes
[VPX_PLANE_V
],
569 vpx_image
->stride
[VPX_PLANE_V
],
572 if (!vpx_codec_alpha_
)
574 if (!vpx_image_alpha
) {
576 vpx_image
->stride
[VPX_PLANE_Y
], vpx_image
->d_h
, video_frame
->get());
579 CopyAPlane(vpx_image_alpha
->planes
[VPX_PLANE_Y
],
580 vpx_image_alpha
->stride
[VPX_PLANE_Y
],
581 vpx_image_alpha
->d_h
,