1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/vpx_video_decoder.h"
11 #include "base/bind.h"
12 #include "base/callback_helpers.h"
13 #include "base/command_line.h"
14 #include "base/location.h"
15 #include "base/logging.h"
16 #include "base/single_thread_task_runner.h"
17 #include "base/stl_util.h"
18 #include "base/strings/string_number_conversions.h"
19 #include "base/sys_byteorder.h"
20 #include "base/sys_info.h"
21 #include "base/trace_event/memory_allocator_dump.h"
22 #include "base/trace_event/memory_dump_manager.h"
23 #include "base/trace_event/memory_dump_provider.h"
24 #include "base/trace_event/process_memory_dump.h"
25 #include "base/trace_event/trace_event.h"
26 #include "media/base/bind_to_current_loop.h"
27 #include "media/base/decoder_buffer.h"
28 #include "media/base/limits.h"
29 #include "media/base/media_switches.h"
30 #include "media/base/pipeline.h"
31 #include "media/base/video_util.h"
33 // Include libvpx header files.
34 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
35 // backwards compatibility for legacy applications using the library.
36 #define VPX_CODEC_DISABLE_COMPAT 1
38 #include "third_party/libvpx/source/libvpx/vpx/vpx_decoder.h"
39 #include "third_party/libvpx/source/libvpx/vpx/vpx_frame_buffer.h"
40 #include "third_party/libvpx/source/libvpx/vpx/vp8dx.h"
45 // Always try to use three threads for video decoding. There is little reason
46 // not to since current day CPUs tend to be multi-core and we measured
47 // performance benefits on older machines such as P4s with hyperthreading.
48 static const int kDecodeThreads
= 2;
49 static const int kMaxDecodeThreads
= 16;
51 // Returns the number of threads.
52 static int GetThreadCount(const VideoDecoderConfig
& config
) {
53 // Refer to http://crbug.com/93932 for tsan suppressions on decoding.
54 int decode_threads
= kDecodeThreads
;
56 const base::CommandLine
* cmd_line
= base::CommandLine::ForCurrentProcess();
57 std::string
threads(cmd_line
->GetSwitchValueASCII(switches::kVideoThreads
));
58 if (threads
.empty() || !base::StringToInt(threads
, &decode_threads
)) {
59 if (config
.codec() == kCodecVP9
) {
60 // For VP9 decode when using the default thread count, increase the number
61 // of decode threads to equal the maximum number of tiles possible for
62 // higher resolution streams.
63 if (config
.coded_size().width() >= 2048)
65 else if (config
.coded_size().width() >= 1024)
69 decode_threads
= std::min(decode_threads
,
70 base::SysInfo::NumberOfProcessors());
71 return decode_threads
;
74 decode_threads
= std::max(decode_threads
, 0);
75 decode_threads
= std::min(decode_threads
, kMaxDecodeThreads
);
76 return decode_threads
;
79 class VpxVideoDecoder::MemoryPool
80 : public base::RefCountedThreadSafe
<VpxVideoDecoder::MemoryPool
>,
81 public base::trace_event::MemoryDumpProvider
{
85 // Callback that will be called by libvpx when it needs a frame buffer.
87 // |user_priv| Private data passed to libvpx (pointer to memory pool).
88 // |min_size| Minimum size needed by libvpx to decompress the next frame.
89 // |fb| Pointer to the frame buffer to update.
90 // Returns 0 on success. Returns < 0 on failure.
91 static int32
GetVP9FrameBuffer(void* user_priv
, size_t min_size
,
92 vpx_codec_frame_buffer
* fb
);
94 // Callback that will be called by libvpx when the frame buffer is no longer
95 // being used by libvpx. Parameters:
96 // |user_priv| Private data passed to libvpx (pointer to memory pool).
97 // |fb| Pointer to the frame buffer that's being released.
98 static int32
ReleaseVP9FrameBuffer(void *user_priv
,
99 vpx_codec_frame_buffer
*fb
);
101 // Generates a "no_longer_needed" closure that holds a reference
103 base::Closure
CreateFrameCallback(void* fb_priv_data
);
105 bool OnMemoryDump(const base::trace_event::MemoryDumpArgs
& args
,
106 base::trace_event::ProcessMemoryDump
* pmd
) override
;
109 friend class base::RefCountedThreadSafe
<VpxVideoDecoder::MemoryPool
>;
110 ~MemoryPool() override
;
112 // Reference counted frame buffers used for VP9 decoding. Reference counting
113 // is done manually because both chromium and libvpx has to release this
114 // before a buffer can be re-used.
115 struct VP9FrameBuffer
{
116 VP9FrameBuffer() : ref_cnt(0) {}
117 std::vector
<uint8
> data
;
121 // Gets the next available frame buffer for use by libvpx.
122 VP9FrameBuffer
* GetFreeFrameBuffer(size_t min_size
);
124 // Method that gets called when a VideoFrame that references this pool gets
126 void OnVideoFrameDestroyed(VP9FrameBuffer
* frame_buffer
);
128 // Frame buffers to be used by libvpx for VP9 Decoding.
129 std::vector
<VP9FrameBuffer
*> frame_buffers_
;
131 DISALLOW_COPY_AND_ASSIGN(MemoryPool
);
134 VpxVideoDecoder::MemoryPool::MemoryPool() {}
136 VpxVideoDecoder::MemoryPool::~MemoryPool() {
137 STLDeleteElements(&frame_buffers_
);
140 VpxVideoDecoder::MemoryPool::VP9FrameBuffer
*
141 VpxVideoDecoder::MemoryPool::GetFreeFrameBuffer(size_t min_size
) {
142 // Check if a free frame buffer exists.
144 for (; i
< frame_buffers_
.size(); ++i
) {
145 if (frame_buffers_
[i
]->ref_cnt
== 0)
149 if (i
== frame_buffers_
.size()) {
150 // Create a new frame buffer.
151 frame_buffers_
.push_back(new VP9FrameBuffer());
154 // Resize the frame buffer if necessary.
155 if (frame_buffers_
[i
]->data
.size() < min_size
)
156 frame_buffers_
[i
]->data
.resize(min_size
);
157 return frame_buffers_
[i
];
160 int32
VpxVideoDecoder::MemoryPool::GetVP9FrameBuffer(
161 void* user_priv
, size_t min_size
, vpx_codec_frame_buffer
* fb
) {
165 VpxVideoDecoder::MemoryPool
* memory_pool
=
166 static_cast<VpxVideoDecoder::MemoryPool
*>(user_priv
);
168 VP9FrameBuffer
* fb_to_use
= memory_pool
->GetFreeFrameBuffer(min_size
);
169 if (fb_to_use
== NULL
)
172 fb
->data
= &fb_to_use
->data
[0];
173 fb
->size
= fb_to_use
->data
.size();
174 ++fb_to_use
->ref_cnt
;
176 // Set the frame buffer's private data to point at the external frame buffer.
177 fb
->priv
= static_cast<void*>(fb_to_use
);
181 int32
VpxVideoDecoder::MemoryPool::ReleaseVP9FrameBuffer(
182 void *user_priv
, vpx_codec_frame_buffer
*fb
) {
183 VP9FrameBuffer
* frame_buffer
= static_cast<VP9FrameBuffer
*>(fb
->priv
);
184 --frame_buffer
->ref_cnt
;
188 base::Closure
VpxVideoDecoder::MemoryPool::CreateFrameCallback(
189 void* fb_priv_data
) {
190 VP9FrameBuffer
* frame_buffer
= static_cast<VP9FrameBuffer
*>(fb_priv_data
);
191 ++frame_buffer
->ref_cnt
;
192 return BindToCurrentLoop(
193 base::Bind(&MemoryPool::OnVideoFrameDestroyed
, this,
197 bool VpxVideoDecoder::MemoryPool::OnMemoryDump(
198 const base::trace_event::MemoryDumpArgs
& args
,
199 base::trace_event::ProcessMemoryDump
* pmd
) {
200 base::trace_event::MemoryAllocatorDump
* memory_dump
=
201 pmd
->CreateAllocatorDump("media/vpx/memory_pool");
202 base::trace_event::MemoryAllocatorDump
* used_memory_dump
=
203 pmd
->CreateAllocatorDump("media/vpx/memory_pool/used");
205 pmd
->AddSuballocation(memory_dump
->guid(),
206 base::trace_event::MemoryDumpManager::GetInstance()
207 ->system_allocator_pool_name());
208 size_t bytes_used
= 0;
209 size_t bytes_reserved
= 0;
210 for (const VP9FrameBuffer
* frame_buffer
: frame_buffers_
) {
211 if (frame_buffer
->ref_cnt
) {
212 bytes_used
+= frame_buffer
->data
.size();
214 bytes_reserved
+= frame_buffer
->data
.size();
217 memory_dump
->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize
,
218 base::trace_event::MemoryAllocatorDump::kUnitsBytes
,
220 used_memory_dump
->AddScalar(
221 base::trace_event::MemoryAllocatorDump::kNameSize
,
222 base::trace_event::MemoryAllocatorDump::kUnitsBytes
, bytes_used
);
227 void VpxVideoDecoder::MemoryPool::OnVideoFrameDestroyed(
228 VP9FrameBuffer
* frame_buffer
) {
229 --frame_buffer
->ref_cnt
;
232 VpxVideoDecoder::VpxVideoDecoder(
233 const scoped_refptr
<base::SingleThreadTaskRunner
>& task_runner
)
234 : task_runner_(task_runner
),
235 state_(kUninitialized
),
237 vpx_codec_alpha_(NULL
) {}
239 VpxVideoDecoder::~VpxVideoDecoder() {
240 DCHECK(task_runner_
->BelongsToCurrentThread());
244 std::string
VpxVideoDecoder::GetDisplayName() const {
245 return "VpxVideoDecoder";
248 void VpxVideoDecoder::Initialize(const VideoDecoderConfig
& config
,
250 const InitCB
& init_cb
,
251 const OutputCB
& output_cb
) {
252 DCHECK(task_runner_
->BelongsToCurrentThread());
253 DCHECK(config
.IsValidConfig());
254 DCHECK(!config
.is_encrypted());
255 DCHECK(decode_cb_
.is_null());
257 InitCB bound_init_cb
= BindToCurrentLoop(init_cb
);
259 if (!ConfigureDecoder(config
)) {
260 bound_init_cb
.Run(false);
267 output_cb_
= BindToCurrentLoop(output_cb
);
268 bound_init_cb
.Run(true);
271 static vpx_codec_ctx
* InitializeVpxContext(vpx_codec_ctx
* context
,
272 const VideoDecoderConfig
& config
) {
273 context
= new vpx_codec_ctx();
274 vpx_codec_dec_cfg_t vpx_config
= {0};
275 vpx_config
.w
= config
.coded_size().width();
276 vpx_config
.h
= config
.coded_size().height();
277 vpx_config
.threads
= GetThreadCount(config
);
279 vpx_codec_err_t status
= vpx_codec_dec_init(context
,
280 config
.codec() == kCodecVP9
?
285 if (status
!= VPX_CODEC_OK
) {
286 LOG(ERROR
) << "vpx_codec_dec_init failed, status=" << status
;
293 bool VpxVideoDecoder::ConfigureDecoder(const VideoDecoderConfig
& config
) {
294 if (config
.codec() != kCodecVP8
&& config
.codec() != kCodecVP9
)
297 #if !defined(DISABLE_FFMPEG_VIDEO_DECODERS)
298 // When FFmpegVideoDecoder is available it handles VP8 that doesn't have
299 // alpha, and VpxVideoDecoder will handle VP8 with alpha.
300 if (config
.codec() == kCodecVP8
&& config
.format() != PIXEL_FORMAT_YV12A
)
306 vpx_codec_
= InitializeVpxContext(vpx_codec_
, config
);
310 // We use our own buffers for VP9 so that there is no need to copy data after
312 if (config
.codec() == kCodecVP9
) {
313 memory_pool_
= new MemoryPool();
314 base::trace_event::MemoryDumpManager::GetInstance()->RegisterDumpProvider(
315 memory_pool_
.get(), task_runner_
);
316 if (vpx_codec_set_frame_buffer_functions(vpx_codec_
,
317 &MemoryPool::GetVP9FrameBuffer
,
318 &MemoryPool::ReleaseVP9FrameBuffer
,
319 memory_pool_
.get())) {
320 LOG(ERROR
) << "Failed to configure external buffers.";
325 if (config
.format() == PIXEL_FORMAT_YV12A
) {
326 vpx_codec_alpha_
= InitializeVpxContext(vpx_codec_alpha_
, config
);
327 if (!vpx_codec_alpha_
)
334 void VpxVideoDecoder::CloseDecoder() {
336 vpx_codec_destroy(vpx_codec_
);
339 base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
343 if (vpx_codec_alpha_
) {
344 vpx_codec_destroy(vpx_codec_alpha_
);
345 delete vpx_codec_alpha_
;
346 vpx_codec_alpha_
= NULL
;
350 void VpxVideoDecoder::Decode(const scoped_refptr
<DecoderBuffer
>& buffer
,
351 const DecodeCB
& decode_cb
) {
352 DCHECK(task_runner_
->BelongsToCurrentThread());
353 DCHECK(!decode_cb
.is_null());
354 CHECK_NE(state_
, kUninitialized
);
355 CHECK(decode_cb_
.is_null()) << "Overlapping decodes are not supported.";
357 decode_cb_
= BindToCurrentLoop(decode_cb
);
359 if (state_
== kError
) {
360 base::ResetAndReturn(&decode_cb_
).Run(kDecodeError
);
364 // Return empty frames if decoding has finished.
365 if (state_
== kDecodeFinished
) {
366 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
370 DecodeBuffer(buffer
);
373 void VpxVideoDecoder::Reset(const base::Closure
& closure
) {
374 DCHECK(task_runner_
->BelongsToCurrentThread());
375 DCHECK(decode_cb_
.is_null());
378 task_runner_
->PostTask(FROM_HERE
, closure
);
381 void VpxVideoDecoder::DecodeBuffer(const scoped_refptr
<DecoderBuffer
>& buffer
) {
382 DCHECK(task_runner_
->BelongsToCurrentThread());
383 DCHECK_NE(state_
, kUninitialized
);
384 DCHECK_NE(state_
, kDecodeFinished
);
385 DCHECK_NE(state_
, kError
);
386 DCHECK(!decode_cb_
.is_null());
387 DCHECK(buffer
.get());
389 // Transition to kDecodeFinished on the first end of stream buffer.
390 if (state_
== kNormal
&& buffer
->end_of_stream()) {
391 state_
= kDecodeFinished
;
392 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
396 scoped_refptr
<VideoFrame
> video_frame
;
397 if (!VpxDecode(buffer
, &video_frame
)) {
399 base::ResetAndReturn(&decode_cb_
).Run(kDecodeError
);
403 if (video_frame
.get())
404 output_cb_
.Run(video_frame
);
406 // VideoDecoderShim expects that |decode_cb| is called only after
408 base::ResetAndReturn(&decode_cb_
).Run(kOk
);
411 bool VpxVideoDecoder::VpxDecode(const scoped_refptr
<DecoderBuffer
>& buffer
,
412 scoped_refptr
<VideoFrame
>* video_frame
) {
414 DCHECK(!buffer
->end_of_stream());
416 // Pass |buffer| to libvpx.
417 int64 timestamp
= buffer
->timestamp().InMicroseconds();
418 void* user_priv
= reinterpret_cast<void*>(×tamp
);
421 TRACE_EVENT1("video", "vpx_codec_decode", "timestamp", timestamp
);
422 vpx_codec_err_t status
= vpx_codec_decode(vpx_codec_
,
427 if (status
!= VPX_CODEC_OK
) {
428 LOG(ERROR
) << "vpx_codec_decode() failed, status=" << status
;
433 // Gets pointer to decoded data.
434 vpx_codec_iter_t iter
= NULL
;
435 const vpx_image_t
* vpx_image
= vpx_codec_get_frame(vpx_codec_
, &iter
);
441 if (vpx_image
->user_priv
!= reinterpret_cast<void*>(×tamp
)) {
442 LOG(ERROR
) << "Invalid output timestamp.";
446 const vpx_image_t
* vpx_image_alpha
= NULL
;
447 if (vpx_codec_alpha_
&& buffer
->side_data_size() >= 8) {
448 // Pass alpha data to libvpx.
449 int64 timestamp_alpha
= buffer
->timestamp().InMicroseconds();
450 void* user_priv_alpha
= reinterpret_cast<void*>(×tamp_alpha
);
452 // First 8 bytes of side data is side_data_id in big endian.
453 const uint64 side_data_id
= base::NetToHost64(
454 *(reinterpret_cast<const uint64
*>(buffer
->side_data())));
455 if (side_data_id
== 1) {
457 TRACE_EVENT1("video", "vpx_codec_decode_alpha",
458 "timestamp_alpha", timestamp_alpha
);
459 vpx_codec_err_t status
= vpx_codec_decode(vpx_codec_alpha_
,
460 buffer
->side_data() + 8,
461 buffer
->side_data_size() - 8,
464 if (status
!= VPX_CODEC_OK
) {
465 LOG(ERROR
) << "vpx_codec_decode() failed on alpha, status=" << status
;
470 // Gets pointer to decoded data.
471 vpx_codec_iter_t iter_alpha
= NULL
;
472 vpx_image_alpha
= vpx_codec_get_frame(vpx_codec_alpha_
, &iter_alpha
);
473 if (!vpx_image_alpha
) {
478 if (vpx_image_alpha
->user_priv
!=
479 reinterpret_cast<void*>(×tamp_alpha
)) {
480 LOG(ERROR
) << "Invalid output timestamp on alpha.";
484 if (vpx_image_alpha
->d_h
!= vpx_image
->d_h
||
485 vpx_image_alpha
->d_w
!= vpx_image
->d_w
) {
486 LOG(ERROR
) << "The alpha plane dimensions are not the same as the "
493 CopyVpxImageTo(vpx_image
, vpx_image_alpha
, video_frame
);
494 (*video_frame
)->set_timestamp(base::TimeDelta::FromMicroseconds(timestamp
));
498 void VpxVideoDecoder::CopyVpxImageTo(const vpx_image
* vpx_image
,
499 const struct vpx_image
* vpx_image_alpha
,
500 scoped_refptr
<VideoFrame
>* video_frame
) {
502 CHECK(vpx_image
->fmt
== VPX_IMG_FMT_I420
||
503 vpx_image
->fmt
== VPX_IMG_FMT_YV12
||
504 vpx_image
->fmt
== VPX_IMG_FMT_I444
);
506 VideoPixelFormat codec_format
= PIXEL_FORMAT_YV12
;
507 int uv_rows
= (vpx_image
->d_h
+ 1) / 2;
509 if (vpx_image
->fmt
== VPX_IMG_FMT_I444
) {
510 CHECK(!vpx_codec_alpha_
);
511 codec_format
= PIXEL_FORMAT_YV24
;
512 uv_rows
= vpx_image
->d_h
;
513 } else if (vpx_codec_alpha_
) {
514 codec_format
= PIXEL_FORMAT_YV12A
;
517 // Default to the color space from the config, but if the bistream specifies
518 // one, prefer that instead.
519 ColorSpace color_space
= config_
.color_space();
520 if (vpx_image
->cs
== VPX_CS_BT_709
)
521 color_space
= COLOR_SPACE_HD_REC709
;
522 else if (vpx_image
->cs
== VPX_CS_BT_601
)
523 color_space
= COLOR_SPACE_SD_REC601
;
525 // The mixed |w|/|d_h| in |coded_size| is intentional. Setting the correct
526 // coded width is necessary to allow coalesced memory access, which may avoid
527 // frame copies. Setting the correct coded height however does not have any
528 // benefit, and only risk copying too much data.
529 const gfx::Size
coded_size(vpx_image
->w
, vpx_image
->d_h
);
530 const gfx::Size
visible_size(vpx_image
->d_w
, vpx_image
->d_h
);
532 if (!vpx_codec_alpha_
&& memory_pool_
.get()) {
533 *video_frame
= VideoFrame::WrapExternalYuvData(
535 coded_size
, gfx::Rect(visible_size
), config_
.natural_size(),
536 vpx_image
->stride
[VPX_PLANE_Y
],
537 vpx_image
->stride
[VPX_PLANE_U
],
538 vpx_image
->stride
[VPX_PLANE_V
],
539 vpx_image
->planes
[VPX_PLANE_Y
],
540 vpx_image
->planes
[VPX_PLANE_U
],
541 vpx_image
->planes
[VPX_PLANE_V
],
543 video_frame
->get()->AddDestructionObserver(
544 memory_pool_
->CreateFrameCallback(vpx_image
->fb_priv
));
545 video_frame
->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE
,
550 *video_frame
= frame_pool_
.CreateFrame(
553 gfx::Rect(visible_size
),
554 config_
.natural_size(),
556 video_frame
->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE
,
559 CopyYPlane(vpx_image
->planes
[VPX_PLANE_Y
],
560 vpx_image
->stride
[VPX_PLANE_Y
],
563 CopyUPlane(vpx_image
->planes
[VPX_PLANE_U
],
564 vpx_image
->stride
[VPX_PLANE_U
],
567 CopyVPlane(vpx_image
->planes
[VPX_PLANE_V
],
568 vpx_image
->stride
[VPX_PLANE_V
],
571 if (!vpx_codec_alpha_
)
573 if (!vpx_image_alpha
) {
575 vpx_image
->stride
[VPX_PLANE_Y
], vpx_image
->d_h
, video_frame
->get());
578 CopyAPlane(vpx_image_alpha
->planes
[VPX_PLANE_Y
],
579 vpx_image_alpha
->stride
[VPX_PLANE_Y
],
580 vpx_image_alpha
->d_h
,