1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/pepper/video_decoder_shim.h"
8 #include <GLES2/gl2ext.h>
9 #include <GLES2/gl2extchromium.h>
11 #include "base/bind.h"
12 #include "base/numerics/safe_conversions.h"
13 #include "base/single_thread_task_runner.h"
14 #include "content/public/renderer/render_thread.h"
15 #include "content/renderer/pepper/pepper_video_decoder_host.h"
16 #include "content/renderer/render_thread_impl.h"
17 #include "gpu/command_buffer/client/gles2_implementation.h"
18 #include "media/base/decoder_buffer.h"
19 #include "media/base/limits.h"
20 #include "media/base/video_decoder.h"
21 #include "media/filters/ffmpeg_video_decoder.h"
22 #include "media/filters/vpx_video_decoder.h"
23 #include "media/video/picture.h"
24 #include "media/video/video_decode_accelerator.h"
25 #include "ppapi/c/pp_errors.h"
26 #include "third_party/libyuv/include/libyuv.h"
27 #include "webkit/common/gpu/context_provider_web_context.h"
31 struct VideoDecoderShim::PendingDecode
{
32 PendingDecode(uint32_t decode_id
,
33 const scoped_refptr
<media::DecoderBuffer
>& buffer
);
36 const uint32_t decode_id
;
37 const scoped_refptr
<media::DecoderBuffer
> buffer
;
40 VideoDecoderShim::PendingDecode::PendingDecode(
42 const scoped_refptr
<media::DecoderBuffer
>& buffer
)
43 : decode_id(decode_id
), buffer(buffer
) {
46 VideoDecoderShim::PendingDecode::~PendingDecode() {
49 struct VideoDecoderShim::PendingFrame
{
50 explicit PendingFrame(uint32_t decode_id
);
51 PendingFrame(uint32_t decode_id
,
52 const gfx::Size
& coded_size
,
53 const gfx::Rect
& visible_rect
);
56 const uint32_t decode_id
;
57 const gfx::Size coded_size
;
58 const gfx::Rect visible_rect
;
59 std::vector
<uint8_t> argb_pixels
;
62 // This could be expensive to copy, so guard against that.
63 DISALLOW_COPY_AND_ASSIGN(PendingFrame
);
66 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id
)
67 : decode_id(decode_id
) {
70 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id
,
71 const gfx::Size
& coded_size
,
72 const gfx::Rect
& visible_rect
)
73 : decode_id(decode_id
),
74 coded_size(coded_size
),
75 visible_rect(visible_rect
),
76 argb_pixels(coded_size
.width() * coded_size
.height() * 4) {
79 VideoDecoderShim::PendingFrame::~PendingFrame() {
82 // DecoderImpl runs the underlying VideoDecoder on the media thread, receiving
83 // calls from the VideoDecodeShim on the main thread and sending results back.
84 // This class is constructed on the main thread, but used and destructed on the
86 class VideoDecoderShim::DecoderImpl
{
88 explicit DecoderImpl(const base::WeakPtr
<VideoDecoderShim
>& proxy
);
91 void Initialize(media::VideoDecoderConfig config
);
92 void Decode(uint32_t decode_id
, scoped_refptr
<media::DecoderBuffer
> buffer
);
97 void OnPipelineStatus(media::PipelineStatus status
);
99 void OnDecodeComplete(uint32_t decode_id
, media::VideoDecoder::Status status
);
100 void OnOutputComplete(const scoped_refptr
<media::VideoFrame
>& frame
);
101 void OnResetComplete();
103 // WeakPtr is bound to main_message_loop_. Use only in shim callbacks.
104 base::WeakPtr
<VideoDecoderShim
> shim_
;
105 scoped_ptr
<media::VideoDecoder
> decoder_
;
106 scoped_refptr
<base::MessageLoopProxy
> main_message_loop_
;
107 // Queue of decodes waiting for the decoder.
108 typedef std::queue
<PendingDecode
> PendingDecodeQueue
;
109 PendingDecodeQueue pending_decodes_
;
110 int max_decodes_at_decoder_
;
111 int num_decodes_at_decoder_
;
112 // VideoDecoder returns pictures without information about the decode buffer
113 // that generated it. Save the decode_id from the last decode that completed,
114 // which is close for most decoders, which only decode one buffer at a time.
118 VideoDecoderShim::DecoderImpl::DecoderImpl(
119 const base::WeakPtr
<VideoDecoderShim
>& proxy
)
121 main_message_loop_(base::MessageLoopProxy::current()),
122 max_decodes_at_decoder_(0),
123 num_decodes_at_decoder_(0),
127 VideoDecoderShim::DecoderImpl::~DecoderImpl() {
128 DCHECK(pending_decodes_
.empty());
131 void VideoDecoderShim::DecoderImpl::Initialize(
132 media::VideoDecoderConfig config
) {
134 #if !defined(MEDIA_DISABLE_LIBVPX)
135 if (config
.codec() == media::kCodecVP9
) {
137 new media::VpxVideoDecoder(base::MessageLoopProxy::current()));
141 scoped_ptr
<media::FFmpegVideoDecoder
> ffmpeg_video_decoder(
142 new media::FFmpegVideoDecoder(base::MessageLoopProxy::current()));
143 ffmpeg_video_decoder
->set_decode_nalus(true);
144 decoder_
= ffmpeg_video_decoder
.Pass();
146 max_decodes_at_decoder_
= decoder_
->GetMaxDecodeRequests();
147 // We can use base::Unretained() safely in decoder callbacks because
148 // |decoder_| is owned by DecoderImpl. During Stop(), the |decoder_| will be
149 // destroyed and all outstanding callbacks will be fired.
150 decoder_
->Initialize(
152 true /* low_delay */,
153 base::Bind(&VideoDecoderShim::DecoderImpl::OnPipelineStatus
,
154 base::Unretained(this)),
155 base::Bind(&VideoDecoderShim::DecoderImpl::OnOutputComplete
,
156 base::Unretained(this)));
159 void VideoDecoderShim::DecoderImpl::Decode(
161 scoped_refptr
<media::DecoderBuffer
> buffer
) {
163 pending_decodes_
.push(PendingDecode(decode_id
, buffer
));
167 void VideoDecoderShim::DecoderImpl::Reset() {
169 // Abort all pending decodes.
170 while (!pending_decodes_
.empty()) {
171 const PendingDecode
& decode
= pending_decodes_
.front();
172 scoped_ptr
<PendingFrame
> pending_frame(new PendingFrame(decode
.decode_id
));
173 main_message_loop_
->PostTask(FROM_HERE
,
174 base::Bind(&VideoDecoderShim::OnDecodeComplete
,
176 media::VideoDecoder::kAborted
,
178 pending_decodes_
.pop();
180 decoder_
->Reset(base::Bind(&VideoDecoderShim::DecoderImpl::OnResetComplete
,
181 base::Unretained(this)));
184 void VideoDecoderShim::DecoderImpl::Stop() {
186 // Clear pending decodes now. We don't want OnDecodeComplete to call DoDecode
188 while (!pending_decodes_
.empty())
189 pending_decodes_
.pop();
191 // This instance is deleted once we exit this scope.
194 void VideoDecoderShim::DecoderImpl::OnPipelineStatus(
195 media::PipelineStatus status
) {
198 case media::PIPELINE_OK
:
201 case media::DECODER_ERROR_NOT_SUPPORTED
:
202 result
= PP_ERROR_NOTSUPPORTED
;
205 result
= PP_ERROR_FAILED
;
209 // Calculate how many textures the shim should create.
210 uint32_t shim_texture_pool_size
=
211 max_decodes_at_decoder_
+ media::limits::kMaxVideoFrames
;
212 main_message_loop_
->PostTask(
214 base::Bind(&VideoDecoderShim::OnInitializeComplete
,
217 shim_texture_pool_size
));
220 void VideoDecoderShim::DecoderImpl::DoDecode() {
221 while (!pending_decodes_
.empty() &&
222 num_decodes_at_decoder_
< max_decodes_at_decoder_
) {
223 num_decodes_at_decoder_
++;
224 const PendingDecode
& decode
= pending_decodes_
.front();
227 base::Bind(&VideoDecoderShim::DecoderImpl::OnDecodeComplete
,
228 base::Unretained(this),
230 pending_decodes_
.pop();
234 void VideoDecoderShim::DecoderImpl::OnDecodeComplete(
236 media::VideoDecoder::Status status
) {
237 num_decodes_at_decoder_
--;
238 decode_id_
= decode_id
;
242 case media::VideoDecoder::kOk
:
243 case media::VideoDecoder::kAborted
:
246 case media::VideoDecoder::kDecodeError
:
247 result
= PP_ERROR_RESOURCE_FAILED
;
251 result
= PP_ERROR_FAILED
;
255 main_message_loop_
->PostTask(
258 &VideoDecoderShim::OnDecodeComplete
, shim_
, result
, decode_id
));
263 void VideoDecoderShim::DecoderImpl::OnOutputComplete(
264 const scoped_refptr
<media::VideoFrame
>& frame
) {
265 scoped_ptr
<PendingFrame
> pending_frame
;
266 if (!frame
->end_of_stream()) {
267 pending_frame
.reset(new PendingFrame(
268 decode_id_
, frame
->coded_size(), frame
->visible_rect()));
269 // Convert the VideoFrame pixels to ABGR to match VideoDecodeAccelerator.
270 libyuv::I420ToABGR(frame
->data(media::VideoFrame::kYPlane
),
271 frame
->stride(media::VideoFrame::kYPlane
),
272 frame
->data(media::VideoFrame::kUPlane
),
273 frame
->stride(media::VideoFrame::kUPlane
),
274 frame
->data(media::VideoFrame::kVPlane
),
275 frame
->stride(media::VideoFrame::kVPlane
),
276 &pending_frame
->argb_pixels
.front(),
277 frame
->coded_size().width() * 4,
278 frame
->coded_size().width(),
279 frame
->coded_size().height());
281 pending_frame
.reset(new PendingFrame(decode_id_
));
284 main_message_loop_
->PostTask(FROM_HERE
,
285 base::Bind(&VideoDecoderShim::OnOutputComplete
,
287 base::Passed(&pending_frame
)));
290 void VideoDecoderShim::DecoderImpl::OnResetComplete() {
291 main_message_loop_
->PostTask(
292 FROM_HERE
, base::Bind(&VideoDecoderShim::OnResetComplete
, shim_
));
295 VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost
* host
)
296 : state_(UNINITIALIZED
),
299 RenderThreadImpl::current()->GetMediaThreadTaskRunner()),
301 RenderThreadImpl::current()->SharedMainThreadContextProvider()),
302 texture_pool_size_(0),
303 num_pending_decodes_(0),
304 weak_ptr_factory_(this) {
306 DCHECK(media_task_runner_
.get());
307 DCHECK(context_provider_
.get());
308 decoder_impl_
.reset(new DecoderImpl(weak_ptr_factory_
.GetWeakPtr()));
311 VideoDecoderShim::~VideoDecoderShim() {
312 DCHECK(RenderThreadImpl::current());
313 // Delete any remaining textures.
314 TextureIdMap::iterator it
= texture_id_map_
.begin();
315 for (; it
!= texture_id_map_
.end(); ++it
)
316 DeleteTexture(it
->second
);
317 texture_id_map_
.clear();
319 FlushCommandBuffer();
321 weak_ptr_factory_
.InvalidateWeakPtrs();
322 // No more callbacks from the delegate will be received now.
324 // The callback now holds the only reference to the DecoderImpl, which will be
325 // deleted when Stop completes.
326 media_task_runner_
->PostTask(
328 base::Bind(&VideoDecoderShim::DecoderImpl::Stop
,
329 base::Owned(decoder_impl_
.release())));
332 bool VideoDecoderShim::Initialize(
333 media::VideoCodecProfile profile
,
334 media::VideoDecodeAccelerator::Client
* client
) {
335 DCHECK_EQ(client
, host_
);
336 DCHECK(RenderThreadImpl::current());
337 DCHECK_EQ(state_
, UNINITIALIZED
);
338 media::VideoCodec codec
= media::kUnknownVideoCodec
;
339 if (profile
<= media::H264PROFILE_MAX
)
340 codec
= media::kCodecH264
;
341 else if (profile
<= media::VP8PROFILE_MAX
)
342 codec
= media::kCodecVP8
;
343 else if (profile
<= media::VP9PROFILE_MAX
)
344 codec
= media::kCodecVP9
;
345 DCHECK_NE(codec
, media::kUnknownVideoCodec
);
347 media::VideoDecoderConfig
config(
350 media::VideoFrame::YV12
,
351 gfx::Size(32, 24), // Small sizes that won't fail.
354 NULL
/* extra_data */, // TODO(bbudge) Verify this isn't needed.
355 0 /* extra_data_size */,
356 false /* decryption */);
358 media_task_runner_
->PostTask(
360 base::Bind(&VideoDecoderShim::DecoderImpl::Initialize
,
361 base::Unretained(decoder_impl_
.get()),
363 // Return success, even though we are asynchronous, to mimic
364 // media::VideoDecodeAccelerator.
368 void VideoDecoderShim::Decode(const media::BitstreamBuffer
& bitstream_buffer
) {
369 DCHECK(RenderThreadImpl::current());
370 DCHECK_EQ(state_
, DECODING
);
372 // We need the address of the shared memory, so we can copy the buffer.
373 const uint8_t* buffer
= host_
->DecodeIdToAddress(bitstream_buffer
.id());
376 media_task_runner_
->PostTask(
379 &VideoDecoderShim::DecoderImpl::Decode
,
380 base::Unretained(decoder_impl_
.get()),
381 bitstream_buffer
.id(),
382 media::DecoderBuffer::CopyFrom(buffer
, bitstream_buffer
.size())));
383 num_pending_decodes_
++;
386 void VideoDecoderShim::AssignPictureBuffers(
387 const std::vector
<media::PictureBuffer
>& buffers
) {
388 DCHECK(RenderThreadImpl::current());
389 DCHECK_EQ(state_
, DECODING
);
390 if (buffers
.empty()) {
394 DCHECK_EQ(buffers
.size(), pending_texture_mailboxes_
.size());
395 GLuint num_textures
= base::checked_cast
<GLuint
>(buffers
.size());
396 std::vector
<uint32_t> local_texture_ids(num_textures
);
397 gpu::gles2::GLES2Interface
* gles2
= context_provider_
->ContextGL();
398 for (uint32_t i
= 0; i
< num_textures
; i
++) {
399 local_texture_ids
[i
] = gles2
->CreateAndConsumeTextureCHROMIUM(
400 GL_TEXTURE_2D
, pending_texture_mailboxes_
[i
].name
);
401 // Map the plugin texture id to the local texture id.
402 uint32_t plugin_texture_id
= buffers
[i
].texture_id();
403 texture_id_map_
[plugin_texture_id
] = local_texture_ids
[i
];
404 available_textures_
.insert(plugin_texture_id
);
406 pending_texture_mailboxes_
.clear();
410 void VideoDecoderShim::ReusePictureBuffer(int32 picture_buffer_id
) {
411 DCHECK(RenderThreadImpl::current());
412 uint32_t texture_id
= static_cast<uint32_t>(picture_buffer_id
);
413 if (textures_to_dismiss_
.find(texture_id
) != textures_to_dismiss_
.end()) {
414 DismissTexture(texture_id
);
415 } else if (texture_id_map_
.find(texture_id
) != texture_id_map_
.end()) {
416 available_textures_
.insert(texture_id
);
423 void VideoDecoderShim::Flush() {
424 DCHECK(RenderThreadImpl::current());
425 DCHECK_EQ(state_
, DECODING
);
429 void VideoDecoderShim::Reset() {
430 DCHECK(RenderThreadImpl::current());
431 DCHECK_EQ(state_
, DECODING
);
433 media_task_runner_
->PostTask(
435 base::Bind(&VideoDecoderShim::DecoderImpl::Reset
,
436 base::Unretained(decoder_impl_
.get())));
439 void VideoDecoderShim::Destroy() {
440 // This will be called, but our destructor does the actual work.
443 void VideoDecoderShim::OnInitializeComplete(int32_t result
,
444 uint32_t texture_pool_size
) {
445 DCHECK(RenderThreadImpl::current());
448 if (result
== PP_OK
) {
450 texture_pool_size_
= texture_pool_size
;
453 host_
->OnInitializeComplete(result
);
456 void VideoDecoderShim::OnDecodeComplete(int32_t result
, uint32_t decode_id
) {
457 DCHECK(RenderThreadImpl::current());
460 if (result
== PP_ERROR_RESOURCE_FAILED
) {
461 host_
->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE
);
465 num_pending_decodes_
--;
466 completed_decodes_
.push(decode_id
);
468 // If frames are being queued because we're out of textures, don't notify
469 // the host that decode has completed. This exerts "back pressure" to keep
470 // the host from sending buffers that will cause pending_frames_ to grow.
471 if (pending_frames_
.empty())
472 NotifyCompletedDecodes();
475 void VideoDecoderShim::OnOutputComplete(scoped_ptr
<PendingFrame
> frame
) {
476 DCHECK(RenderThreadImpl::current());
479 if (!frame
->argb_pixels
.empty()) {
480 if (texture_size_
!= frame
->coded_size
) {
481 // If the size has changed, all current textures must be dismissed. Add
482 // all textures to |textures_to_dismiss_| and dismiss any that aren't in
483 // use by the plugin. We will dismiss the rest as they are recycled.
484 for (TextureIdMap::const_iterator it
= texture_id_map_
.begin();
485 it
!= texture_id_map_
.end();
487 textures_to_dismiss_
.insert(it
->second
);
489 for (TextureIdSet::const_iterator it
= available_textures_
.begin();
490 it
!= available_textures_
.end();
494 available_textures_
.clear();
495 FlushCommandBuffer();
497 DCHECK(pending_texture_mailboxes_
.empty());
498 for (uint32_t i
= 0; i
< texture_pool_size_
; i
++)
499 pending_texture_mailboxes_
.push_back(gpu::Mailbox::Generate());
501 host_
->RequestTextures(texture_pool_size_
,
504 pending_texture_mailboxes_
);
505 texture_size_
= frame
->coded_size
;
508 pending_frames_
.push(linked_ptr
<PendingFrame
>(frame
.release()));
513 void VideoDecoderShim::SendPictures() {
514 DCHECK(RenderThreadImpl::current());
516 while (!pending_frames_
.empty() && !available_textures_
.empty()) {
517 const linked_ptr
<PendingFrame
>& frame
= pending_frames_
.front();
519 TextureIdSet::iterator it
= available_textures_
.begin();
520 uint32_t texture_id
= *it
;
521 available_textures_
.erase(it
);
523 uint32_t local_texture_id
= texture_id_map_
[texture_id
];
524 gpu::gles2::GLES2Interface
* gles2
= context_provider_
->ContextGL();
525 gles2
->ActiveTexture(GL_TEXTURE0
);
526 gles2
->BindTexture(GL_TEXTURE_2D
, local_texture_id
);
527 gles2
->TexImage2D(GL_TEXTURE_2D
,
530 texture_size_
.width(),
531 texture_size_
.height(),
535 &frame
->argb_pixels
.front());
538 media::Picture(texture_id
, frame
->decode_id
, frame
->visible_rect
));
539 pending_frames_
.pop();
542 FlushCommandBuffer();
544 if (pending_frames_
.empty()) {
545 // If frames aren't backing up, notify the host of any completed decodes so
546 // it can send more buffers.
547 NotifyCompletedDecodes();
549 if (state_
== FLUSHING
&& !num_pending_decodes_
) {
551 host_
->NotifyFlushDone();
556 void VideoDecoderShim::OnResetComplete() {
557 DCHECK(RenderThreadImpl::current());
560 while (!pending_frames_
.empty())
561 pending_frames_
.pop();
562 NotifyCompletedDecodes();
564 // Dismiss any old textures now.
565 while (!textures_to_dismiss_
.empty())
566 DismissTexture(*textures_to_dismiss_
.begin());
569 host_
->NotifyResetDone();
572 void VideoDecoderShim::NotifyCompletedDecodes() {
573 while (!completed_decodes_
.empty()) {
574 host_
->NotifyEndOfBitstreamBuffer(completed_decodes_
.front());
575 completed_decodes_
.pop();
579 void VideoDecoderShim::DismissTexture(uint32_t texture_id
) {
581 textures_to_dismiss_
.erase(texture_id
);
582 DCHECK(texture_id_map_
.find(texture_id
) != texture_id_map_
.end());
583 DeleteTexture(texture_id_map_
[texture_id
]);
584 texture_id_map_
.erase(texture_id
);
585 host_
->DismissPictureBuffer(texture_id
);
588 void VideoDecoderShim::DeleteTexture(uint32_t texture_id
) {
589 gpu::gles2::GLES2Interface
* gles2
= context_provider_
->ContextGL();
590 gles2
->DeleteTextures(1, &texture_id
);
593 void VideoDecoderShim::FlushCommandBuffer() {
594 context_provider_
->ContextGL()->Flush();
597 } // namespace content