1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/pepper/video_decoder_shim.h"
8 #include <GLES2/gl2ext.h>
9 #include <GLES2/gl2extchromium.h>
11 #include "base/bind.h"
12 #include "base/numerics/safe_conversions.h"
13 #include "base/single_thread_task_runner.h"
14 #include "cc/blink/context_provider_web_context.h"
15 #include "content/public/renderer/render_thread.h"
16 #include "content/renderer/pepper/pepper_video_decoder_host.h"
17 #include "content/renderer/render_thread_impl.h"
18 #include "gpu/command_buffer/client/gles2_implementation.h"
19 #include "media/base/decoder_buffer.h"
20 #include "media/base/limits.h"
21 #include "media/base/video_decoder.h"
22 #include "media/filters/ffmpeg_video_decoder.h"
23 #include "media/filters/skcanvas_video_renderer.h"
24 #include "media/filters/vpx_video_decoder.h"
25 #include "media/video/picture.h"
26 #include "media/video/video_decode_accelerator.h"
27 #include "ppapi/c/pp_errors.h"
31 struct VideoDecoderShim::PendingDecode
{
32 PendingDecode(uint32_t decode_id
,
33 const scoped_refptr
<media::DecoderBuffer
>& buffer
);
36 const uint32_t decode_id
;
37 const scoped_refptr
<media::DecoderBuffer
> buffer
;
40 VideoDecoderShim::PendingDecode::PendingDecode(
42 const scoped_refptr
<media::DecoderBuffer
>& buffer
)
43 : decode_id(decode_id
), buffer(buffer
) {
46 VideoDecoderShim::PendingDecode::~PendingDecode() {
49 struct VideoDecoderShim::PendingFrame
{
50 explicit PendingFrame(uint32_t decode_id
);
51 PendingFrame(uint32_t decode_id
,
52 const gfx::Size
& coded_size
,
53 const gfx::Rect
& visible_rect
);
56 const uint32_t decode_id
;
57 const gfx::Size coded_size
;
58 const gfx::Rect visible_rect
;
59 std::vector
<uint8_t> argb_pixels
;
62 // This could be expensive to copy, so guard against that.
63 DISALLOW_COPY_AND_ASSIGN(PendingFrame
);
66 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id
)
67 : decode_id(decode_id
) {
70 VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id
,
71 const gfx::Size
& coded_size
,
72 const gfx::Rect
& visible_rect
)
73 : decode_id(decode_id
),
74 coded_size(coded_size
),
75 visible_rect(visible_rect
),
76 argb_pixels(coded_size
.width() * coded_size
.height() * 4) {
79 VideoDecoderShim::PendingFrame::~PendingFrame() {
82 // DecoderImpl runs the underlying VideoDecoder on the media thread, receiving
83 // calls from the VideoDecodeShim on the main thread and sending results back.
84 // This class is constructed on the main thread, but used and destructed on the
86 class VideoDecoderShim::DecoderImpl
{
88 explicit DecoderImpl(const base::WeakPtr
<VideoDecoderShim
>& proxy
);
91 void Initialize(media::VideoDecoderConfig config
);
92 void Decode(uint32_t decode_id
, scoped_refptr
<media::DecoderBuffer
> buffer
);
97 void OnPipelineStatus(media::PipelineStatus status
);
99 void OnDecodeComplete(media::VideoDecoder::Status status
);
100 void OnOutputComplete(const scoped_refptr
<media::VideoFrame
>& frame
);
101 void OnResetComplete();
103 // WeakPtr is bound to main_message_loop_. Use only in shim callbacks.
104 base::WeakPtr
<VideoDecoderShim
> shim_
;
105 scoped_ptr
<media::VideoDecoder
> decoder_
;
106 scoped_refptr
<base::MessageLoopProxy
> main_message_loop_
;
107 // Queue of decodes waiting for the decoder.
108 typedef std::queue
<PendingDecode
> PendingDecodeQueue
;
109 PendingDecodeQueue pending_decodes_
;
110 bool awaiting_decoder_
;
111 // VideoDecoder returns pictures without information about the decode buffer
112 // that generated it, but VideoDecoder implementations used in this class
113 // (media::FFmpegVideoDecoder and media::VpxVideoDecoder) always generate
114 // corresponding frames before decode is finished. |decode_id_| is used to
115 // store id of the current buffer while Decode() call is pending.
119 VideoDecoderShim::DecoderImpl::DecoderImpl(
120 const base::WeakPtr
<VideoDecoderShim
>& proxy
)
122 main_message_loop_(base::MessageLoopProxy::current()),
123 awaiting_decoder_(false),
127 VideoDecoderShim::DecoderImpl::~DecoderImpl() {
128 DCHECK(pending_decodes_
.empty());
131 void VideoDecoderShim::DecoderImpl::Initialize(
132 media::VideoDecoderConfig config
) {
134 #if !defined(MEDIA_DISABLE_LIBVPX)
135 if (config
.codec() == media::kCodecVP9
) {
137 new media::VpxVideoDecoder(base::MessageLoopProxy::current()));
141 scoped_ptr
<media::FFmpegVideoDecoder
> ffmpeg_video_decoder(
142 new media::FFmpegVideoDecoder(base::MessageLoopProxy::current()));
143 ffmpeg_video_decoder
->set_decode_nalus(true);
144 decoder_
= ffmpeg_video_decoder
.Pass();
147 // VpxVideoDecoder and FFmpegVideoDecoder support only one pending Decode()
149 DCHECK_EQ(decoder_
->GetMaxDecodeRequests(), 1);
151 // We can use base::Unretained() safely in decoder callbacks because
152 // |decoder_| is owned by DecoderImpl. During Stop(), the |decoder_| will be
153 // destroyed and all outstanding callbacks will be fired.
154 decoder_
->Initialize(
156 true /* low_delay */,
157 base::Bind(&VideoDecoderShim::DecoderImpl::OnPipelineStatus
,
158 base::Unretained(this)),
159 base::Bind(&VideoDecoderShim::DecoderImpl::OnOutputComplete
,
160 base::Unretained(this)));
163 void VideoDecoderShim::DecoderImpl::Decode(
165 scoped_refptr
<media::DecoderBuffer
> buffer
) {
167 pending_decodes_
.push(PendingDecode(decode_id
, buffer
));
171 void VideoDecoderShim::DecoderImpl::Reset() {
173 // Abort all pending decodes.
174 while (!pending_decodes_
.empty()) {
175 const PendingDecode
& decode
= pending_decodes_
.front();
176 scoped_ptr
<PendingFrame
> pending_frame(new PendingFrame(decode
.decode_id
));
177 main_message_loop_
->PostTask(FROM_HERE
,
178 base::Bind(&VideoDecoderShim::OnDecodeComplete
,
180 media::VideoDecoder::kAborted
,
182 pending_decodes_
.pop();
184 decoder_
->Reset(base::Bind(&VideoDecoderShim::DecoderImpl::OnResetComplete
,
185 base::Unretained(this)));
188 void VideoDecoderShim::DecoderImpl::Stop() {
190 // Clear pending decodes now. We don't want OnDecodeComplete to call DoDecode
192 while (!pending_decodes_
.empty())
193 pending_decodes_
.pop();
195 // This instance is deleted once we exit this scope.
198 void VideoDecoderShim::DecoderImpl::OnPipelineStatus(
199 media::PipelineStatus status
) {
202 case media::PIPELINE_OK
:
205 case media::DECODER_ERROR_NOT_SUPPORTED
:
206 result
= PP_ERROR_NOTSUPPORTED
;
209 result
= PP_ERROR_FAILED
;
213 // Calculate how many textures the shim should create.
214 uint32_t shim_texture_pool_size
= media::limits::kMaxVideoFrames
+ 1;
215 main_message_loop_
->PostTask(
217 base::Bind(&VideoDecoderShim::OnInitializeComplete
,
220 shim_texture_pool_size
));
223 void VideoDecoderShim::DecoderImpl::DoDecode() {
224 if (pending_decodes_
.empty() || awaiting_decoder_
)
227 awaiting_decoder_
= true;
228 const PendingDecode
& decode
= pending_decodes_
.front();
229 decode_id_
= decode
.decode_id
;
230 decoder_
->Decode(decode
.buffer
,
231 base::Bind(&VideoDecoderShim::DecoderImpl::OnDecodeComplete
,
232 base::Unretained(this)));
233 pending_decodes_
.pop();
236 void VideoDecoderShim::DecoderImpl::OnDecodeComplete(
237 media::VideoDecoder::Status status
) {
238 DCHECK(awaiting_decoder_
);
239 awaiting_decoder_
= false;
243 case media::VideoDecoder::kOk
:
244 case media::VideoDecoder::kAborted
:
247 case media::VideoDecoder::kDecodeError
:
248 result
= PP_ERROR_RESOURCE_FAILED
;
252 result
= PP_ERROR_FAILED
;
256 main_message_loop_
->PostTask(
259 &VideoDecoderShim::OnDecodeComplete
, shim_
, result
, decode_id_
));
264 void VideoDecoderShim::DecoderImpl::OnOutputComplete(
265 const scoped_refptr
<media::VideoFrame
>& frame
) {
266 // Software decoders are expected to generated frames only when a Decode()
268 DCHECK(awaiting_decoder_
);
270 scoped_ptr
<PendingFrame
> pending_frame
;
271 if (!frame
->end_of_stream()) {
272 pending_frame
.reset(new PendingFrame(
273 decode_id_
, frame
->coded_size(), frame
->visible_rect()));
274 // Convert the VideoFrame pixels to ABGR to match VideoDecodeAccelerator.
275 media::SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
277 &pending_frame
->argb_pixels
.front(),
278 frame
->coded_size().width() * 4);
280 pending_frame
.reset(new PendingFrame(decode_id_
));
283 main_message_loop_
->PostTask(FROM_HERE
,
284 base::Bind(&VideoDecoderShim::OnOutputComplete
,
286 base::Passed(&pending_frame
)));
289 void VideoDecoderShim::DecoderImpl::OnResetComplete() {
290 main_message_loop_
->PostTask(
291 FROM_HERE
, base::Bind(&VideoDecoderShim::OnResetComplete
, shim_
));
294 VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost
* host
)
295 : state_(UNINITIALIZED
),
298 RenderThreadImpl::current()->GetMediaThreadTaskRunner()),
300 RenderThreadImpl::current()->SharedMainThreadContextProvider()),
301 texture_pool_size_(0),
302 num_pending_decodes_(0),
303 weak_ptr_factory_(this) {
305 DCHECK(media_task_runner_
.get());
306 DCHECK(context_provider_
.get());
307 decoder_impl_
.reset(new DecoderImpl(weak_ptr_factory_
.GetWeakPtr()));
310 VideoDecoderShim::~VideoDecoderShim() {
311 DCHECK(RenderThreadImpl::current());
312 // Delete any remaining textures.
313 TextureIdMap::iterator it
= texture_id_map_
.begin();
314 for (; it
!= texture_id_map_
.end(); ++it
)
315 DeleteTexture(it
->second
);
316 texture_id_map_
.clear();
318 FlushCommandBuffer();
320 weak_ptr_factory_
.InvalidateWeakPtrs();
321 // No more callbacks from the delegate will be received now.
323 // The callback now holds the only reference to the DecoderImpl, which will be
324 // deleted when Stop completes.
325 media_task_runner_
->PostTask(
327 base::Bind(&VideoDecoderShim::DecoderImpl::Stop
,
328 base::Owned(decoder_impl_
.release())));
331 bool VideoDecoderShim::Initialize(
332 media::VideoCodecProfile profile
,
333 media::VideoDecodeAccelerator::Client
* client
) {
334 DCHECK_EQ(client
, host_
);
335 DCHECK(RenderThreadImpl::current());
336 DCHECK_EQ(state_
, UNINITIALIZED
);
337 media::VideoCodec codec
= media::kUnknownVideoCodec
;
338 if (profile
<= media::H264PROFILE_MAX
)
339 codec
= media::kCodecH264
;
340 else if (profile
<= media::VP8PROFILE_MAX
)
341 codec
= media::kCodecVP8
;
342 else if (profile
<= media::VP9PROFILE_MAX
)
343 codec
= media::kCodecVP9
;
344 DCHECK_NE(codec
, media::kUnknownVideoCodec
);
346 media::VideoDecoderConfig
config(
349 media::VideoFrame::YV12
,
350 gfx::Size(32, 24), // Small sizes that won't fail.
353 NULL
/* extra_data */, // TODO(bbudge) Verify this isn't needed.
354 0 /* extra_data_size */,
355 false /* decryption */);
357 media_task_runner_
->PostTask(
359 base::Bind(&VideoDecoderShim::DecoderImpl::Initialize
,
360 base::Unretained(decoder_impl_
.get()),
362 // Return success, even though we are asynchronous, to mimic
363 // media::VideoDecodeAccelerator.
367 void VideoDecoderShim::Decode(const media::BitstreamBuffer
& bitstream_buffer
) {
368 DCHECK(RenderThreadImpl::current());
369 DCHECK_EQ(state_
, DECODING
);
371 // We need the address of the shared memory, so we can copy the buffer.
372 const uint8_t* buffer
= host_
->DecodeIdToAddress(bitstream_buffer
.id());
375 media_task_runner_
->PostTask(
378 &VideoDecoderShim::DecoderImpl::Decode
,
379 base::Unretained(decoder_impl_
.get()),
380 bitstream_buffer
.id(),
381 media::DecoderBuffer::CopyFrom(buffer
, bitstream_buffer
.size())));
382 num_pending_decodes_
++;
385 void VideoDecoderShim::AssignPictureBuffers(
386 const std::vector
<media::PictureBuffer
>& buffers
) {
387 DCHECK(RenderThreadImpl::current());
388 DCHECK_EQ(state_
, DECODING
);
389 if (buffers
.empty()) {
393 DCHECK_EQ(buffers
.size(), pending_texture_mailboxes_
.size());
394 GLuint num_textures
= base::checked_cast
<GLuint
>(buffers
.size());
395 std::vector
<uint32_t> local_texture_ids(num_textures
);
396 gpu::gles2::GLES2Interface
* gles2
= context_provider_
->ContextGL();
397 for (uint32_t i
= 0; i
< num_textures
; i
++) {
398 local_texture_ids
[i
] = gles2
->CreateAndConsumeTextureCHROMIUM(
399 GL_TEXTURE_2D
, pending_texture_mailboxes_
[i
].name
);
400 // Map the plugin texture id to the local texture id.
401 uint32_t plugin_texture_id
= buffers
[i
].texture_id();
402 texture_id_map_
[plugin_texture_id
] = local_texture_ids
[i
];
403 available_textures_
.insert(plugin_texture_id
);
405 pending_texture_mailboxes_
.clear();
409 void VideoDecoderShim::ReusePictureBuffer(int32 picture_buffer_id
) {
410 DCHECK(RenderThreadImpl::current());
411 uint32_t texture_id
= static_cast<uint32_t>(picture_buffer_id
);
412 if (textures_to_dismiss_
.find(texture_id
) != textures_to_dismiss_
.end()) {
413 DismissTexture(texture_id
);
414 } else if (texture_id_map_
.find(texture_id
) != texture_id_map_
.end()) {
415 available_textures_
.insert(texture_id
);
422 void VideoDecoderShim::Flush() {
423 DCHECK(RenderThreadImpl::current());
424 DCHECK_EQ(state_
, DECODING
);
428 void VideoDecoderShim::Reset() {
429 DCHECK(RenderThreadImpl::current());
430 DCHECK_EQ(state_
, DECODING
);
432 media_task_runner_
->PostTask(
434 base::Bind(&VideoDecoderShim::DecoderImpl::Reset
,
435 base::Unretained(decoder_impl_
.get())));
438 void VideoDecoderShim::Destroy() {
442 void VideoDecoderShim::OnInitializeComplete(int32_t result
,
443 uint32_t texture_pool_size
) {
444 DCHECK(RenderThreadImpl::current());
447 if (result
== PP_OK
) {
449 texture_pool_size_
= texture_pool_size
;
452 host_
->OnInitializeComplete(result
);
455 void VideoDecoderShim::OnDecodeComplete(int32_t result
, uint32_t decode_id
) {
456 DCHECK(RenderThreadImpl::current());
459 if (result
== PP_ERROR_RESOURCE_FAILED
) {
460 host_
->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE
);
464 num_pending_decodes_
--;
465 completed_decodes_
.push(decode_id
);
467 // If frames are being queued because we're out of textures, don't notify
468 // the host that decode has completed. This exerts "back pressure" to keep
469 // the host from sending buffers that will cause pending_frames_ to grow.
470 if (pending_frames_
.empty())
471 NotifyCompletedDecodes();
474 void VideoDecoderShim::OnOutputComplete(scoped_ptr
<PendingFrame
> frame
) {
475 DCHECK(RenderThreadImpl::current());
478 if (!frame
->argb_pixels
.empty()) {
479 if (texture_size_
!= frame
->coded_size
) {
480 // If the size has changed, all current textures must be dismissed. Add
481 // all textures to |textures_to_dismiss_| and dismiss any that aren't in
482 // use by the plugin. We will dismiss the rest as they are recycled.
483 for (TextureIdMap::const_iterator it
= texture_id_map_
.begin();
484 it
!= texture_id_map_
.end();
486 textures_to_dismiss_
.insert(it
->second
);
488 for (TextureIdSet::const_iterator it
= available_textures_
.begin();
489 it
!= available_textures_
.end();
493 available_textures_
.clear();
494 FlushCommandBuffer();
496 DCHECK(pending_texture_mailboxes_
.empty());
497 for (uint32_t i
= 0; i
< texture_pool_size_
; i
++)
498 pending_texture_mailboxes_
.push_back(gpu::Mailbox::Generate());
500 host_
->RequestTextures(texture_pool_size_
,
503 pending_texture_mailboxes_
);
504 texture_size_
= frame
->coded_size
;
507 pending_frames_
.push(linked_ptr
<PendingFrame
>(frame
.release()));
512 void VideoDecoderShim::SendPictures() {
513 DCHECK(RenderThreadImpl::current());
515 while (!pending_frames_
.empty() && !available_textures_
.empty()) {
516 const linked_ptr
<PendingFrame
>& frame
= pending_frames_
.front();
518 TextureIdSet::iterator it
= available_textures_
.begin();
519 uint32_t texture_id
= *it
;
520 available_textures_
.erase(it
);
522 uint32_t local_texture_id
= texture_id_map_
[texture_id
];
523 gpu::gles2::GLES2Interface
* gles2
= context_provider_
->ContextGL();
524 gles2
->ActiveTexture(GL_TEXTURE0
);
525 gles2
->BindTexture(GL_TEXTURE_2D
, local_texture_id
);
526 gles2
->TexImage2D(GL_TEXTURE_2D
,
529 texture_size_
.width(),
530 texture_size_
.height(),
534 &frame
->argb_pixels
.front());
536 host_
->PictureReady(media::Picture(texture_id
, frame
->decode_id
,
537 frame
->visible_rect
, false));
538 pending_frames_
.pop();
541 FlushCommandBuffer();
543 if (pending_frames_
.empty()) {
544 // If frames aren't backing up, notify the host of any completed decodes so
545 // it can send more buffers.
546 NotifyCompletedDecodes();
548 if (state_
== FLUSHING
&& !num_pending_decodes_
) {
550 host_
->NotifyFlushDone();
555 void VideoDecoderShim::OnResetComplete() {
556 DCHECK(RenderThreadImpl::current());
559 while (!pending_frames_
.empty())
560 pending_frames_
.pop();
561 NotifyCompletedDecodes();
563 // Dismiss any old textures now.
564 while (!textures_to_dismiss_
.empty())
565 DismissTexture(*textures_to_dismiss_
.begin());
568 host_
->NotifyResetDone();
571 void VideoDecoderShim::NotifyCompletedDecodes() {
572 while (!completed_decodes_
.empty()) {
573 host_
->NotifyEndOfBitstreamBuffer(completed_decodes_
.front());
574 completed_decodes_
.pop();
578 void VideoDecoderShim::DismissTexture(uint32_t texture_id
) {
580 textures_to_dismiss_
.erase(texture_id
);
581 DCHECK(texture_id_map_
.find(texture_id
) != texture_id_map_
.end());
582 DeleteTexture(texture_id_map_
[texture_id
]);
583 texture_id_map_
.erase(texture_id
);
584 host_
->DismissPictureBuffer(texture_id
);
587 void VideoDecoderShim::DeleteTexture(uint32_t texture_id
) {
588 gpu::gles2::GLES2Interface
* gles2
= context_provider_
->ContextGL();
589 gles2
->DeleteTextures(1, &texture_id
);
592 void VideoDecoderShim::FlushCommandBuffer() {
593 context_provider_
->ContextGL()->Flush();
596 } // namespace content