1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/blink/skcanvas_video_renderer.h"
7 #include "gpu/GLES2/gl2extchromium.h"
8 #include "gpu/command_buffer/client/gles2_interface.h"
9 #include "gpu/command_buffer/common/mailbox_holder.h"
10 #include "media/base/video_frame.h"
11 #include "media/base/yuv_convert.h"
12 #include "skia/ext/refptr.h"
13 #include "third_party/libyuv/include/libyuv.h"
14 #include "third_party/skia/include/core/SkCanvas.h"
15 #include "third_party/skia/include/core/SkImage.h"
16 #include "third_party/skia/include/core/SkImageGenerator.h"
17 #include "third_party/skia/include/gpu/GrContext.h"
18 #include "third_party/skia/include/gpu/GrPaint.h"
19 #include "third_party/skia/include/gpu/GrTexture.h"
20 #include "third_party/skia/include/gpu/GrTextureProvider.h"
21 #include "ui/gfx/geometry/rect_f.h"
23 // Skia internal format depends on a platform. On Android it is ABGR, on others
25 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
27 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
28 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
29 #define LIBYUV_I420ALPHA_TO_ARGB libyuv::I420AlphaToARGB
30 #define LIBYUV_J420_TO_ARGB libyuv::J420ToARGB
31 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
33 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR
34 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR
35 #define LIBYUV_I420ALPHA_TO_ARGB libyuv::I420AlphaToABGR
36 #define LIBYUV_J420_TO_ARGB libyuv::J420ToABGR
38 #error Unexpected Skia ARGB_8888 layout!
45 // This class keeps the last image drawn.
46 // We delete the temporary resource if it is not used for 3 seconds.
47 const int kTemporaryResourceDeletionDelay
= 3; // Seconds;
49 bool CheckColorSpace(const VideoFrame
* video_frame
, ColorSpace color_space
) {
51 return video_frame
->metadata()->GetInteger(
52 VideoFrameMetadata::COLOR_SPACE
, &result
) &&
53 result
== color_space
;
56 class SyncPointClientImpl
: public VideoFrame::SyncPointClient
{
58 explicit SyncPointClientImpl(gpu::gles2::GLES2Interface
* gl
) : gl_(gl
) {}
59 ~SyncPointClientImpl() override
{}
60 uint32
InsertSyncPoint() override
{ return gl_
->InsertSyncPointCHROMIUM(); }
61 void WaitSyncPoint(uint32 sync_point
) override
{
62 gl_
->WaitSyncPointCHROMIUM(sync_point
);
66 gpu::gles2::GLES2Interface
* gl_
;
68 DISALLOW_IMPLICIT_CONSTRUCTORS(SyncPointClientImpl
);
71 skia::RefPtr
<SkImage
> NewSkImageFromVideoFrameYUVTextures(
72 const VideoFrame
* video_frame
,
73 const Context3D
& context_3d
) {
74 // Support only TEXTURE_YUV_420.
75 DCHECK(video_frame
->HasTextures());
76 DCHECK_EQ(media::PIXEL_FORMAT_I420
, video_frame
->format());
77 DCHECK_EQ(3u, media::VideoFrame::NumPlanes(video_frame
->format()));
79 gpu::gles2::GLES2Interface
* gl
= context_3d
.gl
;
81 gfx::Size ya_tex_size
= video_frame
->coded_size();
82 gfx::Size
uv_tex_size((ya_tex_size
.width() + 1) / 2,
83 (ya_tex_size
.height() + 1) / 2);
85 unsigned source_textures
[3] = {0};
86 for (size_t i
= 0; i
< media::VideoFrame::NumPlanes(video_frame
->format());
88 // Get the texture from the mailbox and wrap it in a GrTexture.
89 const gpu::MailboxHolder
& mailbox_holder
= video_frame
->mailbox_holder(i
);
90 DCHECK(mailbox_holder
.texture_target
== GL_TEXTURE_2D
||
91 mailbox_holder
.texture_target
== GL_TEXTURE_EXTERNAL_OES
||
92 mailbox_holder
.texture_target
== GL_TEXTURE_RECTANGLE_ARB
);
93 gl
->WaitSyncPointCHROMIUM(mailbox_holder
.sync_point
);
94 source_textures
[i
] = gl
->CreateAndConsumeTextureCHROMIUM(
95 mailbox_holder
.texture_target
, mailbox_holder
.mailbox
.name
);
97 // TODO(dcastagna): avoid this copy once Skia supports native textures
98 // with a texture target different than TEXTURE_2D.
100 if (mailbox_holder
.texture_target
!= GL_TEXTURE_2D
) {
101 unsigned texture_copy
= 0;
102 gl
->GenTextures(1, &texture_copy
);
103 DCHECK(texture_copy
);
104 gl
->BindTexture(GL_TEXTURE_2D
, texture_copy
);
105 gl
->CopyTextureCHROMIUM(GL_TEXTURE_2D
, source_textures
[i
], texture_copy
,
106 GL_RGB
, GL_UNSIGNED_BYTE
, false, true, false);
108 gl
->DeleteTextures(1, &source_textures
[i
]);
109 source_textures
[i
] = texture_copy
;
112 GrBackendObject handles
[3] = {
113 source_textures
[0], source_textures
[1], source_textures
[2]};
115 SkISize yuvSizes
[] = {
116 {ya_tex_size
.width(), ya_tex_size
.height()},
117 {uv_tex_size
.width(), uv_tex_size
.height()},
118 {uv_tex_size
.width(), uv_tex_size
.height()},
121 SkYUVColorSpace color_space
= kRec601_SkYUVColorSpace
;
122 if (CheckColorSpace(video_frame
, media::COLOR_SPACE_JPEG
))
123 color_space
= kJPEG_SkYUVColorSpace
;
124 else if (CheckColorSpace(video_frame
, media::COLOR_SPACE_HD_REC709
))
125 color_space
= kRec709_SkYUVColorSpace
;
127 SkImage
* img
= SkImage::NewFromYUVTexturesCopy(context_3d
.gr_context
,
128 color_space
, handles
, yuvSizes
,
129 kTopLeft_GrSurfaceOrigin
);
130 gl
->DeleteTextures(3, source_textures
);
131 return skia::AdoptRef(img
);
134 // Creates a SkImage from a |video_frame| backed by native resources.
135 // The SkImage will take ownership of the underlying resource.
136 skia::RefPtr
<SkImage
> NewSkImageFromVideoFrameNative(
137 VideoFrame
* video_frame
,
138 const Context3D
& context_3d
) {
139 DCHECK(PIXEL_FORMAT_ARGB
== video_frame
->format() ||
140 PIXEL_FORMAT_NV12
== video_frame
->format() ||
141 PIXEL_FORMAT_UYVY
== video_frame
->format());
143 const gpu::MailboxHolder
& mailbox_holder
= video_frame
->mailbox_holder(0);
144 DCHECK(mailbox_holder
.texture_target
== GL_TEXTURE_2D
||
145 mailbox_holder
.texture_target
== GL_TEXTURE_RECTANGLE_ARB
||
146 mailbox_holder
.texture_target
== GL_TEXTURE_EXTERNAL_OES
)
147 << mailbox_holder
.texture_target
;
149 gpu::gles2::GLES2Interface
* gl
= context_3d
.gl
;
150 unsigned source_texture
= 0;
151 if (mailbox_holder
.texture_target
!= GL_TEXTURE_2D
) {
152 // TODO(dcastagna): At the moment Skia doesn't support targets different
153 // than GL_TEXTURE_2D. Avoid this copy once
154 // https://code.google.com/p/skia/issues/detail?id=3868 is addressed.
155 gl
->GenTextures(1, &source_texture
);
156 DCHECK(source_texture
);
157 gl
->BindTexture(GL_TEXTURE_2D
, source_texture
);
158 SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
159 gl
, video_frame
, source_texture
, GL_RGBA
, GL_UNSIGNED_BYTE
, true,
162 gl
->WaitSyncPointCHROMIUM(mailbox_holder
.sync_point
);
163 source_texture
= gl
->CreateAndConsumeTextureCHROMIUM(
164 mailbox_holder
.texture_target
, mailbox_holder
.mailbox
.name
);
166 GrBackendTextureDesc desc
;
167 desc
.fFlags
= kRenderTarget_GrBackendTextureFlag
;
168 desc
.fOrigin
= kTopLeft_GrSurfaceOrigin
;
169 desc
.fWidth
= video_frame
->coded_size().width();
170 desc
.fHeight
= video_frame
->coded_size().height();
171 desc
.fConfig
= kRGBA_8888_GrPixelConfig
;
172 desc
.fTextureHandle
= source_texture
;
173 return skia::AdoptRef(
174 SkImage::NewFromAdoptedTexture(context_3d
.gr_context
, desc
));
177 } // anonymous namespace
179 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
180 class VideoImageGenerator
: public SkImageGenerator
{
182 VideoImageGenerator(const scoped_refptr
<VideoFrame
>& frame
)
184 SkImageInfo::MakeN32Premul(frame
->visible_rect().width(),
185 frame
->visible_rect().height())),
187 DCHECK(!frame_
->HasTextures());
189 ~VideoImageGenerator() override
{}
192 bool onGetPixels(const SkImageInfo
& info
,
196 int* ctable_count
) override
{
197 // If skia couldn't do the YUV conversion on GPU, we will on CPU.
198 SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(frame_
.get(), pixels
,
203 bool onGetYUV8Planes(SkISize sizes
[3],
206 SkYUVColorSpace
* color_space
) override
{
207 if (!media::IsYuvPlanar(frame_
->format()) ||
208 // TODO(rileya): Skia currently doesn't support YUVA conversion. Remove
209 // this case once it does. As-is we will fall back on the pure-software
210 // path in this case.
211 frame_
->format() == PIXEL_FORMAT_YV12A
) {
216 if (CheckColorSpace(frame_
.get(), COLOR_SPACE_JPEG
))
217 *color_space
= kJPEG_SkYUVColorSpace
;
218 else if (CheckColorSpace(frame_
.get(), COLOR_SPACE_HD_REC709
))
219 *color_space
= kRec709_SkYUVColorSpace
;
221 *color_space
= kRec601_SkYUVColorSpace
;
224 for (int plane
= VideoFrame::kYPlane
; plane
<= VideoFrame::kVPlane
;
227 const gfx::Size size
=
228 VideoFrame::PlaneSize(frame_
->format(), plane
,
229 gfx::Size(frame_
->visible_rect().width(),
230 frame_
->visible_rect().height()));
231 sizes
[plane
].set(size
.width(), size
.height());
233 if (row_bytes
&& planes
) {
236 (frame_
->format() == media::PIXEL_FORMAT_YV16
) ? 0 : 1;
237 if (plane
== VideoFrame::kYPlane
) {
238 offset
= (frame_
->stride(VideoFrame::kYPlane
) *
239 frame_
->visible_rect().y()) +
240 frame_
->visible_rect().x();
242 offset
= (frame_
->stride(VideoFrame::kUPlane
) *
243 (frame_
->visible_rect().y() >> y_shift
)) +
244 (frame_
->visible_rect().x() >> 1);
247 // Copy the frame to the supplied memory.
248 // TODO: Find a way (API change?) to avoid this copy.
249 char* out_line
= static_cast<char*>(planes
[plane
]);
250 int out_line_stride
= row_bytes
[plane
];
251 uint8
* in_line
= frame_
->data(plane
) + offset
;
252 int in_line_stride
= frame_
->stride(plane
);
253 int plane_height
= sizes
[plane
].height();
254 if (in_line_stride
== out_line_stride
) {
255 memcpy(out_line
, in_line
, plane_height
* in_line_stride
);
257 // Different line padding so need to copy one line at a time.
258 int bytes_to_copy_per_line
= out_line_stride
< in_line_stride
261 for (int line_no
= 0; line_no
< plane_height
; line_no
++) {
262 memcpy(out_line
, in_line
, bytes_to_copy_per_line
);
263 in_line
+= in_line_stride
;
264 out_line
+= out_line_stride
;
273 scoped_refptr
<VideoFrame
> frame_
;
275 DISALLOW_IMPLICIT_CONSTRUCTORS(VideoImageGenerator
);
278 SkCanvasVideoRenderer::SkCanvasVideoRenderer()
279 : last_image_deleting_timer_(
281 base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay
),
283 &SkCanvasVideoRenderer::ResetCache
) {}
285 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {
289 void SkCanvasVideoRenderer::Paint(const scoped_refptr
<VideoFrame
>& video_frame
,
291 const gfx::RectF
& dest_rect
,
293 SkXfermode::Mode mode
,
294 VideoRotation video_rotation
,
295 const Context3D
& context_3d
) {
301 dest
.set(dest_rect
.x(), dest_rect
.y(), dest_rect
.right(), dest_rect
.bottom());
304 paint
.setAlpha(alpha
);
306 // Paint black rectangle if there isn't a frame available or the
307 // frame has an unexpected format.
308 if (!video_frame
.get() || video_frame
->natural_size().IsEmpty() ||
309 !(media::IsYuvPlanar(video_frame
->format()) ||
310 video_frame
->HasTextures())) {
311 canvas
->drawRect(dest
, paint
);
316 gpu::gles2::GLES2Interface
* gl
= context_3d
.gl
;
318 if (!last_image_
|| video_frame
->timestamp() != last_timestamp_
) {
320 // Generate a new image.
321 // Note: Skia will hold onto |video_frame| via |video_generator| only when
322 // |video_frame| is software.
323 // Holding |video_frame| longer than this call when using GPUVideoDecoder
324 // could cause problems since the pool of VideoFrames has a fixed size.
325 if (video_frame
->HasTextures()) {
326 DCHECK(context_3d
.gr_context
);
328 if (media::VideoFrame::NumPlanes(video_frame
->format()) == 1) {
330 NewSkImageFromVideoFrameNative(video_frame
.get(), context_3d
);
333 NewSkImageFromVideoFrameYUVTextures(video_frame
.get(), context_3d
);
336 auto video_generator
= new VideoImageGenerator(video_frame
);
337 last_image_
= skia::AdoptRef(SkImage::NewFromGenerator(video_generator
));
339 if (!last_image_
) // Couldn't create the SkImage.
341 last_timestamp_
= video_frame
->timestamp();
343 last_image_deleting_timer_
.Reset();
345 paint
.setXfermodeMode(mode
);
346 paint
.setFilterQuality(kLow_SkFilterQuality
);
348 const bool need_transform
=
349 video_rotation
!= VIDEO_ROTATION_0
||
350 dest_rect
.size() != video_frame
->visible_rect().size() ||
351 !dest_rect
.origin().IsOrigin();
352 if (need_transform
) {
355 SkFloatToScalar(dest_rect
.x() + (dest_rect
.width() * 0.5f
)),
356 SkFloatToScalar(dest_rect
.y() + (dest_rect
.height() * 0.5f
)));
357 SkScalar angle
= SkFloatToScalar(0.0f
);
358 switch (video_rotation
) {
359 case VIDEO_ROTATION_0
:
361 case VIDEO_ROTATION_90
:
362 angle
= SkFloatToScalar(90.0f
);
364 case VIDEO_ROTATION_180
:
365 angle
= SkFloatToScalar(180.0f
);
367 case VIDEO_ROTATION_270
:
368 angle
= SkFloatToScalar(270.0f
);
371 canvas
->rotate(angle
);
373 gfx::SizeF rotated_dest_size
= dest_rect
.size();
374 if (video_rotation
== VIDEO_ROTATION_90
||
375 video_rotation
== VIDEO_ROTATION_270
) {
377 gfx::SizeF(rotated_dest_size
.height(), rotated_dest_size
.width());
380 SkFloatToScalar(rotated_dest_size
.width() / last_image_
->width()),
381 SkFloatToScalar(rotated_dest_size
.height() / last_image_
->height()));
382 canvas
->translate(-SkFloatToScalar(last_image_
->width() * 0.5f
),
383 -SkFloatToScalar(last_image_
->height() * 0.5f
));
385 canvas
->drawImage(last_image_
.get(), 0, 0, &paint
);
389 // Make sure to flush so we can remove the videoframe from the generator.
392 if (video_frame
->HasTextures()) {
394 SyncPointClientImpl
client(gl
);
395 video_frame
->UpdateReleaseSyncPoint(&client
);
399 void SkCanvasVideoRenderer::Copy(const scoped_refptr
<VideoFrame
>& video_frame
,
401 const Context3D
& context_3d
) {
402 Paint(video_frame
, canvas
, gfx::RectF(video_frame
->visible_rect()), 0xff,
403 SkXfermode::kSrc_Mode
, media::VIDEO_ROTATION_0
, context_3d
);
407 void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
408 const VideoFrame
* video_frame
,
411 if (!video_frame
->IsMappable()) {
412 NOTREACHED() << "Cannot extract pixels from non-CPU frame formats.";
415 if (!media::IsYuvPlanar(video_frame
->format())) {
416 NOTREACHED() << "Non YUV formats are not supported";
419 DCHECK_EQ(video_frame
->stride(VideoFrame::kUPlane
),
420 video_frame
->stride(VideoFrame::kVPlane
));
422 switch (video_frame
->format()) {
423 case PIXEL_FORMAT_YV12
:
424 case PIXEL_FORMAT_I420
:
425 if (CheckColorSpace(video_frame
, COLOR_SPACE_JPEG
)) {
427 video_frame
->visible_data(VideoFrame::kYPlane
),
428 video_frame
->stride(VideoFrame::kYPlane
),
429 video_frame
->visible_data(VideoFrame::kUPlane
),
430 video_frame
->stride(VideoFrame::kUPlane
),
431 video_frame
->visible_data(VideoFrame::kVPlane
),
432 video_frame
->stride(VideoFrame::kVPlane
),
433 static_cast<uint8
*>(rgb_pixels
),
435 video_frame
->visible_rect().width(),
436 video_frame
->visible_rect().height());
437 } else if (CheckColorSpace(video_frame
, COLOR_SPACE_HD_REC709
)) {
438 ConvertYUVToRGB32(video_frame
->visible_data(VideoFrame::kYPlane
),
439 video_frame
->visible_data(VideoFrame::kUPlane
),
440 video_frame
->visible_data(VideoFrame::kVPlane
),
441 static_cast<uint8
*>(rgb_pixels
),
442 video_frame
->visible_rect().width(),
443 video_frame
->visible_rect().height(),
444 video_frame
->stride(VideoFrame::kYPlane
),
445 video_frame
->stride(VideoFrame::kUPlane
), row_bytes
,
449 video_frame
->visible_data(VideoFrame::kYPlane
),
450 video_frame
->stride(VideoFrame::kYPlane
),
451 video_frame
->visible_data(VideoFrame::kUPlane
),
452 video_frame
->stride(VideoFrame::kUPlane
),
453 video_frame
->visible_data(VideoFrame::kVPlane
),
454 video_frame
->stride(VideoFrame::kVPlane
),
455 static_cast<uint8
*>(rgb_pixels
),
457 video_frame
->visible_rect().width(),
458 video_frame
->visible_rect().height());
461 case PIXEL_FORMAT_YV16
:
463 video_frame
->visible_data(VideoFrame::kYPlane
),
464 video_frame
->stride(VideoFrame::kYPlane
),
465 video_frame
->visible_data(VideoFrame::kUPlane
),
466 video_frame
->stride(VideoFrame::kUPlane
),
467 video_frame
->visible_data(VideoFrame::kVPlane
),
468 video_frame
->stride(VideoFrame::kVPlane
),
469 static_cast<uint8
*>(rgb_pixels
),
471 video_frame
->visible_rect().width(),
472 video_frame
->visible_rect().height());
475 case PIXEL_FORMAT_YV12A
:
476 LIBYUV_I420ALPHA_TO_ARGB(
477 video_frame
->visible_data(VideoFrame::kYPlane
),
478 video_frame
->stride(VideoFrame::kYPlane
),
479 video_frame
->visible_data(VideoFrame::kUPlane
),
480 video_frame
->stride(VideoFrame::kUPlane
),
481 video_frame
->visible_data(VideoFrame::kVPlane
),
482 video_frame
->stride(VideoFrame::kVPlane
),
483 video_frame
->visible_data(VideoFrame::kAPlane
),
484 video_frame
->stride(VideoFrame::kAPlane
),
485 static_cast<uint8
*>(rgb_pixels
),
487 video_frame
->visible_rect().width(),
488 video_frame
->visible_rect().height());
491 case PIXEL_FORMAT_YV24
:
493 video_frame
->visible_data(VideoFrame::kYPlane
),
494 video_frame
->stride(VideoFrame::kYPlane
),
495 video_frame
->visible_data(VideoFrame::kUPlane
),
496 video_frame
->stride(VideoFrame::kUPlane
),
497 video_frame
->visible_data(VideoFrame::kVPlane
),
498 video_frame
->stride(VideoFrame::kVPlane
),
499 static_cast<uint8
*>(rgb_pixels
),
501 video_frame
->visible_rect().width(),
502 video_frame
->visible_rect().height());
503 #if SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
505 libyuv::ARGBToABGR(static_cast<uint8
*>(rgb_pixels
),
507 static_cast<uint8
*>(rgb_pixels
),
509 video_frame
->visible_rect().width(),
510 video_frame
->visible_rect().height());
513 case PIXEL_FORMAT_NV12
:
514 case PIXEL_FORMAT_NV21
:
515 case PIXEL_FORMAT_UYVY
:
516 case PIXEL_FORMAT_YUY2
:
517 case PIXEL_FORMAT_ARGB
:
518 case PIXEL_FORMAT_XRGB
:
519 case PIXEL_FORMAT_RGB24
:
520 case PIXEL_FORMAT_RGB32
:
521 case PIXEL_FORMAT_MJPEG
:
522 case PIXEL_FORMAT_UNKNOWN
:
528 void SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
529 gpu::gles2::GLES2Interface
* gl
,
530 VideoFrame
* video_frame
,
531 unsigned int texture
,
532 unsigned int internal_format
,
534 bool premultiply_alpha
,
537 DCHECK(video_frame
->HasTextures());
538 DCHECK_EQ(1u, VideoFrame::NumPlanes(video_frame
->format()));
540 const gpu::MailboxHolder
& mailbox_holder
= video_frame
->mailbox_holder(0);
541 DCHECK(mailbox_holder
.texture_target
== GL_TEXTURE_2D
||
542 mailbox_holder
.texture_target
== GL_TEXTURE_RECTANGLE_ARB
||
543 mailbox_holder
.texture_target
== GL_TEXTURE_EXTERNAL_OES
)
544 << mailbox_holder
.texture_target
;
546 gl
->WaitSyncPointCHROMIUM(mailbox_holder
.sync_point
);
547 uint32 source_texture
= gl
->CreateAndConsumeTextureCHROMIUM(
548 mailbox_holder
.texture_target
, mailbox_holder
.mailbox
.name
);
550 // The video is stored in a unmultiplied format, so premultiply
552 // Application itself needs to take care of setting the right |flip_y|
553 // value down to get the expected result.
554 // "flip_y == true" means to reverse the video orientation while
555 // "flip_y == false" means to keep the intrinsic orientation.
556 gl
->CopyTextureCHROMIUM(GL_TEXTURE_2D
, source_texture
, texture
,
557 internal_format
, type
,
558 flip_y
, premultiply_alpha
, false);
560 gl
->DeleteTextures(1, &source_texture
);
563 SyncPointClientImpl
client(gl
);
564 video_frame
->UpdateReleaseSyncPoint(&client
);
567 void SkCanvasVideoRenderer::ResetCache() {
568 // Clear cached values.
569 last_image_
= nullptr;
570 last_timestamp_
= kNoTimestamp();