1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "cc/resources/video_resource_updater.h"
10 #include "base/trace_event/trace_event.h"
11 #include "cc/output/gl_renderer.h"
12 #include "cc/resources/resource_provider.h"
13 #include "gpu/GLES2/gl2extchromium.h"
14 #include "gpu/command_buffer/client/gles2_interface.h"
15 #include "media/base/video_frame.h"
16 #include "media/filters/skcanvas_video_renderer.h"
17 #include "third_party/khronos/GLES2/gl2.h"
18 #include "third_party/khronos/GLES2/gl2ext.h"
19 #include "ui/gfx/geometry/size_conversions.h"
25 const ResourceFormat kRGBResourceFormat
= RGBA_8888
;
27 class SyncPointClientImpl
: public media::VideoFrame::SyncPointClient
{
29 explicit SyncPointClientImpl(gpu::gles2::GLES2Interface
* gl
) : gl_(gl
) {}
30 ~SyncPointClientImpl() override
{}
31 uint32
InsertSyncPoint() override
{
32 return GLC(gl_
, gl_
->InsertSyncPointCHROMIUM());
34 void WaitSyncPoint(uint32 sync_point
) override
{
35 GLC(gl_
, gl_
->WaitSyncPointCHROMIUM(sync_point
));
39 gpu::gles2::GLES2Interface
* gl_
;
44 VideoResourceUpdater::PlaneResource::PlaneResource(
45 unsigned int resource_id
,
46 const gfx::Size
& resource_size
,
47 ResourceFormat resource_format
,
49 : resource_id(resource_id
),
50 resource_size(resource_size
),
51 resource_format(resource_format
),
58 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
59 const PlaneResource
& plane_resource
,
60 const media::VideoFrame
* video_frame
,
62 return plane_resource
.frame_ptr
== video_frame
&&
63 plane_resource
.plane_index
== plane_index
&&
64 plane_resource
.timestamp
== video_frame
->timestamp();
67 void VideoResourceUpdater::SetPlaneResourceUniqueId(
68 const media::VideoFrame
* video_frame
,
70 PlaneResource
* plane_resource
) {
71 plane_resource
->frame_ptr
= video_frame
;
72 plane_resource
->plane_index
= plane_index
;
73 plane_resource
->timestamp
= video_frame
->timestamp();
76 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE
) {}
78 VideoFrameExternalResources::~VideoFrameExternalResources() {}
80 VideoResourceUpdater::VideoResourceUpdater(ContextProvider
* context_provider
,
81 ResourceProvider
* resource_provider
)
82 : context_provider_(context_provider
),
83 resource_provider_(resource_provider
) {
86 VideoResourceUpdater::~VideoResourceUpdater() {
87 for (const PlaneResource
& plane_resource
: all_resources_
)
88 resource_provider_
->DeleteResource(plane_resource
.resource_id
);
91 VideoResourceUpdater::ResourceList::iterator
92 VideoResourceUpdater::AllocateResource(const gfx::Size
& plane_size
,
93 ResourceFormat format
,
95 // TODO(danakj): Abstract out hw/sw resource create/delete from
96 // ResourceProvider and stop using ResourceProvider in this class.
97 const ResourceProvider::ResourceId resource_id
=
98 resource_provider_
->CreateResource(plane_size
, GL_CLAMP_TO_EDGE
,
99 ResourceProvider::TextureHintImmutable
,
101 if (resource_id
== 0)
102 return all_resources_
.end();
104 gpu::Mailbox mailbox
;
106 DCHECK(context_provider_
);
108 gpu::gles2::GLES2Interface
* gl
= context_provider_
->ContextGL();
110 GLC(gl
, gl
->GenMailboxCHROMIUM(mailbox
.name
));
111 ResourceProvider::ScopedWriteLockGL
lock(resource_provider_
, resource_id
);
112 GLC(gl
, gl
->ProduceTextureDirectCHROMIUM(lock
.texture_id(), GL_TEXTURE_2D
,
115 all_resources_
.push_front(
116 PlaneResource(resource_id
, plane_size
, format
, mailbox
));
117 return all_resources_
.begin();
120 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it
) {
121 DCHECK_EQ(resource_it
->ref_count
, 0);
122 resource_provider_
->DeleteResource(resource_it
->resource_id
);
123 all_resources_
.erase(resource_it
);
126 VideoFrameExternalResources
VideoResourceUpdater::
127 CreateExternalResourcesFromVideoFrame(
128 const scoped_refptr
<media::VideoFrame
>& video_frame
) {
129 if (!VerifyFrame(video_frame
))
130 return VideoFrameExternalResources();
132 if (video_frame
->format() == media::VideoFrame::NATIVE_TEXTURE
)
133 return CreateForHardwarePlanes(video_frame
);
135 return CreateForSoftwarePlanes(video_frame
);
138 bool VideoResourceUpdater::VerifyFrame(
139 const scoped_refptr
<media::VideoFrame
>& video_frame
) {
140 switch (video_frame
->format()) {
141 // Acceptable inputs.
142 case media::VideoFrame::YV12
:
143 case media::VideoFrame::I420
:
144 case media::VideoFrame::YV12A
:
145 case media::VideoFrame::YV16
:
146 case media::VideoFrame::YV12J
:
147 case media::VideoFrame::YV12HD
:
148 case media::VideoFrame::YV24
:
149 case media::VideoFrame::NATIVE_TEXTURE
:
150 #if defined(VIDEO_HOLE)
151 case media::VideoFrame::HOLE
:
152 #endif // defined(VIDEO_HOLE)
153 case media::VideoFrame::ARGB
:
156 // Unacceptable inputs. ¯\(°_o)/¯
157 case media::VideoFrame::UNKNOWN
:
158 case media::VideoFrame::NV12
:
164 // For frames that we receive in software format, determine the dimensions of
165 // each plane in the frame.
166 static gfx::Size
SoftwarePlaneDimension(
167 const scoped_refptr
<media::VideoFrame
>& input_frame
,
168 bool software_compositor
,
169 size_t plane_index
) {
170 if (!software_compositor
) {
171 return media::VideoFrame::PlaneSize(
172 input_frame
->format(), plane_index
, input_frame
->coded_size());
174 return input_frame
->coded_size();
177 VideoFrameExternalResources
VideoResourceUpdater::CreateForSoftwarePlanes(
178 const scoped_refptr
<media::VideoFrame
>& video_frame
) {
179 TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
180 media::VideoFrame::Format input_frame_format
= video_frame
->format();
182 #if defined(VIDEO_HOLE)
183 if (input_frame_format
== media::VideoFrame::HOLE
) {
184 VideoFrameExternalResources external_resources
;
185 external_resources
.type
= VideoFrameExternalResources::HOLE
;
186 return external_resources
;
188 #endif // defined(VIDEO_HOLE)
190 // Only YUV software video frames are supported.
191 if (input_frame_format
!= media::VideoFrame::YV12
&&
192 input_frame_format
!= media::VideoFrame::I420
&&
193 input_frame_format
!= media::VideoFrame::YV12A
&&
194 input_frame_format
!= media::VideoFrame::YV12J
&&
195 input_frame_format
!= media::VideoFrame::YV12HD
&&
196 input_frame_format
!= media::VideoFrame::YV16
&&
197 input_frame_format
!= media::VideoFrame::YV24
) {
198 NOTREACHED() << input_frame_format
;
199 return VideoFrameExternalResources();
202 bool software_compositor
= context_provider_
== NULL
;
204 ResourceFormat output_resource_format
=
205 resource_provider_
->yuv_resource_format();
206 size_t output_plane_count
= media::VideoFrame::NumPlanes(input_frame_format
);
208 // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
209 // conversion here. That involves an extra copy of each frame to a bitmap.
210 // Obviously, this is suboptimal and should be addressed once ubercompositor
211 // starts shaping up.
212 if (software_compositor
) {
213 output_resource_format
= kRGBResourceFormat
;
214 output_plane_count
= 1;
217 // Drop recycled resources that are the wrong format.
218 for (auto it
= all_resources_
.begin(); it
!= all_resources_
.end();) {
219 if (it
->ref_count
== 0 && it
->resource_format
!= output_resource_format
)
220 DeleteResource(it
++);
225 const int max_resource_size
= resource_provider_
->max_texture_size();
226 std::vector
<ResourceList::iterator
> plane_resources
;
227 for (size_t i
= 0; i
< output_plane_count
; ++i
) {
228 gfx::Size output_plane_resource_size
=
229 SoftwarePlaneDimension(video_frame
, software_compositor
, i
);
230 if (output_plane_resource_size
.IsEmpty() ||
231 output_plane_resource_size
.width() > max_resource_size
||
232 output_plane_resource_size
.height() > max_resource_size
) {
236 // Try recycle a previously-allocated resource.
237 ResourceList::iterator resource_it
= all_resources_
.end();
238 for (auto it
= all_resources_
.begin(); it
!= all_resources_
.end(); ++it
) {
239 if (it
->resource_size
== output_plane_resource_size
&&
240 it
->resource_format
== output_resource_format
) {
241 if (PlaneResourceMatchesUniqueID(*it
, video_frame
.get(), i
)) {
242 // Bingo, we found a resource that already contains the data we are
243 // planning to put in it. It's safe to reuse it even if
244 // resource_provider_ holds some references to it, because those
245 // references are read-only.
250 // This extra check is needed because resources backed by SharedMemory
251 // are not ref-counted, unlike mailboxes. Full discussion in
252 // codereview.chromium.org/145273021.
254 software_compositor
&&
255 resource_provider_
->InUseByConsumer(it
->resource_id
);
256 if (it
->ref_count
== 0 && !in_use
) {
257 // We found a resource with the correct size that we can overwrite.
263 // Check if we need to allocate a new resource.
264 if (resource_it
== all_resources_
.end()) {
266 AllocateResource(output_plane_resource_size
, output_resource_format
,
267 !software_compositor
);
269 if (resource_it
== all_resources_
.end())
272 ++resource_it
->ref_count
;
273 plane_resources
.push_back(resource_it
);
276 if (plane_resources
.size() != output_plane_count
) {
277 // Allocation failed, nothing will be returned so restore reference counts.
278 for (ResourceList::iterator resource_it
: plane_resources
)
279 --resource_it
->ref_count
;
280 return VideoFrameExternalResources();
283 VideoFrameExternalResources external_resources
;
285 if (software_compositor
) {
286 DCHECK_EQ(plane_resources
.size(), 1u);
287 PlaneResource
& plane_resource
= *plane_resources
[0];
288 DCHECK_EQ(plane_resource
.resource_format
, kRGBResourceFormat
);
289 DCHECK(plane_resource
.mailbox
.IsZero());
291 if (!PlaneResourceMatchesUniqueID(plane_resource
, video_frame
.get(), 0)) {
292 // We need to transfer data from |video_frame| to the plane resource.
293 if (!video_renderer_
)
294 video_renderer_
.reset(new media::SkCanvasVideoRenderer
);
296 ResourceProvider::ScopedWriteLockSoftware
lock(
297 resource_provider_
, plane_resource
.resource_id
);
298 SkCanvas
canvas(lock
.sk_bitmap());
299 // This is software path, so canvas and video_frame are always backed
301 video_renderer_
->Copy(video_frame
, &canvas
, media::Context3D());
302 SetPlaneResourceUniqueId(video_frame
.get(), 0, &plane_resource
);
305 external_resources
.software_resources
.push_back(plane_resource
.resource_id
);
306 external_resources
.software_release_callback
=
307 base::Bind(&RecycleResource
, AsWeakPtr(), plane_resource
.resource_id
);
308 external_resources
.type
= VideoFrameExternalResources::SOFTWARE_RESOURCE
;
309 return external_resources
;
312 for (size_t i
= 0; i
< plane_resources
.size(); ++i
) {
313 PlaneResource
& plane_resource
= *plane_resources
[i
];
314 // Update each plane's resource id with its content.
315 DCHECK_EQ(plane_resource
.resource_format
,
316 resource_provider_
->yuv_resource_format());
318 if (!PlaneResourceMatchesUniqueID(plane_resource
, video_frame
.get(), i
)) {
319 // We need to transfer data from |video_frame| to the plane resource.
320 const uint8_t* input_plane_pixels
= video_frame
->data(i
);
322 gfx::Rect
image_rect(0, 0, video_frame
->stride(i
),
323 plane_resource
.resource_size
.height());
324 gfx::Rect
source_rect(plane_resource
.resource_size
);
325 resource_provider_
->SetPixels(plane_resource
.resource_id
,
326 input_plane_pixels
, image_rect
, source_rect
,
328 SetPlaneResourceUniqueId(video_frame
.get(), i
, &plane_resource
);
331 external_resources
.mailboxes
.push_back(
332 TextureMailbox(plane_resource
.mailbox
, GL_TEXTURE_2D
, 0));
333 external_resources
.release_callbacks
.push_back(
334 base::Bind(&RecycleResource
, AsWeakPtr(), plane_resource
.resource_id
));
337 external_resources
.type
= VideoFrameExternalResources::YUV_RESOURCE
;
338 return external_resources
;
342 void VideoResourceUpdater::ReturnTexture(
343 base::WeakPtr
<VideoResourceUpdater
> updater
,
344 const scoped_refptr
<media::VideoFrame
>& video_frame
,
347 BlockingTaskRunner
* main_thread_task_runner
) {
348 // TODO(dshwang) this case should be forwarded to the decoder as lost
350 if (lost_resource
|| !updater
.get())
352 // VideoFrame::UpdateReleaseSyncPoint() creates new sync point using the same
353 // GL context which created the given |sync_point|, so discard the
355 SyncPointClientImpl
client(updater
->context_provider_
->ContextGL());
356 video_frame
->UpdateReleaseSyncPoint(&client
);
359 VideoFrameExternalResources
VideoResourceUpdater::CreateForHardwarePlanes(
360 const scoped_refptr
<media::VideoFrame
>& video_frame
) {
361 TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
362 media::VideoFrame::Format frame_format
= video_frame
->format();
364 DCHECK_EQ(frame_format
, media::VideoFrame::NATIVE_TEXTURE
);
365 if (frame_format
!= media::VideoFrame::NATIVE_TEXTURE
)
366 return VideoFrameExternalResources();
368 if (!context_provider_
)
369 return VideoFrameExternalResources();
371 const gpu::MailboxHolder
* mailbox_holder
= video_frame
->mailbox_holder();
372 VideoFrameExternalResources external_resources
;
373 switch (mailbox_holder
->texture_target
) {
375 external_resources
.type
= VideoFrameExternalResources::RGB_RESOURCE
;
377 case GL_TEXTURE_EXTERNAL_OES
:
378 external_resources
.type
=
379 VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE
;
381 case GL_TEXTURE_RECTANGLE_ARB
:
382 external_resources
.type
= VideoFrameExternalResources::IO_SURFACE
;
386 return VideoFrameExternalResources();
389 external_resources
.mailboxes
.push_back(
390 TextureMailbox(mailbox_holder
->mailbox
,
391 mailbox_holder
->texture_target
,
392 mailbox_holder
->sync_point
));
393 external_resources
.mailboxes
.back().set_allow_overlay(
394 video_frame
->allow_overlay());
395 external_resources
.release_callbacks
.push_back(
396 base::Bind(&ReturnTexture
, AsWeakPtr(), video_frame
));
397 return external_resources
;
401 void VideoResourceUpdater::RecycleResource(
402 base::WeakPtr
<VideoResourceUpdater
> updater
,
403 ResourceProvider::ResourceId resource_id
,
406 BlockingTaskRunner
* main_thread_task_runner
) {
407 if (!updater
.get()) {
408 // Resource was already deleted.
412 const ResourceList::iterator resource_it
= std::find_if(
413 updater
->all_resources_
.begin(), updater
->all_resources_
.end(),
414 [resource_id
](const PlaneResource
& plane_resource
) {
415 return plane_resource
.resource_id
== resource_id
;
417 if (resource_it
== updater
->all_resources_
.end())
420 ContextProvider
* context_provider
= updater
->context_provider_
;
421 if (context_provider
&& sync_point
) {
422 GLC(context_provider
->ContextGL(),
423 context_provider
->ContextGL()->WaitSyncPointCHROMIUM(sync_point
));
427 resource_it
->ref_count
= 0;
428 updater
->DeleteResource(resource_it
);
432 --resource_it
->ref_count
;
433 DCHECK_GE(resource_it
->ref_count
, 0);