cc/video_layer_impl.cc

   1 // Copyright 2011 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/video_layer_impl.h"
   6
   7 #include "base/logging.h"
   8 #include "cc/io_surface_draw_quad.h"
   9 #include "cc/layer_tree_impl.h"
  10 #include "cc/math_util.h"
  11 #include "cc/quad_sink.h"
  12 #include "cc/renderer.h"
  13 #include "cc/resource_provider.h"
  14 #include "cc/stream_video_draw_quad.h"
  15 #include "cc/texture_draw_quad.h"
  16 #include "cc/video_frame_provider_client_impl.h"
  17 #include "cc/yuv_video_draw_quad.h"
  18 #include "gpu/GLES2/gl2extchromium.h"
  19 #include "media/filters/skcanvas_video_renderer.h"
  20 #include "third_party/khronos/GLES2/gl2.h"
  21 #include "third_party/khronos/GLES2/gl2ext.h"
  22
  23 namespace cc {
  24
  25 // static
  26 scoped_ptr<VideoLayerImpl> VideoLayerImpl::create(LayerTreeImpl* treeImpl, int id, VideoFrameProvider* provider)
  27 {
  28     scoped_ptr<VideoLayerImpl> layer(new VideoLayerImpl(treeImpl, id));
  29     layer->setProviderClientImpl(VideoFrameProviderClientImpl::Create(provider));
  30     DCHECK(treeImpl->proxy()->isImplThread());
  31     DCHECK(treeImpl->proxy()->isMainThreadBlocked());
  32     return layer.Pass();
  33 }
  34
  35 VideoLayerImpl::VideoLayerImpl(LayerTreeImpl* treeImpl, int id)
  36     : LayerImpl(treeImpl, id)
  37     , m_frame(0)
  38     , m_format(GL_INVALID_VALUE)
  39     , m_convertYUV(false)
  40     , m_externalTextureResource(0)
  41 {
  42 }
  43
  44 VideoLayerImpl::~VideoLayerImpl()
  45 {
  46     if (!m_providerClientImpl->Stopped()) {
  47         // In impl side painting, we may have a pending and active layer
  48         // associated with the video provider at the same time. Both have a ref
  49         // on the VideoFrameProviderClientImpl, but we stop when the first
  50         // LayerImpl (the one on the pending tree) is destroyed since we know
  51         // the main thread is blocked for this commit.
  52         DCHECK(layerTreeImpl()->proxy()->isImplThread());
  53         DCHECK(layerTreeImpl()->proxy()->isMainThreadBlocked());
  54         m_providerClientImpl->Stop();
  55     }
  56     freePlaneData(layerTreeImpl()->resource_provider());
  57
  58 #ifndef NDEBUG
  59     for (size_t i = 0; i < media::VideoFrame::kMaxPlanes; ++i)
  60         DCHECK(!m_framePlanes[i].resourceId);
  61     DCHECK(!m_externalTextureResource);
  62 #endif
  63 }
  64
  65 scoped_ptr<LayerImpl> VideoLayerImpl::createLayerImpl(LayerTreeImpl* treeImpl)
  66 {
  67     return scoped_ptr<LayerImpl>(new VideoLayerImpl(treeImpl, id()));
  68 }
  69
  70 void VideoLayerImpl::pushPropertiesTo(LayerImpl* layer)
  71 {
  72     LayerImpl::pushPropertiesTo(layer);
  73
  74     VideoLayerImpl* other = static_cast<VideoLayerImpl*>(layer);
  75     other->setProviderClientImpl(m_providerClientImpl);
  76 }
  77
  78 void VideoLayerImpl::didBecomeActive()
  79 {
  80     m_providerClientImpl->set_active_video_layer(this);
  81 }
  82
  83 // Convert media::VideoFrame::Format to OpenGL enum values.
  84 static GLenum convertVFCFormatToGLenum(const media::VideoFrame& frame)
  85 {
  86     switch (frame.format()) {
  87     case media::VideoFrame::YV12:
  88     case media::VideoFrame::YV16:
  89         return GL_LUMINANCE;
  90     case media::VideoFrame::NATIVE_TEXTURE:
  91         return frame.texture_target();
  92     case media::VideoFrame::INVALID:
  93     case media::VideoFrame::RGB32:
  94     case media::VideoFrame::EMPTY:
  95     case media::VideoFrame::I420:
  96         NOTREACHED();
  97         break;
  98     }
  99     return GL_INVALID_VALUE;
 100 }
 101
 102 size_t VideoLayerImpl::numPlanes() const
 103 {
 104     if (!m_frame)
 105         return 0;
 106
 107     if (m_convertYUV)
 108         return 1;
 109
 110     switch (m_frame->format()) {
 111     case media::VideoFrame::RGB32:
 112         return 1;
 113     case media::VideoFrame::YV12:
 114     case media::VideoFrame::YV16:
 115         return 3;
 116     case media::VideoFrame::INVALID:
 117     case media::VideoFrame::EMPTY:
 118     case media::VideoFrame::I420:
 119         break;
 120     case media::VideoFrame::NATIVE_TEXTURE:
 121         return 0;
 122     }
 123     NOTREACHED();
 124     return 0;
 125 }
 126
 127 void VideoLayerImpl::willDraw(ResourceProvider* resourceProvider)
 128 {
 129     LayerImpl::willDraw(resourceProvider);
 130
 131
 132     // Explicitly acquire and release the provider mutex so it can be held from
 133     // willDraw to didDraw. Since the compositor thread is in the middle of
 134     // drawing, the layer will not be destroyed before didDraw is called.
 135     // Therefore, the only thing that will prevent this lock from being released
 136     // is the GPU process locking it. As the GPU process can't cause the
 137     // destruction of the provider (calling stopUsingProvider), holding this
 138     // lock should not cause a deadlock.
 139     m_frame = m_providerClientImpl->AcquireLockAndCurrentFrame();
 140
 141     willDrawInternal(resourceProvider);
 142     freeUnusedPlaneData(resourceProvider);
 143
 144     if (!m_frame)
 145         m_providerClientImpl->ReleaseLock();
 146 }
 147
 148 void VideoLayerImpl::willDrawInternal(ResourceProvider* resourceProvider)
 149 {
 150     DCHECK(!m_externalTextureResource);
 151
 152     if (!m_frame)
 153         return;
 154
 155     m_format = convertVFCFormatToGLenum(*m_frame);
 156
 157     // If these fail, we'll have to add draw logic that handles offset bitmap/
 158     // texture UVs.  For now, just expect (0, 0) offset, since all our decoders
 159     // so far don't offset.
 160     DCHECK_EQ(m_frame->visible_rect().x(), 0);
 161     DCHECK_EQ(m_frame->visible_rect().y(), 0);
 162
 163     if (m_format == GL_INVALID_VALUE) {
 164         m_providerClientImpl->PutCurrentFrame(m_frame);
 165         m_frame = 0;
 166         return;
 167     }
 168
 169     // FIXME: If we're in software compositing mode, we do the YUV -> RGB
 170     // conversion here. That involves an extra copy of each frame to a bitmap.
 171     // Obviously, this is suboptimal and should be addressed once ubercompositor
 172     // starts shaping up.
 173     m_convertYUV = resourceProvider->defaultResourceType() == ResourceProvider::Bitmap &&
 174         (m_frame->format() == media::VideoFrame::YV12 ||
 175          m_frame->format() == media::VideoFrame::YV16);
 176
 177     if (m_convertYUV)
 178         m_format = GL_RGBA;
 179
 180     if (!allocatePlaneData(resourceProvider)) {
 181         m_providerClientImpl->PutCurrentFrame(m_frame);
 182         m_frame = 0;
 183         return;
 184     }
 185
 186     if (!copyPlaneData(resourceProvider)) {
 187         m_providerClientImpl->PutCurrentFrame(m_frame);
 188         m_frame = 0;
 189         return;
 190     }
 191
 192     if (m_format == GL_TEXTURE_2D)
 193         m_externalTextureResource = resourceProvider->createResourceFromExternalTexture(m_frame->texture_id());
 194 }
 195
 196 void VideoLayerImpl::appendQuads(QuadSink& quadSink, AppendQuadsData& appendQuadsData)
 197 {
 198     if (!m_frame)
 199         return;
 200
 201     SharedQuadState* sharedQuadState = quadSink.useSharedQuadState(createSharedQuadState());
 202     appendDebugBorderQuad(quadSink, sharedQuadState, appendQuadsData);
 203
 204     // FIXME: When we pass quads out of process, we need to double-buffer, or
 205     // otherwise synchonize use of all textures in the quad.
 206
 207     gfx::Rect quadRect(gfx::Point(), contentBounds());
 208     gfx::Rect opaqueRect(contentsOpaque() ? quadRect : gfx::Rect());
 209     gfx::Rect visibleRect = m_frame->visible_rect();
 210     gfx::Size codedSize = m_frame->coded_size();
 211
 212     // pixels for macroblocked formats.
 213     const float texWidthScale =
 214         static_cast<float>(visibleRect.width()) / codedSize.width();
 215     const float texHeightScale =
 216         static_cast<float>(visibleRect.height()) / codedSize.height();
 217
 218     switch (m_format) {
 219     case GL_LUMINANCE: {
 220         // YUV software decoder.
 221         const FramePlane& yPlane = m_framePlanes[media::VideoFrame::kYPlane];
 222         const FramePlane& uPlane = m_framePlanes[media::VideoFrame::kUPlane];
 223         const FramePlane& vPlane = m_framePlanes[media::VideoFrame::kVPlane];
 224         gfx::SizeF texScale(texWidthScale, texHeightScale);
 225         scoped_ptr<YUVVideoDrawQuad> yuvVideoQuad = YUVVideoDrawQuad::Create();
 226         yuvVideoQuad->SetNew(sharedQuadState, quadRect, opaqueRect, texScale, yPlane, uPlane, vPlane);
 227         quadSink.append(yuvVideoQuad.PassAs<DrawQuad>(), appendQuadsData);
 228         break;
 229     }
 230     case GL_RGBA: {
 231         // RGBA software decoder.
 232         const FramePlane& plane = m_framePlanes[media::VideoFrame::kRGBPlane];
 233         bool premultipliedAlpha = true;
 234         gfx::PointF uvTopLeft(0.f, 0.f);
 235         gfx::PointF uvBottomRight(texWidthScale, texHeightScale);
 236         const float opacity[] = {1.0f, 1.0f, 1.0f, 1.0f};
 237         bool flipped = false;
 238         scoped_ptr<TextureDrawQuad> textureQuad = TextureDrawQuad::Create();
 239         textureQuad->SetNew(sharedQuadState, quadRect, opaqueRect, plane.resourceId, premultipliedAlpha, uvTopLeft, uvBottomRight, opacity, flipped);
 240         quadSink.append(textureQuad.PassAs<DrawQuad>(), appendQuadsData);
 241         break;
 242     }
 243     case GL_TEXTURE_2D: {
 244         // NativeTexture hardware decoder.
 245         bool premultipliedAlpha = true;
 246         gfx::PointF uvTopLeft(0.f, 0.f);
 247         gfx::PointF uvBottomRight(texWidthScale, texHeightScale);
 248         const float opacity[] = {1.0f, 1.0f, 1.0f, 1.0f};
 249         bool flipped = false;
 250         scoped_ptr<TextureDrawQuad> textureQuad = TextureDrawQuad::Create();
 251         textureQuad->SetNew(sharedQuadState, quadRect, opaqueRect, m_externalTextureResource, premultipliedAlpha, uvTopLeft, uvBottomRight, opacity, flipped);
 252         quadSink.append(textureQuad.PassAs<DrawQuad>(), appendQuadsData);
 253         break;
 254     }
 255     case GL_TEXTURE_RECTANGLE_ARB: {
 256         gfx::Size visibleSize(visibleRect.width(), visibleRect.height());
 257         scoped_ptr<IOSurfaceDrawQuad> ioSurfaceQuad = IOSurfaceDrawQuad::Create();
 258         ioSurfaceQuad->SetNew(sharedQuadState, quadRect, opaqueRect, visibleSize, m_frame->texture_id(), IOSurfaceDrawQuad::UNFLIPPED);
 259         quadSink.append(ioSurfaceQuad.PassAs<DrawQuad>(), appendQuadsData);
 260         break;
 261     }
 262     case GL_TEXTURE_EXTERNAL_OES: {
 263         // StreamTexture hardware decoder.
 264         gfx::Transform transform(m_providerClientImpl->stream_texture_matrix());
 265         transform.Scale(texWidthScale, texHeightScale);
 266         scoped_ptr<StreamVideoDrawQuad> streamVideoQuad = StreamVideoDrawQuad::Create();
 267         streamVideoQuad->SetNew(sharedQuadState, quadRect, opaqueRect, m_frame->texture_id(), transform);
 268         quadSink.append(streamVideoQuad.PassAs<DrawQuad>(), appendQuadsData);
 269         break;
 270     }
 271     default:
 272         NOTREACHED();  // Someone updated convertVFCFormatToGLenum above but update this!
 273         break;
 274     }
 275 }
 276
 277 void VideoLayerImpl::didDraw(ResourceProvider* resourceProvider)
 278 {
 279     LayerImpl::didDraw(resourceProvider);
 280
 281     if (!m_frame)
 282         return;
 283
 284     if (m_format == GL_TEXTURE_2D) {
 285         DCHECK(m_externalTextureResource);
 286         // FIXME: the following assert will not be true when sending resources to a
 287         // parent compositor. We will probably need to hold on to m_frame for
 288         // longer, and have several "current frames" in the pipeline.
 289         DCHECK(!resourceProvider->inUseByConsumer(m_externalTextureResource));
 290         resourceProvider->deleteResource(m_externalTextureResource);
 291         m_externalTextureResource = 0;
 292     }
 293
 294     m_providerClientImpl->PutCurrentFrame(m_frame);
 295     m_frame = 0;
 296
 297     m_providerClientImpl->ReleaseLock();
 298 }
 299
 300 static gfx::Size videoFrameDimension(media::VideoFrame* frame, int plane) {
 301     gfx::Size dimensions = frame->coded_size();
 302     switch (frame->format()) {
 303     case media::VideoFrame::YV12:
 304         if (plane != media::VideoFrame::kYPlane) {
 305             dimensions.set_width(dimensions.width() / 2);
 306             dimensions.set_height(dimensions.height() / 2);
 307         }
 308         break;
 309     case media::VideoFrame::YV16:
 310         if (plane != media::VideoFrame::kYPlane) {
 311             dimensions.set_width(dimensions.width() / 2);
 312         }
 313         break;
 314     default:
 315         break;
 316     }
 317     return dimensions;
 318 }
 319
 320 bool VideoLayerImpl::FramePlane::allocateData(
 321     ResourceProvider* resourceProvider)
 322 {
 323     if (resourceId)
 324         return true;
 325
 326     resourceId = resourceProvider->createResource(size, format, ResourceProvider::TextureUsageAny);
 327     return resourceId;
 328 }
 329
 330 void VideoLayerImpl::FramePlane::freeData(ResourceProvider* resourceProvider)
 331 {
 332     if (!resourceId)
 333         return;
 334
 335     resourceProvider->deleteResource(resourceId);
 336     resourceId = 0;
 337 }
 338
 339 bool VideoLayerImpl::allocatePlaneData(ResourceProvider* resourceProvider)
 340 {
 341     const int maxTextureSize = resourceProvider->maxTextureSize();
 342     const size_t planeCount = numPlanes();
 343     for (unsigned planeIdx = 0; planeIdx < planeCount; ++planeIdx) {
 344         VideoLayerImpl::FramePlane& plane = m_framePlanes[planeIdx];
 345
 346         gfx::Size requiredTextureSize = videoFrameDimension(m_frame, planeIdx);
 347         // FIXME: Remove the test against maxTextureSize when tiled layers are
 348         // implemented.
 349         if (requiredTextureSize.IsEmpty() ||
 350             requiredTextureSize.width() > maxTextureSize ||
 351             requiredTextureSize.height() > maxTextureSize)
 352             return false;
 353
 354         if (plane.size != requiredTextureSize || plane.format != m_format) {
 355             plane.freeData(resourceProvider);
 356             plane.size = requiredTextureSize;
 357             plane.format = m_format;
 358         }
 359
 360         if (!plane.allocateData(resourceProvider))
 361             return false;
 362     }
 363     return true;
 364 }
 365
 366 bool VideoLayerImpl::copyPlaneData(ResourceProvider* resourceProvider)
 367 {
 368     const size_t planeCount = numPlanes();
 369     if (!planeCount)
 370         return true;
 371
 372     if (m_convertYUV) {
 373         if (!m_videoRenderer)
 374             m_videoRenderer.reset(new media::SkCanvasVideoRenderer);
 375         VideoLayerImpl::FramePlane& plane = m_framePlanes[media::VideoFrame::kRGBPlane];
 376         ResourceProvider::ScopedWriteLockSoftware lock(resourceProvider, plane.resourceId);
 377         m_videoRenderer->Paint(m_frame, lock.skCanvas(), m_frame->visible_rect(), 0xFF);
 378         return true;
 379     }
 380
 381     for (size_t planeIndex = 0; planeIndex < planeCount; ++planeIndex) {
 382         VideoLayerImpl::FramePlane& plane = m_framePlanes[planeIndex];
 383         // Only non-FormatNativeTexture planes should need upload.
 384         DCHECK_EQ(plane.format, GL_LUMINANCE);
 385         const uint8_t* softwarePlanePixels = m_frame->data(planeIndex);
 386         gfx::Rect imageRect(0, 0, m_frame->stride(planeIndex), plane.size.height());
 387         gfx::Rect sourceRect(gfx::Point(), plane.size);
 388         resourceProvider->setPixels(plane.resourceId, softwarePlanePixels, imageRect, sourceRect, gfx::Vector2d());
 389     }
 390     return true;
 391 }
 392
 393 void VideoLayerImpl::freePlaneData(ResourceProvider* resourceProvider)
 394 {
 395     for (size_t i = 0; i < media::VideoFrame::kMaxPlanes; ++i)
 396         m_framePlanes[i].freeData(resourceProvider);
 397 }
 398
 399 void VideoLayerImpl::freeUnusedPlaneData(ResourceProvider* resourceProvider)
 400 {
 401     size_t firstUnusedPlane = numPlanes();
 402     for (size_t i = firstUnusedPlane; i < media::VideoFrame::kMaxPlanes; ++i)
 403         m_framePlanes[i].freeData(resourceProvider);
 404 }
 405
 406 void VideoLayerImpl::didLoseOutputSurface()
 407 {
 408     freePlaneData(layerTreeImpl()->resource_provider());
 409 }
 410
 411 void VideoLayerImpl::setNeedsRedraw()
 412 {
 413     layerTreeImpl()->SetNeedsRedraw();
 414 }
 415
 416 void VideoLayerImpl::setProviderClientImpl(scoped_refptr<VideoFrameProviderClientImpl> providerClientImpl)
 417 {
 418     m_providerClientImpl = providerClientImpl;
 419 }
 420
 421 const char* VideoLayerImpl::layerTypeAsString() const
 422 {
 423     return "VideoLayer";
 424 }
 425
 426 }  // namespace cc