cc/video_layer_impl.cc

   1 // Copyright 2011 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/video_layer_impl.h"
   6
   7 #include "base/logging.h"
   8 #include "cc/io_surface_draw_quad.h"
   9 #include "cc/layer_tree_impl.h"
  10 #include "cc/math_util.h"
  11 #include "cc/quad_sink.h"
  12 #include "cc/renderer.h"
  13 #include "cc/resource_provider.h"
  14 #include "cc/stream_video_draw_quad.h"
  15 #include "cc/texture_draw_quad.h"
  16 #include "cc/yuv_video_draw_quad.h"
  17 #include "media/filters/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20
  21 namespace cc {
  22
  23 VideoLayerImpl::VideoLayerImpl(LayerTreeImpl* treeImpl, int id, VideoFrameProvider* provider)
  24     : LayerImpl(treeImpl, id)
  25     , m_provider(provider)
  26     , m_frame(0)
  27     , m_format(GL_INVALID_VALUE)
  28     , m_convertYUV(false)
  29     , m_externalTextureResource(0)
  30 {
  31     // This matrix is the default transformation for stream textures, and flips on the Y axis.
  32     m_streamTextureMatrix = MathUtil::createGfxTransform(
  33         1, 0, 0, 0,
  34         0, -1, 0, 0,
  35         0, 0, 1, 0,
  36         0, 1, 0, 1);
  37
  38     // This only happens during a commit on the compositor thread while the main
  39     // thread is blocked. That makes this a thread-safe call to set the video
  40     // frame provider client that does not require a lock. The same is true of
  41     // the call in the destructor.
  42     m_provider->SetVideoFrameProviderClient(this);
  43 }
  44
  45 VideoLayerImpl::~VideoLayerImpl()
  46 {
  47     // See comment in constructor for why this doesn't need a lock.
  48     if (m_provider) {
  49         m_provider->SetVideoFrameProviderClient(0);
  50         m_provider = 0;
  51     }
  52     freePlaneData(layerTreeImpl()->resource_provider());
  53
  54 #ifndef NDEBUG
  55     for (size_t i = 0; i < media::VideoFrame::kMaxPlanes; ++i)
  56         DCHECK(!m_framePlanes[i].resourceId);
  57     DCHECK(!m_externalTextureResource);
  58 #endif
  59 }
  60
  61 void VideoLayerImpl::StopUsingProvider()
  62 {
  63     // Block the provider from shutting down until this client is done
  64     // using the frame.
  65     base::AutoLock locker(m_providerLock);
  66     DCHECK(!m_frame);
  67     m_provider = 0;
  68 }
  69
  70 // Convert media::VideoFrame::Format to OpenGL enum values.
  71 static GLenum convertVFCFormatToGLenum(const media::VideoFrame& frame)
  72 {
  73     switch (frame.format()) {
  74     case media::VideoFrame::YV12:
  75     case media::VideoFrame::YV16:
  76         return GL_LUMINANCE;
  77     case media::VideoFrame::NATIVE_TEXTURE:
  78         return frame.texture_target();
  79     case media::VideoFrame::INVALID:
  80     case media::VideoFrame::RGB32:
  81     case media::VideoFrame::EMPTY:
  82     case media::VideoFrame::I420:
  83         NOTREACHED();
  84         break;
  85     }
  86     return GL_INVALID_VALUE;
  87 }
  88
  89 size_t VideoLayerImpl::numPlanes() const
  90 {
  91     if (!m_frame)
  92         return 0;
  93
  94     if (m_convertYUV)
  95         return 1;
  96
  97     switch (m_frame->format()) {
  98     case media::VideoFrame::RGB32:
  99         return 1;
 100     case media::VideoFrame::YV12:
 101     case media::VideoFrame::YV16:
 102         return 3;
 103     case media::VideoFrame::INVALID:
 104     case media::VideoFrame::EMPTY:
 105     case media::VideoFrame::I420:
 106         break;
 107     case media::VideoFrame::NATIVE_TEXTURE:
 108         return 0;
 109     }
 110     NOTREACHED();
 111     return 0;
 112 }
 113
 114 void VideoLayerImpl::willDraw(ResourceProvider* resourceProvider)
 115 {
 116     LayerImpl::willDraw(resourceProvider);
 117
 118     // Explicitly acquire and release the provider mutex so it can be held from
 119     // willDraw to didDraw. Since the compositor thread is in the middle of
 120     // drawing, the layer will not be destroyed before didDraw is called.
 121     // Therefore, the only thing that will prevent this lock from being released
 122     // is the GPU process locking it. As the GPU process can't cause the
 123     // destruction of the provider (calling stopUsingProvider), holding this
 124     // lock should not cause a deadlock.
 125     m_providerLock.Acquire();
 126
 127     willDrawInternal(resourceProvider);
 128     freeUnusedPlaneData(resourceProvider);
 129
 130     if (!m_frame)
 131         m_providerLock.Release();
 132 }
 133
 134 void VideoLayerImpl::willDrawInternal(ResourceProvider* resourceProvider)
 135 {
 136     DCHECK(!m_externalTextureResource);
 137
 138     if (!m_provider) {
 139         m_frame = 0;
 140         return;
 141     }
 142
 143     m_frame = m_provider->GetCurrentFrame();
 144
 145     if (!m_frame)
 146         return;
 147
 148     m_format = convertVFCFormatToGLenum(*m_frame);
 149
 150     // If these fail, we'll have to add draw logic that handles offset bitmap/
 151     // texture UVs.  For now, just expect (0, 0) offset, since all our decoders
 152     // so far don't offset.
 153     DCHECK_EQ(m_frame->visible_rect().x(), 0);
 154     DCHECK_EQ(m_frame->visible_rect().y(), 0);
 155
 156     if (m_format == GL_INVALID_VALUE) {
 157         m_provider->PutCurrentFrame(m_frame);
 158         m_frame = 0;
 159         return;
 160     }
 161
 162     // FIXME: If we're in software compositing mode, we do the YUV -> RGB
 163     // conversion here. That involves an extra copy of each frame to a bitmap.
 164     // Obviously, this is suboptimal and should be addressed once ubercompositor
 165     // starts shaping up.
 166     m_convertYUV = resourceProvider->defaultResourceType() == ResourceProvider::Bitmap &&
 167         (m_frame->format() == media::VideoFrame::YV12 ||
 168          m_frame->format() == media::VideoFrame::YV16);
 169
 170     if (m_convertYUV)
 171         m_format = GL_RGBA;
 172
 173     if (!allocatePlaneData(resourceProvider)) {
 174         m_provider->PutCurrentFrame(m_frame);
 175         m_frame = 0;
 176         return;
 177     }
 178
 179     if (!copyPlaneData(resourceProvider)) {
 180         m_provider->PutCurrentFrame(m_frame);
 181         m_frame = 0;
 182         return;
 183     }
 184
 185     if (m_format == GL_TEXTURE_2D)
 186         m_externalTextureResource = resourceProvider->createResourceFromExternalTexture(m_frame->texture_id());
 187 }
 188
 189 void VideoLayerImpl::appendQuads(QuadSink& quadSink, AppendQuadsData& appendQuadsData)
 190 {
 191     if (!m_frame)
 192         return;
 193
 194     SharedQuadState* sharedQuadState = quadSink.useSharedQuadState(createSharedQuadState());
 195     appendDebugBorderQuad(quadSink, sharedQuadState, appendQuadsData);
 196
 197     // FIXME: When we pass quads out of process, we need to double-buffer, or
 198     // otherwise synchonize use of all textures in the quad.
 199
 200     gfx::Rect quadRect(gfx::Point(), contentBounds());
 201     gfx::Rect opaqueRect(contentsOpaque() ? quadRect : gfx::Rect());
 202     gfx::Rect visibleRect = m_frame->visible_rect();
 203     gfx::Size codedSize = m_frame->coded_size();
 204
 205     // pixels for macroblocked formats.
 206     const float texWidthScale =
 207         static_cast<float>(visibleRect.width()) / codedSize.width();
 208     const float texHeightScale =
 209         static_cast<float>(visibleRect.height()) / codedSize.height();
 210
 211     switch (m_format) {
 212     case GL_LUMINANCE: {
 213         // YUV software decoder.
 214         const FramePlane& yPlane = m_framePlanes[media::VideoFrame::kYPlane];
 215         const FramePlane& uPlane = m_framePlanes[media::VideoFrame::kUPlane];
 216         const FramePlane& vPlane = m_framePlanes[media::VideoFrame::kVPlane];
 217         gfx::SizeF texScale(texWidthScale, texHeightScale);
 218         scoped_ptr<YUVVideoDrawQuad> yuvVideoQuad = YUVVideoDrawQuad::Create();
 219         yuvVideoQuad->SetNew(sharedQuadState, quadRect, opaqueRect, texScale, yPlane, uPlane, vPlane);
 220         quadSink.append(yuvVideoQuad.PassAs<DrawQuad>(), appendQuadsData);
 221         break;
 222     }
 223     case GL_RGBA: {
 224         // RGBA software decoder.
 225         const FramePlane& plane = m_framePlanes[media::VideoFrame::kRGBPlane];
 226         bool premultipliedAlpha = true;
 227         gfx::RectF uvRect(0, 0, texWidthScale, texHeightScale);
 228         const float opacity[] = {1.0f, 1.0f, 1.0f, 1.0f};
 229         bool flipped = false;
 230         scoped_ptr<TextureDrawQuad> textureQuad = TextureDrawQuad::Create();
 231         textureQuad->SetNew(sharedQuadState, quadRect, opaqueRect, plane.resourceId, premultipliedAlpha, uvRect, opacity, flipped);
 232         quadSink.append(textureQuad.PassAs<DrawQuad>(), appendQuadsData);
 233         break;
 234     }
 235     case GL_TEXTURE_2D: {
 236         // NativeTexture hardware decoder.
 237         bool premultipliedAlpha = true;
 238         gfx::RectF uvRect(0, 0, texWidthScale, texHeightScale);
 239         const float opacity[] = {1.0f, 1.0f, 1.0f, 1.0f};
 240         bool flipped = false;
 241         scoped_ptr<TextureDrawQuad> textureQuad = TextureDrawQuad::Create();
 242         textureQuad->SetNew(sharedQuadState, quadRect, opaqueRect, m_externalTextureResource, premultipliedAlpha, uvRect, opacity, flipped);
 243         quadSink.append(textureQuad.PassAs<DrawQuad>(), appendQuadsData);
 244         break;
 245     }
 246     case GL_TEXTURE_RECTANGLE_ARB: {
 247         gfx::Size visibleSize(visibleRect.width(), visibleRect.height());
 248         scoped_ptr<IOSurfaceDrawQuad> ioSurfaceQuad = IOSurfaceDrawQuad::Create();
 249         ioSurfaceQuad->SetNew(sharedQuadState, quadRect, opaqueRect, visibleSize, m_frame->texture_id(), IOSurfaceDrawQuad::UNFLIPPED);
 250         quadSink.append(ioSurfaceQuad.PassAs<DrawQuad>(), appendQuadsData);
 251         break;
 252     }
 253     case GL_TEXTURE_EXTERNAL_OES: {
 254         // StreamTexture hardware decoder.
 255         gfx::Transform transform(m_streamTextureMatrix);
 256         transform.Scale(texWidthScale, texHeightScale);
 257         scoped_ptr<StreamVideoDrawQuad> streamVideoQuad = StreamVideoDrawQuad::Create();
 258         streamVideoQuad->SetNew(sharedQuadState, quadRect, opaqueRect, m_frame->texture_id(), transform);
 259         quadSink.append(streamVideoQuad.PassAs<DrawQuad>(), appendQuadsData);
 260         break;
 261     }
 262     default:
 263         NOTREACHED();  // Someone updated convertVFCFormatToGLenum above but update this!
 264         break;
 265     }
 266 }
 267
 268 void VideoLayerImpl::didDraw(ResourceProvider* resourceProvider)
 269 {
 270     LayerImpl::didDraw(resourceProvider);
 271
 272     if (!m_frame)
 273         return;
 274
 275     if (m_format == GL_TEXTURE_2D) {
 276         DCHECK(m_externalTextureResource);
 277         // FIXME: the following assert will not be true when sending resources to a
 278         // parent compositor. We will probably need to hold on to m_frame for
 279         // longer, and have several "current frames" in the pipeline.
 280         DCHECK(!resourceProvider->inUseByConsumer(m_externalTextureResource));
 281         resourceProvider->deleteResource(m_externalTextureResource);
 282         m_externalTextureResource = 0;
 283     }
 284
 285     m_provider->PutCurrentFrame(m_frame);
 286     m_frame = 0;
 287
 288     m_providerLock.Release();
 289 }
 290
 291 static gfx::Size videoFrameDimension(media::VideoFrame* frame, int plane) {
 292     gfx::Size dimensions = frame->coded_size();
 293     switch (frame->format()) {
 294     case media::VideoFrame::YV12:
 295         if (plane != media::VideoFrame::kYPlane) {
 296             dimensions.set_width(dimensions.width() / 2);
 297             dimensions.set_height(dimensions.height() / 2);
 298         }
 299         break;
 300     case media::VideoFrame::YV16:
 301         if (plane != media::VideoFrame::kYPlane) {
 302             dimensions.set_width(dimensions.width() / 2);
 303         }
 304         break;
 305     default:
 306         break;
 307     }
 308     return dimensions;
 309 }
 310
 311 bool VideoLayerImpl::FramePlane::allocateData(
 312     ResourceProvider* resourceProvider)
 313 {
 314     if (resourceId)
 315         return true;
 316
 317     resourceId = resourceProvider->createResource(size, format, ResourceProvider::TextureUsageAny);
 318     return resourceId;
 319 }
 320
 321 void VideoLayerImpl::FramePlane::freeData(ResourceProvider* resourceProvider)
 322 {
 323     if (!resourceId)
 324         return;
 325
 326     resourceProvider->deleteResource(resourceId);
 327     resourceId = 0;
 328 }
 329
 330 bool VideoLayerImpl::allocatePlaneData(ResourceProvider* resourceProvider)
 331 {
 332     const int maxTextureSize = resourceProvider->maxTextureSize();
 333     const size_t planeCount = numPlanes();
 334     for (unsigned planeIdx = 0; planeIdx < planeCount; ++planeIdx) {
 335         VideoLayerImpl::FramePlane& plane = m_framePlanes[planeIdx];
 336
 337         gfx::Size requiredTextureSize = videoFrameDimension(m_frame, planeIdx);
 338         // FIXME: Remove the test against maxTextureSize when tiled layers are
 339         // implemented.
 340         if (requiredTextureSize.IsEmpty() ||
 341             requiredTextureSize.width() > maxTextureSize ||
 342             requiredTextureSize.height() > maxTextureSize)
 343             return false;
 344
 345         if (plane.size != requiredTextureSize || plane.format != m_format) {
 346             plane.freeData(resourceProvider);
 347             plane.size = requiredTextureSize;
 348             plane.format = m_format;
 349         }
 350
 351         if (!plane.allocateData(resourceProvider))
 352             return false;
 353     }
 354     return true;
 355 }
 356
 357 bool VideoLayerImpl::copyPlaneData(ResourceProvider* resourceProvider)
 358 {
 359     const size_t planeCount = numPlanes();
 360     if (!planeCount)
 361         return true;
 362
 363     if (m_convertYUV) {
 364         if (!m_videoRenderer)
 365             m_videoRenderer.reset(new media::SkCanvasVideoRenderer);
 366         VideoLayerImpl::FramePlane& plane = m_framePlanes[media::VideoFrame::kRGBPlane];
 367         ResourceProvider::ScopedWriteLockSoftware lock(resourceProvider, plane.resourceId);
 368         m_videoRenderer->Paint(m_frame, lock.skCanvas(), m_frame->visible_rect(), 0xFF);
 369         return true;
 370     }
 371
 372     for (size_t planeIndex = 0; planeIndex < planeCount; ++planeIndex) {
 373         VideoLayerImpl::FramePlane& plane = m_framePlanes[planeIndex];
 374         // Only non-FormatNativeTexture planes should need upload.
 375         DCHECK_EQ(plane.format, GL_LUMINANCE);
 376         const uint8_t* softwarePlanePixels = m_frame->data(planeIndex);
 377         gfx::Rect imageRect(0, 0, m_frame->stride(planeIndex), plane.size.height());
 378         gfx::Rect sourceRect(gfx::Point(), plane.size);
 379         resourceProvider->setPixels(plane.resourceId, softwarePlanePixels, imageRect, sourceRect, gfx::Vector2d());
 380     }
 381     return true;
 382 }
 383
 384 void VideoLayerImpl::freePlaneData(ResourceProvider* resourceProvider)
 385 {
 386     for (size_t i = 0; i < media::VideoFrame::kMaxPlanes; ++i)
 387         m_framePlanes[i].freeData(resourceProvider);
 388 }
 389
 390 void VideoLayerImpl::freeUnusedPlaneData(ResourceProvider* resourceProvider)
 391 {
 392     size_t firstUnusedPlane = numPlanes();
 393     for (size_t i = firstUnusedPlane; i < media::VideoFrame::kMaxPlanes; ++i)
 394         m_framePlanes[i].freeData(resourceProvider);
 395 }
 396
 397 void VideoLayerImpl::DidReceiveFrame()
 398 {
 399     setNeedsRedraw();
 400 }
 401
 402 void VideoLayerImpl::DidUpdateMatrix(const float matrix[16])
 403 {
 404     m_streamTextureMatrix = MathUtil::createGfxTransform(
 405         matrix[0], matrix[1], matrix[2], matrix[3],
 406         matrix[4], matrix[5], matrix[6], matrix[7],
 407         matrix[8], matrix[9], matrix[10], matrix[11],
 408         matrix[12], matrix[13], matrix[14], matrix[15]);
 409     setNeedsRedraw();
 410 }
 411
 412 void VideoLayerImpl::didLoseOutputSurface()
 413 {
 414     freePlaneData(layerTreeImpl()->resource_provider());
 415 }
 416
 417 void VideoLayerImpl::setNeedsRedraw()
 418 {
 419     layerTreeImpl()->SetNeedsRedraw();
 420 }
 421
 422 const char* VideoLayerImpl::layerTypeAsString() const
 423 {
 424     return "VideoLayer";
 425 }
 426
 427 }  // namespace cc