gfx/webrender_bindings/RenderAndroidSurfaceTextureHost.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "RenderAndroidSurfaceTextureHost.h"
   8
   9 #include "GLReadTexImageHelper.h"
  10 #include "mozilla/gfx/Logging.h"
  11 #include "mozilla/webrender/RenderThread.h"
  12 #include "GLContext.h"
  13 #include "AndroidSurfaceTexture.h"
  14
  15 namespace mozilla {
  16 namespace wr {
  17
  18 RenderAndroidSurfaceTextureHost::RenderAndroidSurfaceTextureHost(
  19     const java::GeckoSurfaceTexture::GlobalRef& aSurfTex, gfx::IntSize aSize,
  20     gfx::SurfaceFormat aFormat, bool aContinuousUpdate,
  21     Maybe<gfx::Matrix4x4> aTransformOverride, bool aIsRemoteTexture)
  22     : mSurfTex(aSurfTex),
  23       mSize(aSize),
  24       mFormat(aFormat),
  25       mContinuousUpdate(aContinuousUpdate),
  26       mTransformOverride(aTransformOverride),
  27       mPrepareStatus(STATUS_NONE),
  28       mAttachedToGLContext(false),
  29       mIsRemoteTexture(aIsRemoteTexture) {
  30   MOZ_COUNT_CTOR_INHERITED(RenderAndroidSurfaceTextureHost, RenderTextureHost);
  31
  32   if (mSurfTex) {
  33     mSurfTex->IncrementUse();
  34   }
  35 }
  36
  37 RenderAndroidSurfaceTextureHost::~RenderAndroidSurfaceTextureHost() {
  38   MOZ_ASSERT(RenderThread::IsInRenderThread());
  39   MOZ_COUNT_DTOR_INHERITED(RenderAndroidSurfaceTextureHost, RenderTextureHost);
  40   // The SurfaceTexture gets destroyed when its use count reaches zero.
  41   if (mSurfTex) {
  42     mSurfTex->DecrementUse();
  43   }
  44 }
  45
  46 wr::WrExternalImage RenderAndroidSurfaceTextureHost::Lock(uint8_t aChannelIndex,
  47                                                           gl::GLContext* aGL) {
  48   MOZ_ASSERT(aChannelIndex == 0);
  49   MOZ_ASSERT((mPrepareStatus == STATUS_PREPARED) ||
  50              (!mSurfTex->IsSingleBuffer() &&
  51               mPrepareStatus == STATUS_UPDATE_TEX_IMAGE_NEEDED) ||
  52              mIsRemoteTexture);
  53
  54   if (mIsRemoteTexture) {
  55     EnsureAttachedToGLContext();
  56   }
  57
  58   if (mGL.get() != aGL) {
  59     // This should not happen. On android, SingletonGL is used.
  60     MOZ_ASSERT_UNREACHABLE("Unexpected GL context");
  61     return InvalidToWrExternalImage();
  62   }
  63
  64   if (!mSurfTex || !mGL || !mGL->MakeCurrent()) {
  65     return InvalidToWrExternalImage();
  66   }
  67
  68   MOZ_ASSERT(mAttachedToGLContext);
  69   if (!mAttachedToGLContext) {
  70     return InvalidToWrExternalImage();
  71   }
  72
  73   UpdateTexImageIfNecessary();
  74
  75   const gfx::Matrix4x4 transform = GetTextureTransform();
  76   // We expect this transform to always be rectilinear, usually just a
  77   // y-flip and sometimes an x and y scale/translation. This allows us
  78   // to simply transform 2 points here instead of 4.
  79   MOZ_ASSERT(transform.IsRectilinear(),
  80              "Unexpected non-rectilinear transform returned from "
  81              "SurfaceTexture.GetTransformMatrix()");
  82   gfx::Point uv0(0.0, 0.0);
  83   gfx::Point uv1(1.0, 1.0);
  84   uv0 = transform.TransformPoint(uv0);
  85   uv1 = transform.TransformPoint(uv1);
  86
  87   return NativeTextureToWrExternalImage(mSurfTex->GetTexName(), uv0.x, uv0.y,
  88                                         uv1.x, uv1.y);
  89 }
  90
  91 void RenderAndroidSurfaceTextureHost::Unlock() {}
  92
  93 bool RenderAndroidSurfaceTextureHost::EnsureAttachedToGLContext() {
  94   // During handling WebRenderError, GeckoSurfaceTexture should not be attached
  95   // to GLContext.
  96   if (RenderThread::Get()->IsHandlingWebRenderError()) {
  97     return false;
  98   }
  99
 100   if (mAttachedToGLContext) {
 101     return true;
 102   }
 103
 104   if (!mGL) {
 105     mGL = RenderThread::Get()->SingletonGL();
 106   }
 107
 108   if (!mSurfTex || !mGL || !mGL->MakeCurrent()) {
 109     return false;
 110   }
 111
 112   if (!mSurfTex->IsAttachedToGLContext((int64_t)mGL.get())) {
 113     GLuint texName;
 114     mGL->fGenTextures(1, &texName);
 115     ActivateBindAndTexParameteri(mGL, LOCAL_GL_TEXTURE0,
 116                                  LOCAL_GL_TEXTURE_EXTERNAL_OES, texName);
 117
 118     if (NS_FAILED(mSurfTex->AttachToGLContext((int64_t)mGL.get(), texName))) {
 119       MOZ_ASSERT(0);
 120       mGL->fDeleteTextures(1, &texName);
 121       return false;
 122     }
 123   }
 124
 125   mAttachedToGLContext = true;
 126   return true;
 127 }
 128
 129 void RenderAndroidSurfaceTextureHost::PrepareForUse() {
 130   // When SurfaceTexture is single buffer mode, UpdateTexImage needs to be
 131   // called only once for each publish. If UpdateTexImage is called more
 132   // than once, it causes hang on puglish side. And UpdateTexImage needs to
 133   // be called on render thread, since the SurfaceTexture is consumed on render
 134   // thread.
 135   MOZ_ASSERT(RenderThread::IsInRenderThread());
 136   MOZ_ASSERT(mPrepareStatus == STATUS_NONE);
 137
 138   if (mContinuousUpdate || !mSurfTex) {
 139     return;
 140   }
 141
 142   mPrepareStatus = STATUS_MIGHT_BE_USED_BY_WR;
 143
 144   if (mSurfTex->IsSingleBuffer()) {
 145     EnsureAttachedToGLContext();
 146     // When SurfaceTexture is single buffer mode, it is OK to call
 147     // UpdateTexImage() here.
 148     mSurfTex->UpdateTexImage();
 149     mPrepareStatus = STATUS_PREPARED;
 150   }
 151 }
 152
 153 void RenderAndroidSurfaceTextureHost::NotifyForUse() {
 154   MOZ_ASSERT(RenderThread::IsInRenderThread());
 155
 156   if (mPrepareStatus == STATUS_MIGHT_BE_USED_BY_WR) {
 157     // This happens when SurfaceTexture of video is rendered on WebRender.
 158     // There is a case that SurfaceTexture is not rendered on WebRender, instead
 159     // it is rendered to WebGL and the SurfaceTexture should not be attached to
 160     // gl context of WebRender. It is ugly. But it is same as Compositor
 161     // rendering.
 162     MOZ_ASSERT(!mSurfTex->IsSingleBuffer());
 163     if (!EnsureAttachedToGLContext()) {
 164       return;
 165     }
 166     mPrepareStatus = STATUS_UPDATE_TEX_IMAGE_NEEDED;
 167   }
 168 }
 169
 170 void RenderAndroidSurfaceTextureHost::NotifyNotUsed() {
 171   MOZ_ASSERT(RenderThread::IsInRenderThread());
 172
 173   if (!mSurfTex) {
 174     MOZ_ASSERT(mPrepareStatus == STATUS_NONE);
 175     return;
 176   }
 177
 178   if (mIsRemoteTexture) {
 179     UpdateTexImageIfNecessary();
 180   }
 181
 182   if (mSurfTex->IsSingleBuffer()) {
 183     MOZ_ASSERT(mPrepareStatus == STATUS_PREPARED);
 184     MOZ_ASSERT(mAttachedToGLContext);
 185     // Release SurfaceTexture's buffer to client side.
 186     mGL->MakeCurrent();
 187     mSurfTex->ReleaseTexImage();
 188   } else if (mPrepareStatus == STATUS_UPDATE_TEX_IMAGE_NEEDED) {
 189     MOZ_ASSERT(mAttachedToGLContext);
 190     // This could happen when video frame was skipped. UpdateTexImage() neeeds
 191     // to be called for adjusting SurfaceTexture's buffer status.
 192     mSurfTex->UpdateTexImage();
 193   }
 194
 195   mPrepareStatus = STATUS_NONE;
 196 }
 197
 198 void RenderAndroidSurfaceTextureHost::UpdateTexImageIfNecessary() {
 199   if (mIsRemoteTexture) {
 200     EnsureAttachedToGLContext();
 201     if (mPrepareStatus == STATUS_NONE) {
 202       PrepareForUse();
 203     }
 204     if (mPrepareStatus == STATUS_MIGHT_BE_USED_BY_WR) {
 205       NotifyForUse();
 206     }
 207   }
 208
 209   if (mContinuousUpdate) {
 210     MOZ_ASSERT(!mSurfTex->IsSingleBuffer());
 211     mSurfTex->UpdateTexImage();
 212   } else if (mPrepareStatus == STATUS_UPDATE_TEX_IMAGE_NEEDED) {
 213     MOZ_ASSERT(!mSurfTex->IsSingleBuffer());
 214     // When SurfaceTexture is not single buffer mode, call UpdateTexImage() once
 215     // just before rendering. During playing video, one SurfaceTexture is used
 216     // for all RenderAndroidSurfaceTextureHosts of video.
 217     mSurfTex->UpdateTexImage();
 218     mPrepareStatus = STATUS_PREPARED;
 219   }
 220 }
 221
 222 gfx::SurfaceFormat RenderAndroidSurfaceTextureHost::GetFormat() const {
 223   MOZ_ASSERT(mFormat == gfx::SurfaceFormat::R8G8B8A8 ||
 224              mFormat == gfx::SurfaceFormat::R8G8B8X8);
 225
 226   if (mFormat == gfx::SurfaceFormat::R8G8B8A8) {
 227     return gfx::SurfaceFormat::B8G8R8A8;
 228   }
 229
 230   if (mFormat == gfx::SurfaceFormat::R8G8B8X8) {
 231     return gfx::SurfaceFormat::B8G8R8X8;
 232   }
 233
 234   gfxCriticalNoteOnce
 235       << "Unexpected color format of RenderAndroidSurfaceTextureHost";
 236
 237   return gfx::SurfaceFormat::UNKNOWN;
 238 }
 239
 240 already_AddRefed<gfx::DataSourceSurface>
 241 RenderAndroidSurfaceTextureHost::ReadTexImage() {
 242   if (!mGL) {
 243     mGL = RenderThread::Get()->SingletonGL();
 244     if (!mGL) {
 245       return nullptr;
 246     }
 247   }
 248
 249   /* Allocate resulting image surface */
 250   int32_t stride = mSize.width * BytesPerPixel(GetFormat());
 251   RefPtr<gfx::DataSourceSurface> surf =
 252       gfx::Factory::CreateDataSourceSurfaceWithStride(mSize, GetFormat(),
 253                                                       stride);
 254   if (!surf) {
 255     return nullptr;
 256   }
 257
 258   layers::ShaderConfigOGL config = layers::ShaderConfigFromTargetAndFormat(
 259       LOCAL_GL_TEXTURE_EXTERNAL, mFormat);
 260   int shaderConfig = config.mFeatures;
 261
 262   bool ret = mGL->ReadTexImageHelper()->ReadTexImage(
 263       surf, mSurfTex->GetTexName(), LOCAL_GL_TEXTURE_EXTERNAL, mSize,
 264       GetTextureTransform(), shaderConfig, /* aYInvert */ false);
 265   if (!ret) {
 266     return nullptr;
 267   }
 268
 269   return surf.forget();
 270 }
 271
 272 bool RenderAndroidSurfaceTextureHost::MapPlane(RenderCompositor* aCompositor,
 273                                                uint8_t aChannelIndex,
 274                                                PlaneInfo& aPlaneInfo) {
 275   UpdateTexImageIfNecessary();
 276
 277   RefPtr<gfx::DataSourceSurface> readback = ReadTexImage();
 278   if (!readback) {
 279     return false;
 280   }
 281
 282   gfx::DataSourceSurface::MappedSurface map;
 283   if (!readback->Map(gfx::DataSourceSurface::MapType::READ, &map)) {
 284     return false;
 285   }
 286
 287   mReadback = readback;
 288   aPlaneInfo.mSize = mSize;
 289   aPlaneInfo.mStride = map.mStride;
 290   aPlaneInfo.mData = map.mData;
 291   return true;
 292 }
 293
 294 void RenderAndroidSurfaceTextureHost::UnmapPlanes() {
 295   if (mReadback) {
 296     mReadback->Unmap();
 297     mReadback = nullptr;
 298   }
 299 }
 300
 301 gfx::Matrix4x4 RenderAndroidSurfaceTextureHost::GetTextureTransform() const {
 302   gfx::Matrix4x4 transform;
 303
 304   // GetTransformMatrix() returns the transform set by the producer side of the
 305   // SurfaceTexture that must be applied to texture coordinates when
 306   // sampling. In some cases we may have set an override value, such as in
 307   // AndroidNativeWindowTextureData where we own the producer side, or for
 308   // MediaCodec output on devices where where we know the value is incorrect.
 309   if (mTransformOverride) {
 310     transform = *mTransformOverride;
 311   } else if (mSurfTex) {
 312     const auto& surf = java::sdk::SurfaceTexture::LocalRef(
 313         java::sdk::SurfaceTexture::Ref::From(mSurfTex));
 314     gl::AndroidSurfaceTexture::GetTransformMatrix(surf, &transform);
 315   }
 316
 317   return transform;
 318 }
 319
 320 RefPtr<layers::TextureSource>
 321 RenderAndroidSurfaceTextureHost::CreateTextureSource(
 322     layers::TextureSourceProvider* aProvider) {
 323   UpdateTexImageIfNecessary();
 324   return new layers::SurfaceTextureSource(
 325       aProvider, mSurfTex, mFormat, LOCAL_GL_TEXTURE_EXTERNAL,
 326       LOCAL_GL_CLAMP_TO_EDGE, mSize, mTransformOverride);
 327 }
 328
 329 }  // namespace wr
 330 }  // namespace mozilla