remoting/android/java/src/org/chromium/chromoting/CardboardDesktopRenderer.java

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 package org.chromium.chromoting;
   6
   7 import android.content.Context;
   8 import android.graphics.Bitmap;
   9 import android.opengl.GLES20;
  10 import android.opengl.Matrix;
  11
  12 import com.google.vrtoolkit.cardboard.CardboardView;
  13 import com.google.vrtoolkit.cardboard.Eye;
  14 import com.google.vrtoolkit.cardboard.HeadTransform;
  15 import com.google.vrtoolkit.cardboard.Viewport;
  16
  17 import org.chromium.chromoting.jni.JniInterface;
  18
  19 import java.nio.ByteBuffer;
  20 import java.nio.ByteOrder;
  21 import java.nio.FloatBuffer;
  22
  23 import javax.microedition.khronos.egl.EGLConfig;
  24
  25 /**
  26  * Renderer for Cardboard view.
  27  */
  28 public class CardboardDesktopRenderer implements CardboardView.StereoRenderer {
  29     private static final String TAG = "cr.CardboardRenderer";
  30
  31     private static final int BYTE_PER_FLOAT = 4;
  32     private static final int POSITION_DATA_SIZE = 3;
  33     private static final int TEXTURE_COORDINATE_DATA_SIZE = 2;
  34     private static final float Z_NEAR = 0.1f;
  35     private static final float Z_FAR = 100.0f;
  36     private static final float INIT_DESKTOP_POSITION_X = 0.0f;
  37     private static final float INIT_DESKTOP_POSITION_Y = 0.0f;
  38     private static final float INIT_DESKTOP_POSITION_Z = -3.0f;
  39
  40     private static final FloatBuffer DESKTOP_COORDINATES = makeFloatBuffer(new float[] {
  41             // Desktop model coordinates.
  42             -1.0f, 1.0f, 1.0f,
  43             -1.0f, -1.0f, 1.0f,
  44             1.0f, 1.0f, 1.0f,
  45             -1.0f, -1.0f, 1.0f,
  46             1.0f, -1.0f, 1.0f,
  47             1.0f, 1.0f, 1.0f
  48     });
  49
  50     private static final FloatBuffer DESKTOP_TEXTURE_COORDINATES = makeFloatBuffer(new float[] {
  51             // Texture coordinate data.
  52             0.0f, 0.0f,
  53             0.0f, 1.0f,
  54             1.0f, 0.0f,
  55             0.0f, 1.0f,
  56             1.0f, 1.0f,
  57             1.0f, 0.0f
  58     });
  59
  60     private static final String VERTEX_SHADER =
  61             "uniform mat4 u_CombinedMatrix;"
  62             + "attribute vec4 a_Position;"
  63             + "attribute vec2 a_TexCoordinate;"
  64             + "varying vec2 v_TexCoordinate;"
  65             + "void main() {"
  66             + "   v_TexCoordinate = a_TexCoordinate;"
  67             + "   gl_Position = u_CombinedMatrix * a_Position;"
  68             + "}";
  69
  70     private static final String FRAGMENT_SHADER =
  71             "precision mediump float;"
  72             + "uniform sampler2D u_Texture;"
  73             + "varying vec2 v_TexCoordinate;"
  74             + "void main() {"
  75             + "   gl_FragColor = texture2D(u_Texture, v_TexCoordinate);"
  76             + "}";
  77
  78     private final Context mActivityContext;
  79
  80     // Flag to indicate whether reload the desktop texture or not.
  81     private boolean mReloadTexture;
  82
  83     private float[] mCameraMatrix;
  84     private float[] mModelMatrix;
  85     private float[] mViewMatrix;
  86     private float[] mProjectionMatrix;
  87     private float[] mCombinedMatrix;
  88
  89     private int mCombinedMatrixHandle;
  90     private int mPositionHandle;
  91     private int mTextureDataHandle;
  92     private int mTextureUniformHandle;
  93     private int mTextureCoordinateHandle;
  94     private int mProgramHandle;
  95     private int mVertexShaderHandle;
  96     private int mFragmentShaderHandle;
  97
  98     /** Lock to allow multithreaded access to mReloadTexture. */
  99     private Object mReloadTextureLock = new Object();
 100
 101     public CardboardDesktopRenderer(Context context) {
 102         mActivityContext = context;
 103         mReloadTexture = false;
 104
 105         mCameraMatrix = new float[16];
 106         mModelMatrix = new float[16];
 107         mViewMatrix = new float[16];
 108         mProjectionMatrix = new float[16];
 109         mCombinedMatrix = new float[16];
 110
 111         // Provide the callback for JniInterface.
 112         JniInterface.provideRedrawCallback(new Runnable() {
 113             @Override
 114             public void run() {
 115                 synchronized (mReloadTextureLock) {
 116                     mReloadTexture = true;
 117                 }
 118             }
 119         });
 120     }
 121
 122     @Override
 123     public void onSurfaceCreated(EGLConfig config) {
 124         // Set the background clear color to black.
 125         GLES20.glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
 126
 127         // Use culling to remove back faces.
 128         GLES20.glEnable(GLES20.GL_CULL_FACE);
 129
 130         // Enable depth testing.
 131         GLES20.glEnable(GLES20.GL_DEPTH_TEST);
 132
 133         // Set handles for desktop drawing.
 134         mVertexShaderHandle =
 135                 ShaderHelper.compileShader(GLES20.GL_VERTEX_SHADER, VERTEX_SHADER);
 136         mFragmentShaderHandle =
 137                 ShaderHelper.compileShader(GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER);
 138         mProgramHandle = ShaderHelper.createAndLinkProgram(mVertexShaderHandle,
 139                 mFragmentShaderHandle, new String[] {"a_Position", "a_TexCoordinate"});
 140         mCombinedMatrixHandle = GLES20.glGetUniformLocation(mProgramHandle, "u_CombinedMatrix");
 141         mTextureUniformHandle = GLES20.glGetUniformLocation(mProgramHandle, "u_Texture");
 142         mPositionHandle = GLES20.glGetAttribLocation(mProgramHandle, "a_Position");
 143         mTextureCoordinateHandle = GLES20.glGetAttribLocation(mProgramHandle, "a_TexCoordinate");
 144         mTextureDataHandle = TextureHelper.createTextureHandle();
 145
 146         // Position the eye at the origin.
 147         float eyeX = 0.0f;
 148         float eyeY = 0.0f;
 149         float eyeZ = 0.0f;
 150
 151         // We are looking toward the negative Z direction.
 152         float lookX = 0.0f;
 153         float lookY = 0.0f;
 154         float lookZ = -1.0f;
 155
 156         // Set our up vector. This is where our head would be pointing were we holding the camera.
 157         float upX = 0.0f;
 158         float upY = 1.0f;
 159         float upZ = 0.0f;
 160
 161         Matrix.setLookAtM(mCameraMatrix, 0, eyeX, eyeY, eyeZ, lookX, lookY, lookZ, upX, upY, upZ);
 162
 163         JniInterface.redrawGraphics();
 164     }
 165
 166     @Override
 167     public void onSurfaceChanged(int width, int height) {
 168     }
 169
 170     @Override
 171     public void onNewFrame(HeadTransform headTransform) {
 172         maybeLoadTexture(mTextureDataHandle);
 173     }
 174
 175     @Override
 176     public void onDrawEye(Eye eye) {
 177         GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
 178
 179         // Apply the eye transformation to the camera.
 180         Matrix.multiplyMM(mViewMatrix, 0, eye.getEyeView(), 0, mCameraMatrix, 0);
 181
 182         mProjectionMatrix = eye.getPerspective(Z_NEAR, Z_FAR);
 183
 184         // Translate the desktop model
 185         Matrix.setIdentityM(mModelMatrix, 0);
 186         Matrix.translateM(mModelMatrix, 0, INIT_DESKTOP_POSITION_X,
 187                 INIT_DESKTOP_POSITION_Y, INIT_DESKTOP_POSITION_Z);
 188
 189         // Pass in Model View Matrix and Model View Project Matrix
 190         Matrix.multiplyMM(mCombinedMatrix, 0, mViewMatrix, 0, mModelMatrix, 0);
 191         Matrix.multiplyMM(mCombinedMatrix, 0, mProjectionMatrix, 0, mCombinedMatrix, 0);
 192
 193         drawDesktop();
 194     }
 195
 196     @Override
 197     public void onRendererShutdown() {
 198         GLES20.glDeleteShader(mVertexShaderHandle);
 199         GLES20.glDeleteShader(mFragmentShaderHandle);
 200         GLES20.glDeleteTextures(1, new int[]{mTextureDataHandle}, 0);
 201     }
 202
 203     @Override
 204     public void onFinishFrame(Viewport viewport) {
 205     }
 206
 207     private void drawDesktop() {
 208         GLES20.glUseProgram(mProgramHandle);
 209
 210         // Pass in model view project matrix.
 211         GLES20.glUniformMatrix4fv(mCombinedMatrixHandle, 1, false, mCombinedMatrix, 0);
 212
 213         // Pass in texture data.
 214         GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
 215         GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureDataHandle);
 216         GLES20.glUniform1i(mTextureUniformHandle, 0);
 217
 218         // Pass in the desktop position.
 219         GLES20.glVertexAttribPointer(mPositionHandle, POSITION_DATA_SIZE, GLES20.GL_FLOAT, false,
 220                 0, DESKTOP_COORDINATES);
 221         GLES20.glEnableVertexAttribArray(mPositionHandle);
 222
 223         // Pass in texture coordinate.
 224         GLES20.glVertexAttribPointer(mTextureCoordinateHandle, TEXTURE_COORDINATE_DATA_SIZE,
 225                 GLES20.GL_FLOAT, false, 0, DESKTOP_TEXTURE_COORDINATES);
 226         GLES20.glEnableVertexAttribArray(mTextureCoordinateHandle);
 227
 228         // Draw the desktop.
 229         int totalPointNumber = 6;
 230         GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, totalPointNumber);
 231     }
 232
 233     /**
 234      * Link desktop texture with textureDataHandle if {@link mReloadTexture} is true.
 235      * @param textureDataHandle the handle we want attach texture to
 236      */
 237     private void maybeLoadTexture(int textureDataHandle) {
 238         synchronized (mReloadTextureLock) {
 239             if (!mReloadTexture) {
 240                 return;
 241             }
 242         }
 243
 244         // TODO(shichengfeng): Record the time desktop drawing takes.
 245         Bitmap bitmap = JniInterface.getVideoFrame();
 246
 247         if (bitmap == null) {
 248             // This can happen if the client is connected, but a complete video frame has not yet
 249             // been decoded.
 250             return;
 251         }
 252
 253         TextureHelper.linkTexture(textureDataHandle, bitmap);
 254
 255         synchronized (mReloadTextureLock) {
 256             mReloadTexture = false;
 257         }
 258     }
 259
 260     /**
 261      * Convert float array to a FloatBuffer for use in OpenGL calls.
 262      */
 263     private static FloatBuffer makeFloatBuffer(float[] data) {
 264         FloatBuffer result = ByteBuffer
 265                 .allocateDirect(data.length * BYTE_PER_FLOAT)
 266                 .order(ByteOrder.nativeOrder()).asFloatBuffer();
 267         result.put(data).position(0);
 268         return result;
 269     }
 270 }