androidopengl-esrender-to-texture
Drawing on top of previous frame with an offscreen texture
I am very new to OpenGL ES 2.0.
I'm trying to write a fingerpaint app using OpenGL ES 2.0. The idea is to draw from touches each frame onto a texture incrementally (without calling glClear(int)), and sampling the texture onto a full-screen quad.
Referring to my code below, when I draw the GlCircle and GlLine onto the default Framebuffer, everything works fine.
But when I try to draw on top of the previous frame by using an offscreen texture, the coordinate on the rendered texture seems to be off:
- Y-axis is inverted.
- There's an offset on the Y-axis
The screenshot below should visually show what's wrong (the red/blue outline shows the actual touch coordinates on the screen, white dots are drawn to/from texture):
What am I doing wrong? Is there a better way of achieving this?
Here's my GLSurfaceView.Renderer:
package com.oaskamay.whiteboard.opengl;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.os.Bundle;
import android.util.Log;
import android.view.MotionEvent;
import com.oaskamay.whiteboard.opengl.base.GlSurfaceView;
import com.oaskamay.whiteboard.opengl.drawable.GlCircle;
import com.oaskamay.whiteboard.opengl.drawable.GlLine;
import com.oaskamay.whiteboard.opengl.drawable.GlTexturedQuad;
import java.util.ArrayList;
import java.util.List;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
public class GlDrawingRenderer implements GlSurfaceView.Renderer {
/*
* Keys used to store/restore the state of this renderer.
*/
private static final String EXTRA_MOTION_EVENTS = "extra_motion_events";
private static final float[] COLOR_BG = new float[]{0.0f, 0.0f, 0.0f, 1.0f};
private static final float[] COLOR_BRUSH = new float[]{1.0f, 1.0f, 1.0f, 1.0f};
/*
* Model-view-projection matrix used to map normalized GL coordinates to the screen's.
*/
private final float[] mMvpMatrix;
private final float[] mViewMatrix;
private final float[] mProjectionMatrix;
private final float[] mTextureProjectionMatrix;
private final float[] mTextureMvpMatrix;
/*
* Offscreen texture rendering handles.
*/
private int[] mFrameBufferHandle;
private int[] mRenderTextureHandle;
/*
* Lists of vertices to draw each frame.
*/
private List<Float> mLineVertexData;
private List<Float> mCircleVertexData;
/*
* List of stored MotionEvents and PacketData, required to store/restore state of Renderer.
*/
private ArrayList<MotionEvent> mMotionEvents;
private boolean mRestoreMotionEvents = false;
private GlLine mLine;
private GlCircle mCircle;
private GlTexturedQuad mTexturedQuad;
/*
* Variables to calculate FPS throughput.
*/
private long mStartTime = System.nanoTime();
private int mFrameCount = 0;
public GlDrawingRenderer() {
mMvpMatrix = new float[16];
mViewMatrix = new float[16];
mProjectionMatrix = new float[16];
mTextureProjectionMatrix = new float[16];
mTextureMvpMatrix = new float[16];
mFrameBufferHandle = new int[1];
mRenderTextureHandle = new int[1];
mLineVertexData = new ArrayList<>();
mCircleVertexData = new ArrayList<>();
mMotionEvents = new ArrayList<>();
}
@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config) {
// one time feature initializations
GLES20.glDisable(GLES20.GL_DEPTH_TEST);
GLES20.glDisable(GLES20.GL_DITHER);
// clear attachment buffers
GLES20.glClearColor(COLOR_BG[0], COLOR_BG[1], COLOR_BG[2],
COLOR_BG[3]);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
// initialize drawables
mLine = new GlLine();
mCircle = new GlCircle(5.0f);
mTexturedQuad = new GlTexturedQuad();
}
@Override
public void onSurfaceChanged(GL10 unused, int width, int height) {
GLES20.glViewport(0, 0, width, height);
// calculate projection, camera matrix and MVP matrix for touch events
Matrix.setLookAtM(mViewMatrix, 0, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f);
Matrix.orthoM(mProjectionMatrix, 0, 0.0f, width, height, 0.0f, 0.0f, 1.0f);
Matrix.multiplyMM(mMvpMatrix, 0, mProjectionMatrix, 0, mViewMatrix, 0);
mLine.setMvpMatrix(mMvpMatrix);
mCircle.setMvpMatrix(mMvpMatrix);
// calculate projection and MVP matrix for texture
Matrix.setIdentityM(mTextureProjectionMatrix, 0);
Matrix.multiplyMM(mTextureMvpMatrix, 0, mTextureProjectionMatrix, 0, mViewMatrix, 0);
mTexturedQuad.setMvpMatrix(mTextureMvpMatrix);
// setup buffers for offscreen texture
GLES20.glGenFramebuffers(1, mFrameBufferHandle, 0);
GLES20.glGenTextures(1, mRenderTextureHandle, 0);
mTexturedQuad.initTexture(width, height, mRenderTextureHandle[0]);
}
@Override
public void onDrawFrame(GL10 unused) {
// use offscreen texture frame buffer
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFrameBufferHandle[0]);
GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
GLES20.GL_TEXTURE_2D, mRenderTextureHandle[0], 0);
GlUtil.glCheckFramebufferStatus();
// restore and draw saved MotionEvents onto texture if they exist
if (mRestoreMotionEvents) {
mRestoreMotionEvents = false;
processStoredMotionEvents();
}
// draw current MotionEvents onto texture
drawObjects();
// use window frame buffer
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
GLES20.glClearColor(COLOR_BG[0], COLOR_BG[1], COLOR_BG[2], COLOR_BG[3]);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
// draw texture onto full-screen quad onto the window surface
drawTexturedQuad();
logFps();
}
/**
* Draws any available line and circle vertex data. Objects including {@code GlCircle} and
* {@code GlLine} are to be drawn on the offscreen texture. The offscreen texture will then be
* drawn onto a fullscreen quad in the default window framebuffer.
*/
private void drawObjects() {
if (!mLineVertexData.isEmpty()) {
drawLines();
}
if (!mCircleVertexData.isEmpty()) {
drawCircles();
}
}
/**
* Draws circles. OpenGL points cannot have radii, hence we draw circles on down key events
* instead of points.
*/
private void drawCircles() {
GLES20.glUseProgram(mCircle.getProgramHandle());
// read offsets
float dx = mCircleVertexData.remove(0);
float dy = mCircleVertexData.remove(0);
float dz = mCircleVertexData.remove(0);
mCircle.setTranslateMatrix(dx, dy, dz);
// read color
float r = mCircleVertexData.remove(0);
float g = mCircleVertexData.remove(0);
float b = mCircleVertexData.remove(0);
float a = mCircleVertexData.remove(0);
mCircle.setColor(r, g, b, a);
mCircle.draw();
}
/**
* Draws lines from touch start points to touch end points.
*/
private void drawLines() {
GLES20.glUseProgram(mLine.getProgramHandle());
// read offsets
float x1 = mLineVertexData.remove(0);
float y1 = mLineVertexData.remove(0);
float z1 = mLineVertexData.remove(0);
float x2 = mLineVertexData.remove(0);
float y2 = mLineVertexData.remove(0);
float z2 = mLineVertexData.remove(0);
mLine.setTranslateMatrix(x1, y1, z1, x2, y2, z2);
// read color
float r = mLineVertexData.remove(0);
float g = mLineVertexData.remove(0);
float b = mLineVertexData.remove(0);
float a = mLineVertexData.remove(0);
mLine.setColor(r, g, b, a);
mLine.draw();
}
/**
* Draws the offscreen texture onto the fullscreen quad, and draws the quad onto the default
* window framebuffer.
*/
private void drawTexturedQuad() {
GLES20.glUseProgram(mTexturedQuad.getProgramHandle());
mTexturedQuad.draw();
}
/**
* Processes MotionEvent.
* Sets vertex and color data based on MotionEvent information.
*
* @param event MotionEvent to process.
* @param store Pass true when processing fresh MotionEvents to store them to support parent
* activity recreations, pass false otherwise.
*/
public void processMotionEvent(MotionEvent event, boolean store) {
if (store) {
mMotionEvents.add(MotionEvent.obtain(event));
}
int action = event.getActionMasked();
switch (action) {
case MotionEvent.ACTION_POINTER_DOWN:
case MotionEvent.ACTION_DOWN:
case MotionEvent.ACTION_MOVE:
// set centroid
mCircleVertexData.add(event.getX());
mCircleVertexData.add(event.getY());
mCircleVertexData.add(0.0f);
// set color
mCircleVertexData.add(COLOR_BRUSH[0]);
mCircleVertexData.add(COLOR_BRUSH[1]);
mCircleVertexData.add(COLOR_BRUSH[2]);
mCircleVertexData.add(COLOR_BRUSH[3]);
break;
}
}
/**
* Draws stored MotionEvents.
* Required to be able to restore state of this Renderer.
*/
private void processStoredMotionEvents() {
for (MotionEvent event : mMotionEvents) {
processMotionEvent(event, false);
drawObjects();
}
}
/**
* Prints out current frames-per-second throughput.
*/
private void logFps() {
mFrameCount++;
if (System.nanoTime() - mStartTime >= 1000000000L) {
Log.d("GlDrawingRenderer", "FPS: " + mFrameCount);
mFrameCount = 0;
mStartTime = System.nanoTime();
}
}
/**
* Saves line and circle vertex data into the {@code Bundle} argument. Call when the parent
* {@code GLSurfaceView} calls its corresponding {@code onSaveInstanceState()} method.
*
* @param bundle Destination {@code Bundle} to save the renderer state into.
*/
public void onSaveInstanceState(Bundle bundle) {
bundle.putParcelableArrayList(EXTRA_MOTION_EVENTS, mMotionEvents);
}
/**
* Restores line and circle vertex data from the {@code Bundle} argument. Call when the parent
* {@code GLSurfaceView} calls its corresponding {@code onRestoreInstanceState(Parcelable)}
* method.
*
* @param bundle Source {@code Bundle} to save the renderer state from.
*/
public void onRestoreInstanceState(Bundle bundle) {
ArrayList<MotionEvent> motionEvents = bundle.getParcelableArrayList(EXTRA_MOTION_EVENTS);
if (motionEvents != null && !motionEvents.isEmpty()) {
mMotionEvents.addAll(motionEvents);
mRestoreMotionEvents = true;
}
}
}
And here's the GlTexturedQuad class:
package com.oaskamay.whiteboard.opengl.drawable;
import android.opengl.GLES20;
import com.oaskamay.whiteboard.opengl.GlUtil;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.ShortBuffer;
public class GlTexturedQuad {
/*
* Vertex metadata: we have 3 coordinates per vertex, and a quad can be drawn with 2 triangles.
*/
private static final int VERTEX_COORDS = 3;
private static final String VERTEX_SHADER_SOURCE =
"uniform mat4 u_MvpMatrix; \n" +
"attribute vec4 a_Position; \n" +
"attribute vec2 a_TextureCoord; \n" +
"varying vec2 v_TextureCoord; \n" +
" \n" +
"void main() { \n" +
" v_TextureCoord = a_TextureCoord; \n" +
" gl_Position = u_MvpMatrix * a_Position; \n" +
"} \n";
private static final String FRAGMENT_SHADER_SOURCE =
"uniform sampler2D u_Texture; \n" +
"varying vec2 v_TextureCoord; \n" +
" \n" +
"void main() { \n" +
" gl_FragColor = texture2D(u_Texture, v_TextureCoord);\n" +
"} \n";
/*
* Vertex locations. The quad will cover the whole screen, and is in normalized device
* coordinates. The projection matrix for this quad should be identity.
*/
private static final float[] VERTICES = {
-1.0f, +1.0f, 0.0f,
-1.0f, -1.0f, 0.0f,
+1.0f, -1.0f, 0.0f,
+1.0f, +1.0f, 0.0f
};
/*
* Describes the order in which vertices are to be rendered.
*/
private static final short[] VERTICES_ORDER = {
0, 1, 2,
0, 2, 3
};
/*
* (u, v) texture coordinates to be sent to the vertex and fragment shaders.
*/
private static final float[] TEXTURE_COORDS = {
0.0f, 0.0f,
0.0f, 1.0f,
1.0f, 1.0f,
1.0f, 0.0f
};
private float mMvpMatrix[];
private int mRenderTexture;
/*
* FloatBuffers used to store vertices and their order to draw.
*/
private final FloatBuffer mVertexBuffer;
private final ShortBuffer mVertexOrderBuffer;
private final FloatBuffer mTextureCoordsBuffer;
/*
* OpenGL handles to shader program, attributes, and uniforms.
*/
private final int mProgramHandle;
private final int mMvpMatrixHandle;
private final int mPositionHandle;
private final int mTextureHandle;
private final int mTextureCoordHandle;
/**
* Default constructor. Refrain from calling this multiple times as it may be expensive due to
* compilation of shader sources.
*/
public GlTexturedQuad() {
// initialize vertex buffer
ByteBuffer vertexBuffer = ByteBuffer.allocateDirect(VERTICES.length * 4);
vertexBuffer.order(ByteOrder.nativeOrder());
mVertexBuffer = vertexBuffer.asFloatBuffer();
mVertexBuffer.put(VERTICES);
mVertexBuffer.position(0);
// initialize vertex order buffer
ByteBuffer vertexOrderBuffer = ByteBuffer.allocateDirect(VERTICES_ORDER.length * 2);
vertexOrderBuffer.order(ByteOrder.nativeOrder());
mVertexOrderBuffer = vertexOrderBuffer.asShortBuffer();
mVertexOrderBuffer.put(VERTICES_ORDER);
mVertexOrderBuffer.position(0);
// initialize texture coordinates
ByteBuffer textureCoordsBuffer = ByteBuffer.allocateDirect(TEXTURE_COORDS.length * 4);
textureCoordsBuffer.order(ByteOrder.nativeOrder());
mTextureCoordsBuffer = textureCoordsBuffer.asFloatBuffer();
mTextureCoordsBuffer.put(TEXTURE_COORDS);
mTextureCoordsBuffer.position(0);
// compile vertex and fragment shader sources
int vertexShader = GlUtil.glLoadShader(GLES20.GL_VERTEX_SHADER,
VERTEX_SHADER_SOURCE);
int fragmentShader = GlUtil.glLoadShader(GLES20.GL_FRAGMENT_SHADER,
FRAGMENT_SHADER_SOURCE);
// create shader program and attach compiled sources
mProgramHandle = GLES20.glCreateProgram();
GLES20.glAttachShader(mProgramHandle, vertexShader);
GLES20.glAttachShader(mProgramHandle, fragmentShader);
GLES20.glLinkProgram(mProgramHandle);
// store attribute / uniform handles
mMvpMatrixHandle = GLES20.glGetUniformLocation(mProgramHandle, "u_MvpMatrix");
mTextureHandle = GLES20.glGetUniformLocation(mProgramHandle, "u_Texture");
mPositionHandle = GLES20.glGetAttribLocation(mProgramHandle, "a_Position");
mTextureCoordHandle = GLES20.glGetAttribLocation(mProgramHandle, "a_TextureCoord");
}
/**
* Initializes texture components.
*
* @param width Width of texture in pixels.
* @param height Height of texture in pixels.
*/
public void initTexture(int width, int height, int renderTexture) {
mRenderTexture = renderTexture;
// allocate pixel buffer for texture
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * 4);
byteBuffer.order(ByteOrder.nativeOrder());
IntBuffer texturePixelBuffer = byteBuffer.asIntBuffer();
// initialize texture
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mRenderTexture);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S,
GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T,
GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGB, width, height,
0, GLES20.GL_RGB, GLES20.GL_UNSIGNED_SHORT_5_6_5, texturePixelBuffer);
}
/**
* Draws this object. The model-view-projection matrix must be set with
* {@link #setMvpMatrix(float[])}.
*/
public final void draw() {
GLES20.glEnableVertexAttribArray(mPositionHandle);
GLES20.glEnableVertexAttribArray(mTextureCoordHandle);
// set vertex position and MVP matrix in shader
GLES20.glVertexAttribPointer(mPositionHandle, VERTEX_COORDS, GLES20.GL_FLOAT,
false, VERTEX_COORDS * 4, mVertexBuffer);
GLES20.glUniformMatrix4fv(mMvpMatrixHandle, 1, false, mMvpMatrix, 0);
// bind texture
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mRenderTexture);
// set texture data and coordinate
GLES20.glVertexAttribPointer(mTextureCoordHandle, 2, GLES20.GL_FLOAT, false, 0,
mTextureCoordsBuffer);
GLES20.glUniform1i(mTextureHandle, 0);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, VERTICES_ORDER.length, GLES20.GL_UNSIGNED_SHORT,
mVertexOrderBuffer);
GLES20.glDisableVertexAttribArray(mPositionHandle);
GLES20.glDisableVertexAttribArray(mTextureCoordHandle);
}
/**
* Sets the model-view-projection matrix in the vertex shader. Necessary to map the normalized
* GL coordinate system to that of the display.
*
* @param mvpMatrix Matrix to use as the model-view-projection matrix.
*/
public void setMvpMatrix(float[] mvpMatrix) {
mMvpMatrix = mvpMatrix;
}
public int getProgramHandle() {
return mProgramHandle;
}
}
Solution
EDIT (12/11/2015):
@reto-koradi suggested a much better solution. Invert the V-axis by changing the texture coordinates. This fix is also simple:
Change this (initialization of TEXTURE_COORDS array in GlTexturedQuad):
/*
* (u, v) texture coordinates to be sent to the vertex and fragment shaders.
*/
private static final float[] TEXTURE_COORDS = {
0.0f, 0.0f,
0.0f, 1.0f,
1.0f, 1.0f,
1.0f, 0.0f
};
To this:
/*
* (u, v) texture coordinates to be sent to the vertex and fragment shaders.
*/
private static final float[] TEXTURE_COORDS = {
0.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f,
1.0f, 1.0f
};
I've fixed the issue. The problem was with the projection matrix used for the GlTexturedQuad. The fix was simple:
I changed this (in onSurfaceChanged(GL10, int, int) in GlDrawingRenderer):
// calculate projection and MVP matrix for texture
Matrix.setIdentityM(mTextureProjectionMatrix, 0);
Matrix.multiplyMM(mTextureMvpMatrix, 0, mTextureProjectionMatrix, 0, mViewMatrix, 0);
mTexturedQuad.setMvpMatrix(mTextureMvpMatrix);
To this:
// calculate projection and MVP matrix for texture
Matrix.orthoM(mTextureProjectionMatrix, 0, -1.0f, 1.0f, 1.0f, -1.0f, 0.0f, 1.0f);
Matrix.multiplyMM(mTextureMvpMatrix, 0, mTextureProjectionMatrix, 0, mViewMatrix, 0);
mTexturedQuad.setMvpMatrix(mTextureMvpMatrix);
So now mTextureProjectionMatrix takes into account the V-axis inversion of the texture. Again, I'm an OpenGL ES 2.0 beginner, my explanation might be wrong. But it works :)
I hope this post helped someone out there!