I am attempting to make a stencil mask in OpenGL. I have been following the model from this source (http://open.gl/depthstencils and more specifically, http://open.gl/content/code/c5_reflection.txt), and as far as I can tell, I have followed the example properly. My code is drawing one square stencil, and then another square on top of it. I expected to see only the parts of the second rotating green square that are covering the same space as the first. What I actually see is the two overlapping squares, one rotating with no transparency. One notable difference from the example is that I am not using a texture. Is that a problem? I figured that this would be a simpler example.
I'm fairly new to ES2.0, so if I'm doing something obviously stupid, please let me know.
Initialization:
GLuint attributes[] = { GLKVertexAttribPosition, GLKVertexAttribColor, GLKVertexAttribTexCoord0 };
const char *attributeNames[] = { "position", "color", "texcoord0" };
// These are all global GLuint variables
// vshSrc and fshSrc are const char* filenames (the files are found properly)
_myProgram = loadProgram(vshSrc, fshSrc, 3, attributes, attributeNames);
_myProgramUniformMVP = glGetUniformLocation(_myProgram, "modelViewProjectionMatrix");
_myProgramUniformTex = glGetUniformLocation(_myProgram, "tex");
_myProgramUniformOverrideColor = glGetUniformLocation(_myProgram, "overrideColor");
The draw loop:
glEnable(GL_DEPTH_TEST);
glUseProgram(_myProgram);
glDisable(GL_BLEND);
glClearColor(1.0, 1.0, 1.0, 1.0);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
GLfloat gSquare[20] = { // not using the textures currently
// posX, posY, posZ, texX, texY,
-0.5f, -0.5f, 0, 0.0f, 0.0f,
0.5f, -0.5f, 0, 1.0f, 0.0f,
-0.5f, 0.5f, 0, 0.0f, 1.0f,
0.5f, 0.5f, 0, 1.0f, 1.0f
};
// Projection matrix
float aspect = fabsf(self.view.bounds.size.width / self.view.bounds.size.height);
GLKMatrix4 projectionMatrix = GLKMatrix4MakePerspective(GLKMathDegreesToRadians(65.0f), aspect, 0.1f, 100.0f);
// Put the squares where they can be seen
GLKMatrix4 baseModelViewMatrix = GLKMatrix4MakeTranslation(0.0f, 0.0f, -4.0f);
glEnable(GL_STENCIL_TEST);
// Build the stencil
glStencilFunc(GL_ALWAYS, 1, 0xFF); // Set any stencil to 1
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
glStencilMask(0xFF); // Write to stencil buffer
glDepthMask(GL_FALSE); // Don't write to depth buffer
glClear(GL_STENCIL_BUFFER_BIT); // Clear stencil buffer (0 by default)
GLKMatrix4 mvp = GLKMatrix4Multiply(projectionMatrix, baseModelViewMatrix);
// Draw a stationary red square for the stencil (though the color shouldn't matter)
glUniformMatrix4fv(_chetProgramUniformMVP, 1, 0, mvp.m);
glUniform1i(_chetProgramUniformTex, 0);
glUniform4f(_chetProgramUniformOverrideColor, 1.0f, 1.0f, 1.0f,0.0f);
glVertexAttrib4f(GLKVertexAttribColor, 1, 0, 0, 1);
glEnableVertexAttribArray(GLKVertexAttribPosition);
glVertexAttribPointer(GLKVertexAttribPosition, 3, GL_FLOAT, GL_FALSE, 20, gSquare);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// Prepare the mask
glStencilFunc(GL_EQUAL, 1, 0xFF); // Pass test if stencil value is 1
glStencilMask(0x00); // Don't write anything to stencil buffer
glDepthMask(GL_TRUE); // Write to depth buffer
glUniform4f(_myProgramUniformOverrideColor, 0.3f, 0.3f, 0.3f,1.0f);
// A slow rotating green square to be masked by the stencil
static float rotation = 0;
rotation += 0.01;
baseModelViewMatrix = GLKMatrix4Rotate(baseModelViewMatrix, rotation, 0, 0, 1);
mvp = GLKMatrix4Multiply(projectionMatrix, baseModelViewMatrix);
glUniformMatrix4fv(_myProgramUniformMVP, 1, 0, mvp.m);//The transformation matrix
glUniform1i(_myProgramUniformTex, 0); // The texture
glVertexAttrib4f(GLKVertexAttribColor, 0, 1, 0, 1);
glEnableVertexAttribArray(GLKVertexAttribPosition);
glVertexAttribPointer(GLKVertexAttribPosition, 3, GL_FLOAT, GL_FALSE, 20, gSquare);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glDisable(GL_STENCIL_TEST);
EDIT: The following shader stuff is irrelevant to the problem I was having. The stenciling does not take place in the shader.
Vertex Shader:
attribute vec4 position;
attribute vec4 color;
attribute vec2 texcoord0;
varying lowp vec4 colorVarying;
varying lowp vec2 texcoord;
uniform mat4 modelViewProjectionMatrix;
uniform vec4 overrideColor;
void main()
{
colorVarying = overrideColor * color;
texcoord = texcoord0;
gl_Position = modelViewProjectionMatrix * position;
}
Fragment Shader:
varying lowp vec4 colorVarying;
varying lowp vec2 texcoord;
uniform sampler2D tex;
void main()
{
gl_FragColor = colorVarying * texture2D(tex, texcoord);
}
It was necessary to initialize a stencil buffer. Here is the code that fixed it.
glGenRenderbuffersOES(1, &depthStencilRenderbuffer);
glBindRenderbufferOES(GL_RENDERBUFFER_OES, depthStencilRenderbuffer);
glRenderbufferStorageOES(GL_RENDERBUFFER_OES, GL_DEPTH24_STENCIL8_OES, framebufferWidth, framebufferHeight);
glFramebufferRenderbufferOES(GL_FRAMEBUFFER_OES, GL_DEPTH_ATTACHMENT_OES, GL_RENDERBUFFER_OES, depthStencilRenderbuffer);
glFramebufferRenderbufferOES(GL_FRAMEBUFFER_OES, GL_STENCIL_ATTACHMENT_OES, GL_RENDERBUFFER_OES, depthStencilRenderbuffer);