Search code examples
opengldirectxstencil-bufferzbuffer

Stencil Options between DirectX and OpenGL


I'm implementing an emulation layer. Everything otherwise works, except the stencil stuff (my shadows overlap).

I'm just wondering if I'm doing things logically - if I'm making the proper conclusions/assumptions:

case D3DRS_STENCILENABLE:
    glAble(GL_STENCIL_TEST, value);
    break;
case D3DRS_STENCILFAIL:
    sStencilFail = glFromD3DSTENCILOP((D3DSTENCILOP)value);
    AssertGL(glStencilOp(sStencilFail, sStencilPassDepthFail, sStencilPassDepthPass));
    break;
case D3DRS_STENCILZFAIL:
    sStencilPassDepthFail = glFromD3DSTENCILOP((D3DSTENCILOP)value);
    AssertGL(glStencilOp(sStencilFail, sStencilPassDepthFail, sStencilPassDepthPass));
    break;
case D3DRS_STENCILPASS:
    sStencilPassDepthPass = glFromD3DSTENCILOP((D3DSTENCILOP)value);
    AssertGL(glStencilOp(sStencilFail, sStencilPassDepthFail, sStencilPassDepthPass));
    break;
case D3DRS_STENCILFUNC:
    sStencilFunc = glFromD3DCMPFUNC((D3DCMPFUNC)value);
    AssertGL(glStencilFunc(sStencilFunc, sStencilRef, sStencilValueMask));
    break;
case D3DRS_STENCILREF:
    sStencilRef = value;
    AssertGL(glStencilFunc(sStencilFunc, sStencilRef, sStencilValueMask));
    break;
case D3DRS_STENCILMASK:
    sStencilValueMask = value;
    AssertGL(glStencilFunc(sStencilFunc, sStencilRef, sStencilValueMask));
    break;
case D3DRS_STENCILWRITEMASK:
    AssertGL(glStencilMask(value));
    break;

The following are used above. glAble() simply is a wrapper for glEnable/glDisable.

static GLenum glFromD3DCMPFUNC(D3DCMPFUNC value) {
    return(GL_NEVER + value - 1);
}

static GLenum glFromD3DSTENCILOP(D3DSTENCILOP value) {
    switch (value) {
    case D3DSTENCILOP_KEEP: return(GL_KEEP);
    case D3DSTENCILOP_ZERO: return(GL_ZERO);
    case D3DSTENCILOP_REPLACE: return(GL_REPLACE);
    case D3DSTENCILOP_INVERT: return(GL_INVERT);
    case D3DSTENCILOP_INCRSAT:
    case D3DSTENCILOP_INCR:
        return(GL_INCR);
    case D3DSTENCILOP_DECRSAT:
    case D3DSTENCILOP_DECR:
        return(GL_DECR);
    default: Assert(!"Unsupported!"); return(0);
    }
}

Solution

  • Figured it out, I forgot to set my PIXELFORMATDESCRIPTOR:

    pfd.cStencilBits = 8;