Opengl Selecting 3D Models drawn with different projection matrics

Question

I need to draw a cube and project it with the default projection matrix. Also, I want to draw a hud controlling the orientation of the sphere. The hud is projected with another projection matrix.

render()
{
    DrawGUI(); // project GUI with another projection matrix

    glPushMatrix();
    glutSolidCube(); // project the cube with the default projection matrix
    glPopMatrix();

    glutSwapBuffers();
}

reshape()
{
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    glOrtho(...);

    ...
}

DrawGUI()
{
    glMatrixMode(GL_PROJECTION);
    glPushMatrix();
    glLoadIdentity();
    gluOrtho2D(...); // project the GUI with this matrix

    glMatrixMode(GL_MODELVEIW);
    glPushMatrix();
    glLoadIdentity();


    glBegin();
    //... drawing GUI
    glEnd();

    glPopMatrix();
    glMatrixMode(GL_PROJECTION);
    glPopMatrix();
    glMatrixMode(GL_MODELVIEW);

}

#define BUFFER_LENGTH 64

void processSelection(int xPos, int yPos)
{

    static GLuint selectBuff[BUFFER_LENGTH];
    GLint hits, viewport[4];

    glSelectBuffer(BUFFER_LENGTH, selectBuff);
    glGetIntegerv(GL_VIEWPORT, viewport);

    // Switch to projection and save the matrix
    glMatrixMode(GL_PROJECTION);
    glPushMatrix();

    glRenderMode(GL_SELECT);
    glLoadIdentity();

    gluPickMatrix(xPos, viewport[3] - yPos, 2,2, viewport);

    glOrtho (-100, 100, -100, 100, -100, 100); // this line of code is the most

    glMatrixMode(GL_MODELVIEW);
    render();

    hits = glRenderMode(GL_RENDER);

    //...process hits

    // Restore the projection matrix
    glMatrixMode(GL_PROJECTION);
    glPopMatrix();
    glMatrixMode(GL_MODELVIEW);
}

The render part works well. Both the GUI and the cube are drawn without problem. However, the selection does not work as intended.

My question is: Since I project 3D models with different projection matrix, how should I deal with selection? I try to implement the typical selection buffer approach, but every time I click in the window, the selection buffer always contains the GUI even if I do not click on the GUI. Also, if I click on the cube, the selection buffer ends up with both the cube and the GUI.

Answer 1

If you use the selection buffer approach, you render with mixed projections as you do when doing the usual render. The only difference, is, that you apply that pick matrix as well. Also don't try to be too clever with the matrix pushing / poping. It rarely makes sense to use that in the projection matrix stack (hence it requires to have only 2 push levels, instead of the 32 for the modelview). Also don't use the reshape function to define the projection matrix.

DrawCube()
{
    glMatrixMode(GL_MODELVEIW);
    glLoadIdentity();

    glutSolidCube();
}

DrawGUI()
{
    glMatrixMode(GL_MODELVEIW);
    glLoadIdentity();

    glBegin();
    //... drawing GUI
    glEnd();
}

void render()
{
    // base the projection on whats already in the projection
    // matrix stack. For normal render this is identity, for
    // selection it is a pick matrix.

    glMatrixMode(GL_PROJECTION);
    glPushMatrix();
    gluOrtho2D(...); // project the GUI with this matrix
    DrawGUI();
    glMatrixMode(GL_PROJECTION);
    glPopMatrix();

    glMatrixMode(GL_PROJECTION);
    glPushMatrix();
    gluPerspective(...);
    glMatrixMode(GL_PROJECTION);
    glPopMatrix();
}

void display()
{
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();

    render();

    glutSwapBuffers();
}

#define BUFFER_LENGTH 64

void select(int xPos, int yPos)
{
    static GLuint selectBuff[BUFFER_LENGTH];
    GLint hits, viewport[4];

    glSelectBuffer(BUFFER_LENGTH, selectBuff);
    glGetIntegerv(GL_VIEWPORT, viewport);

    // Switch to projection and augment it with a picking matrix
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    gluPickMatrix(xPos, viewport[3] - yPos, 2,2, viewport);

    glRenderMode(GL_SELECT);
    render();

    hits = glRenderMode(GL_RENDER);

    //...process hits
}

Note that OpenGL selection mode is usually not GPU accelerated and hence very slow. Also it's been deprecated and removed from modern OpenGL versions. It's highly recommended to use either index buffer selection (i.e. render each object with a dedicated index "color") or perform manual ray-intersection picking into the scene data.