Reflection and depth in OpenGL

Question

I'm trying to create a mirror in OpenGL. Most references I've found recommend using the stencil buffer to define the boundaries of the mirror itself and using a combination of translation and scaling matrices for the actual reflection. I managed to get this to work when the scene is viewed so that no objects are in front of the mirror. However, when an object is in front of the mirror the overlapping part is not shown, making it look as if it's behind the mirror. There's a good chance that I'm misunderstanding how the stencil buffer works, as this is the first time I've tried using it, but I could also be making some mistake regarding depth or something else. I'd appreciate any suggestions. Here is the code.

#include <gl/glut.h>

// perspective
void view(void) {
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    glOrtho(-2.0, 2.0, -2.0, 2.0, -2.0, 2.0);

    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    glRotatef(-85.0, 1.0, 0.0, 0.0);
    glScalef(0.25, 0.25, 0.25);
}

void init(void) {
    glEnable(GL_DEPTH_TEST);
    glClearStencil(0);
    glEnable(GL_STENCIL_TEST);
}

void mirror(GLboolean inside, GLfloat vertSet[4][3]) {
    GLint i;
    if(inside)
        glBegin(GL_QUADS); // draw inside of mirror
    else
        glBegin(GL_LINE_LOOP); // draw frame of mirror
            for(i = 0; i < 4; i++)
                glVertex3fv(vertSet[i]);
    glEnd();
}

void scene(void) {
    glMatrixMode(GL_MODELVIEW);
    glPushMatrix();
    glRotatef(90.0, 1.0, 0.0, 0.0);
    glutSolidTeapot(1.0);
    glPopMatrix();
}

void display(void) {
    GLfloat vertSet1[4][3] = {{-3.0, 3.0, 0.0}, {2.0, 3.0, 0.0}, 
        {2.0, 3.0, 2.0}, {-3.0, 3.0, 2.0}};

    view();

    // store mirror shape in the stencil buffer.
    glClear(GL_STENCIL_BUFFER_BIT);
    glStencilFunc(GL_ALWAYS, 1, 1);
    glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);
    mirror(true, vertSet1);

    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    // draw mirror frame.
    glColor3f(0.0, 0.0, 0.0);
    mirror(false, vertSet1);

    // draw scene outside mirror
    glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
    glStencilFunc(GL_NOTEQUAL, 1, 1);
    scene();

    // draw reflection of scene in mirror
    glStencilFunc(GL_EQUAL, 1, 1);
    glTranslatef(0.0, 3.0, 0.0);
    glScalef(1.0, -1.0, 1.0);
    glTranslatef(0.0, -3.0, 0.0);
    scene();

    glFlush();
    glutSwapBuffers();
    glutPostRedisplay();
}

int main (int argc, char **argv) {
    glutInit(&argc, argv);
    glutInitWindowSize(500, 500);
    glutInitWindowPosition(0, 0);
    glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGB | GLUT_STENCIL | GLUT_DEPTH);
    glutCreateWindow("");

    glClearColor(1.0, 1.0, 1.0, 0.0);

    init();

    glutDisplayFunc(display);

    glutMainLoop();
}

Answer 1

The idea of the stencil is, that you draw it when you'd normally render the mirror's glass within the scene. You want the drawing of the stencil to be depth tested. When you finished rendering your scene, you add a clip plane in the plane of the mirror, then you clear the depth buffer and redraw the scene. Since you clear the depth buffer you need the stencil to be depth tested first to make the mirror not overdraw already drawn objects in the world.

Note that drawing a mirror is essentially the same as drawing a portal.

Answer 2

This is my code for doing reflection in old opengl, hope this helps Please note that most of the code comes from NeHe tutorial: http://nehe.gamedev.net/tutorial/clipping__reflections_using_the_stencil_buffer/17004/

double eqr[] = { 0.0f, -1.0f, 0.0f, 0.0f };
glColorMask(0, 0, 0, 0);

glEnable(GL_STENCIL_TEST);
glStencilFunc(GL_ALWAYS, 1, 1); 
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);  

glDisable(GL_DEPTH_TEST);   
DrawFloor();    // draw flor to the stencil buffer

glColorMask(1, 1, 1, 1);            
glStencilFunc(GL_EQUAL, 1, 1);      

glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);

glEnable(GL_CLIP_PLANE0);       

glClipPlane(GL_CLIP_PLANE0, eqr);
glPushMatrix();
    glScalef(1.0f, -1.0f, 1.0f);
    //glLightfv(GL_LIGHT0, GL_POSITION, light_pos);     
    glFrontFace(GL_CW);
    glTranslatef(0.0f, 0.3f, 0.0);
    RenderScene();
    glFrontFace(GL_CCW);
glPopMatrix();
glDisable(GL_CLIP_PLANE0);                  
glDisable(GL_STENCIL_TEST); 

glEnable(GL_BLEND);
    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
    glColor4f(0.6f, 0.7f, 1.0f, 0.5f);
    DrawFloor();    // draw floor second time
glDisable(GL_BLEND);

// draw normal scene
glPushMatrix();
    glColor3f(1.0f, 1.0f, 1.0f);
    glTranslatef(0.0f, 0.3f, 0.0);
    RenderScene();
glPopMatrix();

For better result you might use RenderToTexture or use cube maps