GLSL 1.20 shadow mapping, distorting shadow

Question

I have been trying to figure out a weird issue in my application for a week now. I implemented a very simple shadow mapping app and I get a shadow, but it moves and distorts in a weird way as the game camera moves.

An obvious explanation that I thought would be that the matrices are computed in a wrong way, but I have verified that this is not the reason. My shaders are really simple and I get a depth map rendered correctly, I visualize it in the app.

Here is where I got most ideas for the app from: http://www.opengl-tutorial.org/intermediate-tutorials/tutorial-16-shadow-mapping/

Here are my shadow shaders:

shadow_shader.frag

#version 120

varying vec4 shadowCoord;
uniform sampler2D shadowMapSampler;

void main() {   

    // THE WEIRDEST THING IS THAT THIS MUST BE ENABLED
    // works only if this is here, drops fps to 4 though
    if(shadowCoord.w <= 0.0) {
        gl_FragColor  = vec4(0.0);

        return;
    }

    float shadow = 1.0f;
    if(shadowCoord.w > 0.0f) {
        vec4 shadowCoordinateWdivide = shadowCoord / shadowCoord.w;
        shadowCoordinateWdivide.xyz = shadowCoordinateWdivide.xyz;

        if(shadowCoordinateWdivide.x > 0.0f && shadowCoordinateWdivide.x < 1.0f &&
           shadowCoordinateWdivide.y > 0.0f && shadowCoordinateWdivide.y < 1.0f &&
           shadowCoordinateWdivide.z > 0.0f && shadowCoordinateWdivide.z < 1.0f)
        {
            // Used to lower moiré pattern and self-shadowing
            shadowCoordinateWdivide.z -= 0.02f / shadowCoord.w;

            float distanceFromLight =
                texture2D(shadowMapSampler, shadowCoordinateWdivide.st).r;

            shadow = (distanceFromLight < shadowCoordinateWdivide.z) ? 0.4f : 1.0f;

        }

    }


    gl_FragColor = vec4(shadow, 0.0f, 0f, 1.0f);

}

shadow_shader.vert

#version 120

varying vec4 shadowCoord;
uniform mat4 depthBiasMVP;
uniform mat4 MVP;

void main() {
    shadowCoord = depthBiasMVP * gl_Vertex;

    gl_Position = MVP * gl_Vertex;
}

I can get the code working if I write this at the beginning of my fragment shader

...
if(shadowCoord.w <= 0.0) {
    gl_FragColor  = vec4(0.0);

    return;
}
...

My FBO creation is pretty standard I think. Here is the most essential code lines for creating the FBO and rendering stuff:

bool GLPanelObjects::InitFBO() {

    int w = m_viewDepth.w();
    int h = m_viewDepth.h();

    glGenTextures(1, &m_depth_tex);
    glBindTexture(GL_TEXTURE_2D, m_depth_tex);
    GLfloat v_bc[] = {1.0f,1.0f,1.0f,1.0f};
    glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, v_bc);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_NONE);
    glTexParameteri (GL_TEXTURE_2D, GL_DEPTH_TEXTURE_MODE, GL_LUMINANCE);

    glTexImage2D(GL_TEXTURE_2D,
                 0,
                 GL_DEPTH_COMPONENT32, // tried 16 and 24
                 w, h,
                 0,
                 GL_DEPTH_COMPONENT,
                 GL_UNSIGNED_BYTE, // tried GL_FLOAT and GL_UNSIGNED_BYTE
                 NULL);

    glBindTexture(GL_TEXTURE_2D, 0);

    //-------------------------
    glGenFramebuffers(1, &m_fb);
    glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, m_fb);
    glDrawBuffer(GL_NONE); // No color buffer is drawn
    glReadBuffer(GL_NONE);


    //-------------------------
    //Attach depth texture to FBO
    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_DEPTH_ATTACHMENT,
                              GL_TEXTURE_2D, m_depth_tex, 0/*mipmap level*/);


    //-------------------------
    //Does the GPU support current FBO configuration?
    GLenum status = glCheckFramebufferStatusEXT(GL_FRAMEBUFFER_EXT);
    if(status != GL_FRAMEBUFFER_COMPLETE) {
        return false;
    }

    glClearColor(0, 0, 0, 1);

    glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, 0);

    return true;

}


bool GLPanelObjects::create() {

    glHint(GL_PERSPECTIVE_CORRECTION_HINT,GL_NICEST);

    m_viewDepth = types::Rectangle<int>(m_viewport.x(),
                                        m_viewport.y(),
                                        m_viewport.w() * 0.5,
                                        m_viewport.h());

    m_viewScene = types::Rectangle<int>(m_viewport.x() + m_viewport.w()*0.5,
                                        m_viewport.y(),
                                        m_viewport.w() * 0.5,
                                        m_viewport.h());


    GLint maxbuffers;
    GLint maxTexUnits;
    glGetIntegerv(GL_MAX_DRAW_BUFFERS, &maxbuffers);
    glGetIntegerv(GL_MAX_COMBINED_TEXTURE_IMAGE_UNITS, &maxTexUnits);

    printf("max color attachements: %d\n", maxbuffers);
    printf("max texture units: %d\n", maxTexUnits);

    // initialize random seed
    srand(time(NULL));


    glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST);


    m_pShader = new Shader();

    if(!m_pShader->create("shaders/basic.vert",
                          "shaders/basic.frag")) {

        m_strErr = m_pShader->getErrorString();

        return false;
    }
    if(!m_pShader->use()) {
        m_strErr = m_pShader->getErrorString();
        return false;
    }


    m_pShadowShader = new Shader();

    if(!m_pShadowShader->create("shaders/shadow.vert",
                                "shaders/shadow.frag")) {

        m_strErr = m_pShadowShader->getErrorString();

        return false;
    }
    if(!m_pShadowShader->use()) {
        m_strErr = m_pShadowShader->getErrorString();
        return false;
    }

    Shader::useDefault();

    engn::ProjectionData projData;
    projData.m_fFovY = fovy;
    projData.m_fNear = zNear;
    projData.m_fFar = zFar;
    projData.m_fAspect = (float)m_viewScene.w() / (float)m_viewScene.h();
    projData.m_projectionMode = engn::PROJECTION_PERSPECTIVE;

    projData.m_fLeft = -20.0f; 
    projData.m_fRight = 20.0f; 
    projData.m_fBottom = -20.0f; 
    projData.m_fTop = 20.0f;

    projData.m_viewport = m_viewScene;

    Vec3f start(-8.0f, 0.0f, 8.0f);
    Vec3f end(1.0f, 0.0f, -4.0f);

    m_cam.setProjectionData(projData);
    m_cam.lookAt(Vec3f(-8, 0, 0), Vec3f(1, 0.2, 0));

    projData.m_fAspect = (float)m_viewDepth.w() / (float)m_viewDepth.h();
    //        projData.m_projectionMode = engn::PROJECTION_ORTHO;
    projData.m_viewport = m_viewDepth;
    m_camShadow.setProjectionData(projData);
    m_camShadow.lookAt(Vec3f(-10.0f, 0.0f, -4.2f), Vec3f(1.0f, 0.0f, 0.0f));
    //m_camShadow.lookAt(start, end);


    m_pSphere = new GLCube(this, 0.9f);
    m_pSphere->moveTo(Vec3f(0.0f, 0.0f, -0.0f));

    m_pWall = new GLWall(this);
    m_pWall->moveTo(Vec3f(4.0f, 0.0f, -0.0f));
    m_pWall->rotateY(-pi*0.5f);


    // get the uniform locations from the basic shader
    m_uniforms.shadowMap    = m_pShader->getUniformLocation("shadowMapSampler");
    if(m_uniforms.shadowMap == -1) {
        printf("Could not get uniform location: shadowMap\n");
        return false;
    }
    m_uniforms.depthBiasMVP = m_pShader->getUniformLocation("depthBiasMVP");
    if(m_uniforms.depthBiasMVP == -1) {
        printf("Could not get uniform location: depthBiasMVP\n");
        return false;
    }
    m_uniforms.MVP          = m_pShader->getUniformLocation("MVP");
    if(m_uniforms.MVP == -1) {
        printf("Could not get uniform location: MVP\n");
        return false;
    }

    // get the uniform locations from the shadow shader
    m_uniforms.depthMVP = m_pShadowShader->getUniformLocation("depthMVP");
    if(m_uniforms.depthMVP == -1) {
        printf("Could not get uniform location: depthMVP\n");
        return false;
    }


    return InitFBO();

}


void GLPanelObjects::drawShadowScene(std::vector<GL3DObject *> vecObj) {

    glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, m_fb);
    glDrawBuffersARB(0, NULL);

    glShadeModel(GL_FLAT);
    glDisable(GL_LIGHTING);
    glEnable(GL_DEPTH_TEST);

    const GLdouble x[] = {1.0, 0.0, 0.0, 0.0};
    const GLdouble y[] = {0.0, 1.0, 0.0, 0.0};
    const GLdouble z[] = {0.0, 0.0, 1.0, 0.0};
    const GLdouble w[] = {0.0, 0.0, 0.0, 1.0};

    glEnable(GL_TEXTURE_GEN_S);
    glEnable(GL_TEXTURE_GEN_T);
    glEnable(GL_TEXTURE_GEN_R);
    glEnable(GL_TEXTURE_GEN_Q);

    glTexGeni(GL_S, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
    glTexGeni(GL_T, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
    glTexGeni(GL_R, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
    glTexGeni(GL_Q, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);

    glTexGendv(GL_S, GL_EYE_PLANE, x);
    glTexGendv(GL_T, GL_EYE_PLANE, y);
    glTexGendv(GL_R, GL_EYE_PLANE, z);
    glTexGendv(GL_Q, GL_EYE_PLANE, w);

    glClear(GL_DEPTH_BUFFER_BIT);

    glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, 0);
    glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, 0);

    glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE); 

    // glActiveTexture(GL_TEXTURE0);
    glEnable(GL_TEXTURE_2D);
    glBindTexture(GL_TEXTURE_2D, 0);
    glDrawBuffer(GL_NONE); // No color buffer is drawn
    glReadBuffer(GL_NONE);

    glDepthMask(GL_TRUE);
    glDepthFunc(GL_LEQUAL);


    glEnable(GL_DEPTH_TEST);
    glCullFace(GL_BACK);


    const Mat4x4f mtxInvCam  = inverse(m_camShadow.getTransformation());
    const Mat4x4f mtxProj    = m_camShadow.getProjection();
    const Mat4x4f mtxDepthVP = mtxProj * mtxInvCam;

    for(uint i = 0; i < vecObj.size(); ++i) {

        const Mat4x4f &mtxModel = vecObj[i]->getTransformation();

        const Mat4x4f mtxDepthMVP = mtxDepthVP * mtxModel;

        if(!m_pShadowShader->use()) {
            std::string strErr = m_pShadowShader->getErrorString();
            printf("shader use error: %s\n", strErr.c_str());
            return;
        }


        glUniformMatrix4fvARB(m_uniforms.depthMVP,
                              1,
                              GL_FALSE,
                              &mtxDepthMVP(0, 0));


        vecObj[i]->render();

    }

}


void GLPanelObjects::render() {

    const unsigned char *ks = SDL_GetKeyState(NULL);
    moveCam(ks, m_cam);


    if(ks[SDLK_PLUS]) {
        m_camShadow.move(Vec3f(0, 0, -0.01));          
    }
    if(ks[SDLK_MINUS]) {
        m_camShadow.move(Vec3f(0, 0, 0.01));          
    }

    // set the viewport
    glViewport(0,
               0,
               m_viewDepth.w(),
               m_viewDepth.h());


    // place the objects to a vector
    std::vector<GL3DObject *> vecObj;
    vecObj.push_back(m_pWall);
    vecObj.push_back(m_pSphere);

    // draw into the shadow depth buffer
    drawShadowScene(vecObj);


    // switch to the screen buffer
    glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, 0);

    int nUnit = 7;

    // draw the scene to the window
    if(!m_pShader->use()) {
        std::string strErr = m_pShader->getErrorString();
        printf("shader use error: %s\n", strErr.c_str());
        return;
    }

    // set the viewport
    glViewport(m_viewScene.x(),
               m_viewScene.y(),
               m_viewScene.w(),
               m_viewScene.h());

    glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); 

    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    glUniform1iARB(m_uniforms.shadowMap, nUnit);

    glActiveTexture(GL_TEXTURE0 + nUnit);
    glBindTexture(GL_TEXTURE_2D, m_depth_tex);

    glEnable(GL_DEPTH_TEST);

    const Mat4x4f mtxBias(0.5f, 0.0f, 0.0f, 0.5f,  // row 0
                          0.0f, 0.5f, 0.0f, 0.5f,  // row 1
                          0.0f, 0.0f, 0.5f, 0.5f,  // row 2
                          0.0f, 0.0f, 0.0f, 1.0f); // row 3

    const Mat4x4f mtxShadowCamProjection = m_camShadow.getProjection();
    const Mat4x4f mtxShadowCamInv = inverse(m_camShadow.getTransformation());

    const Mat4x4f mtxDepthBiasVP =
        mtxBias * mtxShadowCamProjection * mtxShadowCamInv;

    const Mat4x4f VP =
        m_cam.getProjection() * inverse(m_cam.getTransformation());

    //glDisable(GL_CULL_FACE);

    for(size_t i = 0; i < vecObj.size(); ++i) {

        // draw the scene to the window
        if(!m_pShader->use()) {
            std::string strErr = m_pShader->getErrorString();
            printf("shader use error: %s\n", strErr.c_str());
            return;
        }

        glUniform1iARB(m_uniforms.shadowMap, nUnit);


        const Mat4x4f &mtxModel       = vecObj[i]->getTransformation();
        const Mat4x4f mtxDepthBiasMVP = mtxDepthBiasVP * mtxModel;
        const Mat4x4f mtxMVP          = VP * mtxModel;

        glUniformMatrix4fvARB(m_uniforms.depthBiasMVP,
                              1,
                              GL_FALSE,
                              &mtxDepthBiasMVP(0, 0));

        glUniformMatrix4fvARB(m_uniforms.MVP,
                              1,
                              GL_FALSE,
                              &mtxMVP(0, 0));

        vecObj[i]->render();

    }

    Shader::useDefault();
    glActiveTexture(GL_TEXTURE0);
    glEnable(GL_TEXTURE_2D);
    glBindTexture(GL_TEXTURE_2D, m_depth_tex);

    glDisable(GL_TEXTURE_GEN_S);
    glDisable(GL_TEXTURE_GEN_T);
    glDisable(GL_TEXTURE_GEN_R);
    glDisable(GL_TEXTURE_GEN_Q);

    // set the viewport
    glViewport(m_viewDepth.x(),
               m_viewDepth.y(),
               m_viewDepth.w(),
               m_viewDepth.h());


    // set the projection and model view matrices
    GLWidget::prepareOrtho(m_viewDepth);

    glBegin(GL_QUADS); {
        //glNormal3f(0.0f, 0.0f, 1.0f);

        glTexCoord2d(0.0, 0.0); glVertex2d(0.0,             0.0);
        glTexCoord2d(1.0, 0.0); glVertex2d(m_viewDepth.w(), 0.0);
        glTexCoord2d(1.0, 1.0); glVertex2d(m_viewDepth.w(), m_viewDepth.h());
        glTexCoord2d(0.0, 1.0); glVertex2d(0.0,             m_viewDepth.h());
    } glEnd();

    {
        Mat4x4f mtxProj = m_camShadow.getProjection();
        Mat4x4f mtxView = inverse(m_camShadow.getTransformation());

        Vec4f poi = mtxProj * mtxView * Vec4f(m_hitPoint, 1.0f);
        poi /= poi[3];

        float x = m_viewDepth.x() + (poi[0]*0.5f + 0.5f) * m_viewDepth.w();
        float y = m_viewDepth.y() + (poi[1]*0.5f + 0.5f) * m_viewDepth.h();

        glPointSize(5.0f);
        glBegin(GL_POINTS);
        glVertex2f(x, y);
        glEnd();

    }

}

I am starting to suspect a driver bug here, but as in so many cases, usually the bug is in the code.

I run this app in Linux Mint Debian Edition, and my lspci says:

Intel Corporation Mobile 945GM/GMS, 943/940GML Express Integrated Graphics Controller (rev 03)

and glxinfo says:

server glx version string: 1.4
client glx version string: 1.4
GLX version: 1.4
OpenGL version string: 1.4 Mesa 9.1.6

I have spent hours googling for similar issues without any luck. I am quite confident that my shadow mapping math is ok, but there are some other issues here that I am not seeing. If anyone could shed light into this I would be glad to here about it.

Answer 1

I have verified that it is a driver bug or something similar. Just tested the code with a grapchics card which supports glsl 1.30 and the application worked as it should.