OpenGL Shadow Mapping dual shader

Question

I'm attempting to write a simple shadow mapping engine utilizing two separate and distinct shaders using OpenGL and LWJGL. It uses VAO's/VBO's exclusively. The issue appears when the camera is rotated. It is as if the shadow rotates twice as much as the camera.

I begin by creating a simple cube at the origin, along with a 2D plane beneath the cube.

[code omitted due to length]

I then create a "sun" (that moves over time) which uses orthographic projection to create a shadow map.

The light source projection matrix (near is 1, far is 100):

lightMatrixStore.getProjectionMatrix().m00 = 1;
lightMatrixStore.getProjectionMatrix().m11 = 1;
lightMatrixStore.getProjectionMatrix().m22 = -2f/(100-1);
lightMatrixStore.getProjectionMatrix().m23 = -((100+1f)/(100-1f));
lightMatrixStore.getProjectionMatrix().m33 = 1;
lightMatrixStore.getProjectionMatrix().m32 = 0;  

Set up framebuffer/renderbuffer/texture:

depthTextureId = glGenTextures();
glBindTexture(GL_TEXTURE_2D, depthTextureId);

glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL12.GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL12.GL_CLAMP_TO_EDGE);

glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);

glTexParameteri(GL_TEXTURE_2D, GL14.GL_DEPTH_TEXTURE_MODE, GL_INTENSITY);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FAIL_VALUE_ARB, 0.5f);
glTexGeni(GL_S, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
glTexGeni(GL_T, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
glTexGeni(GL_R, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
glTexGeni(GL_Q, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);

glTexImage2D(
      GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, shadowMapWidth, shadowMapHeight, 0,
      GL_DEPTH_COMPONENT, GL_UNSIGNED_BYTE, (ByteBuffer) null
);
glBindTexture(GL_TEXTURE_2D, 0);
shadowMapWidth = 2048;
shadowMapHeight = 768*2;

rboId = glGenRenderbuffers();
glBindRenderbuffer(GL_RENDERBUFFER, rboId);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT32, shadowMapWidth, shadowMapHeight);

fboId = glGenFramebuffers();
glBindFramebuffer(GL_FRAMEBUFFER, fboId);

glDrawBuffer(GL_NONE);
glReadBuffer(GL_NONE);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, depthTextureId, 0);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, rboId);
glBindFramebuffer(GL_FRAMEBUFFER, 0);

int fboStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (fboStatus != GL_FRAMEBUFFER_COMPLETE) {
  System.err.print("Framebuffer error " + fboStatus + " : " + gluErrorString(glGetError()));
}

Shadow Map Vertex Shader:

#version 150 core

uniform mat4 projectionMatrix;
uniform mat4 viewMatrix;
uniform mat4 modelMatrix;

in vec4 in_Position;
in vec4 in_Color;
in vec2 in_TextureCoord;

out vec4 pass_Color;
out vec2 pass_TextureCoord;

void main(void) {
  gl_Position = projectionMatrix * viewMatrix * modelMatrix * in_Position;
  pass_Color = in_Color;
  pass_TextureCoord = in_TextureCoord;
}

And shadow map fragment shader:

#version 150 core

uniform sampler2D texture_diffuse;

in vec4 pass_Color;
in vec2 pass_TextureCoord;

out vec4 out_Color;

void main(void) {
  out_Color = pass_Color;
}

The program renders the scene from the light's perspective and creates the shadow map, then the second rendering pass does ambient lighting, specular lighting and the attempted shadows.

Final vertex shader:

#version 150 core

varying vec4 varyingVertex;
varying vec4 projectionVertex;
varying vec3 varyingNormal;
varying vec3 varyingPosition;
varying vec4 shadowCoord;

uniform mat4 projectionMatrix;
uniform mat4 viewMatrix;
uniform mat4 modelMatrix;
uniform mat4 lightViewMatrix;
uniform mat4 lightProjectionMatrix;

in vec4 in_Position;
in vec4 in_Color;
in vec2 in_TextureCoord;
in vec3 in_Normal;

out vec4 pass_Color;
out vec2 pass_TextureCoord;

void main() {
  varyingVertex = in_Position;
  varyingNormal = in_Normal;
  gl_Position = projectionMatrix * viewMatrix * modelMatrix * in_Position;
  varyingPosition = gl_Position;

  pass_Color = in_Color;
  pass_TextureCoord = in_TextureCoord;
  shadowCoord =  mat4( 0.5, 0.0, 0.0, 0.0,
                       0.0, 0.5, 0.0, 0.0,
                       0.0, 0.0, 0.5, 0.0,
                       0.5, 0.5, 0.5, 1.0) * lightProjectionMatrix * lightViewMatrix *     inverse(viewMatrix) * modelMatrix * in_Position;
}

Final fragment shader:

#version 150

uniform sampler2D ShadowMap;

varying vec4 varyingVertex;
varying vec4 projectionVertex;
varying vec3 varyingNormal;
varying vec3 varyingPosition;
varying vec4 shadowCoord;

uniform mat4 projectionMatrix;
uniform mat4 viewMatrix;
uniform mat4 modelMatrix;

in vec4 pass_Color;
in vec2 pass_TextureCoord;

out vec4 out_Color;

void main() {
  vec3 vertexPosition = (modelMatrix * viewMatrix * varyingVertex).xyz;
  vec3 surfaceNormal = normalize((mat3(1) * varyingNormal).xyz);
  vec3 lightDirection = normalize(gl_LightSource[0].position.xyz - vertexPosition);
  float diffuseLightIntensity = max(0, dot(surfaceNormal, lightDirection));
  out_Color.rgb = diffuseLightIntensity * pass_Color.rgb;
  out_Color += gl_LightModel.ambient;
  vec3 reflectionDirection = normalize(reflect(-lightDirection, surfaceNormal));
  float specular = max(0.0, dot(surfaceNormal, reflectionDirection));
  if (diffuseLightIntensity != 0) {
    float fspecular = pow(specular, 128);
    out_Color += fspecular;
  }

  vec4 shadowCoordinateWdivide = shadowCoord / shadowCoord.w ;
  shadowCoordinateWdivide.z += 0.0005;
  float distanceFromLight = texture2D(ShadowMap,shadowCoordinateWdivide.st).z;

  float shadow = 1.0;
  if (shadowCoord.w > 0.0)
    shadow = distanceFromLight < shadowCoordinateWdivide.z ? 0.5 : 1.0 ;

  out_Color = shadow * out_Color;
}

I'll admit I feel I am in a bit over my head when it comes to understanding the entire process, and have pieced this together from a variety of sources. The issue appears to be something simple but I can't quite figure it out.

Here are the images:

Not rotated:

Rotated:

Edit:

Changed shadowmap fragment shader to:

#version 150 core

layout(location=0) out float fragmentdepth;

void main(void) {
  fragmentdepth = gl_FragCoord.z;
}

Answer 1

This was unbelievably silly of me but I found the issue...

When I rotated the camera, I was rotating only the view matrix, not the model matrix. Applying the rotation to the model matrix as well fixed the issue.

Thank you for all the help and time! Pointers and suggestions were well taken.

Answer 2

There are a few things that strike me odd in your code.

First off, why are you using glTexGen if there are shaders? Everything glTexGen could be used for, is far easier done using a fragment shader.

Then your shadow map generating, and the resulting shadow map are not what you want for a shadow map. First and foremost you don't set "colours" in a shadow map. In fact when rendering the shadow map there's no need to sample a texture or write into a "colour" component in the first place. What you actually place in a shadow map is the distance from the light, i.e. the distance from the origin (point light) or an arbitrary plane (direction light). The resulting picture is a depth map, that has gradients on almost all geometry (except faces parallel to the projection plane). In your "shadow maps" you got solid colors. This is wrong! And the reason for that is, that you just set some constant color (per object), instead of emitting the per-fragment distance. Well, and then you got problems with face culling, too; your shadow map doesn't look like the cube it should be.

Well, and in your illumination shader after reaching world space you transform into inverse view space? Why in his noodlenes' name are you doing that? That's where your additional rotation comes from. You should transform it by inverse(lightprojection * lightview).

Well, and then there's the stylistic nitpicking. I'm surprised those shaders compiled at all. The in and out specifier have been introduced in later versions of GLSL, replacing the varying specifier. You shouldn't mix them (and if I'm not mistaken – I've to reread the spec on that – you can't mix them in a core profile).