SSAO & Shadow mapping | Shadows do not work with the SSAO

Question

The SSAO in our engine seems to be working, however I cannot get the SSAO to work with shadow mapping. Here is a screenshot of the bug I am currently having when shadows are applied....

With shadows applied

But also, depending on the camera view and camera position, random shadows sometimes appear...

Random Shadows depending on camera view and position

Here is the gbuffer vertex shader..

#version 330 core

layout (location = 0) in vec3 positions;
layout (location = 1) in vec2 texCoords;
layout (location = 2) in vec3 normals;

out vec3 FragPos;
out vec3 ShadowFragPos;
out vec2 TexCoords;
out vec3 Normal;

uniform mat4 model;
uniform mat4 view;
uniform mat4 proj;

void main()
{
    vec4 viewPos = view * model * vec4(positions, 1.0);
    FragPos = viewPos.xyz; 
    TexCoords = texCoords;

    mat3 normalMatrix = transpose(inverse(mat3(view * model)));
    Normal = normalMatrix * normals;

    gl_Position = proj * viewPos;
}

Here is the lighting shader..

#version 330 core

out vec4 FragColor;

in vec2 TexCoords;

uniform sampler2D gPosition;
uniform sampler2D gNormal;
uniform sampler2D gAlbedoSpec;
uniform sampler2D gShadowmap;
uniform sampler2D gSsao;

uniform vec3 cameraPos;

uniform mat4 lightSpaceMatrix;

vec3 Normal;
vec3 FragPos;

uniform vec3 lightPos;

float calculate_shadows(vec4 light_space_pos)
{
    // perform perspective divide
    vec3 projCoords = light_space_pos.xyz / light_space_pos.w;

    // transform to [0,1] range
    projCoords = projCoords * 0.5 + 0.5;

    // get closest depth value from light's perspective (using [0,1] range fragPosLight as coords)
    float closestDepth = texture(gShadowmap, projCoords.xy).r; 

    // get depth of current fragment from light's perspective
    float currentDepth = projCoords.z;

    // check whether current frag pos is in shadow
    vec3 lightDir = normalize(vec3(2.0f, 4.0f, 1.0f) - FragPos);
    float bias = max(0.05 * (1.0 - dot(Normal, lightDir)), 0.005);
    float shadow = 0.0;

    vec2 texelSize = 1.0 / textureSize(gShadowmap, 0);

    // 8x8 kernel PCF
    float x;
    float y;

    for (y = -3.5; y <= 3.5 ; y += 1.0)
    {
        for (x = -3.5; x <= 3.5 ; x += 1.0)
        {
            float pcfDepth = texture(gShadowmap, projCoords.xy + vec2(x, y) * texelSize).r; 
            shadow += currentDepth - bias > pcfDepth ? 1.0 : 0.0;        
        }
    }

    shadow /= 64.0; 

    return shadow;
}

void main(void)
{
    FragPos = texture(gPosition, TexCoords).rgb;
    Normal = texture(gNormal, TexCoords).rgb;
    vec3 Diffuse = texture(gAlbedoSpec, TexCoords).rgb;
    float Specular = texture(gAlbedoSpec, TexCoords).a; 
    float AmbientOcclusion = texture(gSsao, TexCoords).r;

    vec3 lighting = vec3(0.3 * Diffuse * AmbientOcclusion);
    vec3 viewDir = normalize(-FragPos);
    vec3 lightDir = normalize(lightPos - FragPos);
    vec3 diffuse = max(dot(Normal, lightDir), 0.0) * Diffuse * vec3(1.0f, 0.5f, 0.3f);

    vec3 halfwayDir = normalize(lightDir + viewDir);
    float spec = pow(max(dot(Normal, halfwayDir), 0.0), 8.0);
    vec3 specular = vec3(1.0f, 0.5f, 0.3f) * spec * Specular;

    float shadow = calculate_shadows(lightSpaceMatrix * vec4(FragPos, 1.0));    

    lighting += ((1.0 - shadow) * (diffuse + specular)); 

    FragColor = vec4(lighting, 1.0f);
}

The textures are binded in the light pass as follows..

// bind the positions texture and store in the first texture slot/unit
        glActiveTexture(GL_TEXTURE0); // texture unit 0
        glBindTexture(GL_TEXTURE_2D, gbuffer.gPositions); // geometry positions

        // bind the normals texture and store in the second texture slot/unit
        glActiveTexture(GL_TEXTURE1); // texture unit 1
        glBindTexture(GL_TEXTURE_2D, gbuffer.gNormals); // geometry normals

        // bind the albedo & specular texture and store in the third texture slot/unit
        glActiveTexture(GL_TEXTURE2); // texture unit 2
        glBindTexture(GL_TEXTURE_2D, gbuffer.gAlbedoSpec); // geometry albedospec
                                                           // bind the albedo & specular texture and store in the third texture slot/unit

        glActiveTexture(GL_TEXTURE3); // texture                         unit 3
        glBindTexture(GL_TEXTURE_2D, gbuffer.gShadowmap); // geometry albedospec

        glActiveTexture(GL_TEXTURE4); // texture unit 2
        glBindTexture(GL_TEXTURE_2D, gbuffer.ssaoColorBuffer); // geometry albedospec

Finally, here is the calculation of the lightSpaceMatrix..

light_projection = glm::ortho(-10.0f, 10.0f, -10.0f, 10.0f, 1.0f, 7.5f);
        light_view = glm::lookAt(glm::vec3(0.0f, 4.0f, 5.0f), glm::vec3(0.0f, 0.0f, 0.0f), glm::vec3(0.0f, 1.0f, 0.0f));
        light_space_matrix = light_projection * light_view;

Any ideas why this could be happening? how do I get shadows to work with SSAO?

any help is much appreciated.

Answer 1

FragPos is a camera view space position.

light_space_pos, the input parameter to calculate_shadows has to be a clip space coordinate, as seen from the light source.

This mean that when you do

float shadow = calculate_shadows(lightSpaceMatrix * vec4(FragPos, 1.0)); 

lightSpaceMatrix has to be the transformation from the camera view space to the clip space of the light source.

To do so, you have to do 3 transformations:

1. camera view space to world space. This can bed done by the inverse view matrix.

2. world space to light space, which is the transformation by light_view.

3. light view space to light clip space, is the transformation by light_projection.

So the setting of light_space_matrix = light_projection * light_view; is not sufficient, it has to be

light_space_matrix = light_projection * light_view * glm::inverse(view);