Issue converting a world space light position to model space in fragment shader

Question

I am attempting to address the age old issue of model vs world space coordinates when adding light to a scene with transformed models.

I have a house model with two instances. a skull, a single point light in my scene as well as some terrain and a skybox. I noticed that when applying some rotations to my models, the illumination provided by the point light was rotating with it. After some reading it became obvious that I am doing all of my computations in my shaders in model space but my light position/directions are in world space.

So, I realized I needed to uniform my space for my calculations and I think it makes sense to keep the shaders in model space and convert the light components from world to model.

When I don't do the transformation it looks almost right (though not perfect which I'm betting is b/c of the different spaces). I render a small lightbulb model at the location of the light and the tops of the houses and the terrain all illuminate as I expect relatively well.

When I do the conversion of the light position back to model space I'm expecting the light to still be illuminating from the lightbulb model. However it gets wonky.

I have no rotations on the models but some minor translations so I'm confused as to why it seems like the light source gets rotated 90 degrees around the x axis...

  <model name="blackSmith" mesh="black_smith/">
    <transform>
      <position x="2" y="1" z="-2"></position>
      <rotation x="0" y="0" z="0"></rotation>
      <scale x="1" y="1" z="1"></scale>
    </transform>
  </model>
  <model name="blackSmith2" mesh="black_smith/">
    <transform>
      <position x="-2" y="1" z="-2.5"></position>
      <rotation x="0" y="0" z="0"></rotation>
      <scale x="1" y="1" z="1"></scale>
    </transform>
  </model>

I am passing the model-view matrix from my vertex shader into my frag shader then doing an inverse transpose to convert the world space coordinate of the light position into model space. I figured that would be it but obviously I'm either missing something or I've done something wrong.

Here are my shaders (seperated by tags). For this example it is just a point light so u_isPointLight = true and u_isSpotLight = false

#type vertex
#version 330 core

layout(location = 0) in vec3 a_Position;
layout(location = 1) in vec3 aNormal;
layout(location = 2) in vec2 aTexCoord;
layout(location = 3) in mat4 aInstanceModelMatrix;

out vec3 outNormal;
out vec2 outTexCoord;
out vec3 outPos;
out mat4 outMat;

uniform mat4 u_Projection;
uniform mat4 u_View;

void main() {
    vec4 worldPosition = aInstanceModelMatrix * vec4(a_Position, 1.0);
    gl_Position = u_Projection * u_View * worldPosition;
    outPos = vec3(aInstanceModelMatrix * vec4(a_Position, 1.0));
    outNormal = aNormal;
    outTexCoord = aTexCoord;
    outMat = u_View * aInstanceModelMatrix;
}

#type fragment
#version 330 core

out vec4 color;

in vec3 outNormal; in vec2 outTexCoord; in vec3 outPos;
in mat4 outMat;

uniform bool u_DiffuseTextureValid;
uniform sampler2D u_DiffuseTexture;
uniform vec3 u_DiffuseColour;

uniform vec3 u_LightPosition;
uniform vec3 u_LightAttenuation;
uniform vec4 u_LightParams;
uniform vec3 u_SpotDirection;
uniform bool u_isSpotLight;
uniform bool u_isPointLight;
uniform float u_SpotInnerAngle;
uniform float u_SpotOuterAngle;

uniform vec4 uAmbientLight;

uniform bool u_AmbientTextureValid;
uniform sampler2D u_AmbientTexture;

uniform bool u_SpecularTextureValid;
uniform sampler2D u_SpecularTexture;
uniform vec4 u_Specular;

uniform sampler2D u_BumpTexture;
uniform bool u_BumpTextureValid;

uniform float u_MaterialAlpha;

void main() {
    const float kGamma = 0.4545454;
    const float kInverseGamma = 2.2;

    // WorldSpace to ModelSpace conversion
    vec3 lightPosition = (transpose(inverse(outMat)) * vec4(u_LightPosition, 1.0)).xyz;

    vec3 norm;
    if (u_BumpTextureValid) {
        norm = normalize(texture(u_BumpTexture, outTexCoord).xyz);
    } else {
        norm = normalize(outNormal);
    }

    vec3 lightDir;
    if (u_isPointLight || u_isSpotLight) {
        lightDir = normalize(lightPosition - outPos);
    } else {
        lightDir = normalize(-u_SpotDirection);
    }

    vec3 viewFragmentDirection = normalize(outPos);
    vec3 viewNormal = normalize(outNormal);

    float lightDistance = length(lightPosition - outPos);
    float attenuation = 1.0;
    if (u_isPointLight) {
        attenuation = 1.0 / (u_LightAttenuation.x + (lightDistance * u_LightAttenuation.y) + (lightDistance * lightDistance * u_LightAttenuation.z));
    }

    vec3 diffuse = u_LightParams.w * u_DiffuseColour * max(dot(norm, lightDir), 0.0) * u_LightParams.xyz;
    vec3 reflectedLightDirection = reflect(-lightDir, viewNormal);

    vec3 specular;
    if (u_SpecularTextureValid) {
        float spec = pow(max(dot(viewNormal, reflectedLightDirection), 0.0), u_Specular.w);
        specular = u_LightParams.xyz * spec * vec3(texture(u_SpecularTexture, outTexCoord));
    } else {
        float specularStrength = max(0.0, dot(viewFragmentDirection, reflectedLightDirection));
        specular = u_LightParams.w * u_Specular.rgb * pow(u_LightParams.w, u_Specular.w);
    }

    float spotFade = 1.0;
    if (u_isSpotLight && u_isPointLight) {
        float spotlightTheta = dot(lightDir, normalize(-u_SpotDirection));
        spotFade = (u_SpotInnerAngle - spotlightTheta) / (u_SpotOuterAngle - u_SpotInnerAngle);
        if (spotlightTheta < u_SpotInnerAngle) {
            specular = vec3(0, 0, 0);
            diffuse = vec3(0, 0, 0);
        }
    }

    vec4 objectColor;
    if (u_DiffuseTextureValid) {
        objectColor = texture(u_DiffuseTexture, outTexCoord);
    } else {
        objectColor = vec4(1.0, 1.0, 1.0, 1.0);
    }
    objectColor.rgb = pow(objectColor.rgb, vec3(kInverseGamma));

    color.a = objectColor.a * u_MaterialAlpha;
    vec3 ambient = uAmbientLight.a * uAmbientLight.rgb;

    ambient *= attenuation;
    diffuse *= attenuation * spotFade;
    specular *= attenuation * spotFade;

    if (u_DiffuseTextureValid) {
        color.rgb = pow(objectColor.rgb + ambient + diffuse + specular, vec3(kGamma));
    } else {
        color.rgb = pow(objectColor.rgb * (ambient + diffuse) + specular, vec3(kGamma));
    }
}

My questions are:

1. Am I correctly transforming the light position from world space to model space?
2. If I am doing a purely directional light (i.e. u_isSpotLight and u_isPointLight are both false), converting u_SpotDirection to modelspace by the same method should work.... right?
3. Assuming 1 is correct, can anyone tell my why the light seems to have gotten the weird rotation?

Answer 1

The model matrix transforms form model space to world space. Therefore you do the calculation of the light model in world space.

outPos = vec3(aInstanceModelMatrix * vec4(a_Position, 1.0));

However, you also need to convert the normal vector from model space to world space:
(see Why transforming normals with the transpose of the inverse of the modelview matrix?
and In which cases is the inverse matrix equal to the transpose?)

outNormal = transpose(inverse(mat3(aInstanceModelMatrix))) * aNormal;

outMat can be omitted completely.

The viewFragmentDirection vector is the vector from the fragment to the camera. In Viewscece this is easy to calculate because the camera position is (0, 0, 0). But in world space you need to calculate the vector from the fragment to the camera position:

vec3 viewFragmentDirection = normalize(outPos);

vec3 viewFragmentDirection = normalize(cameraPos - outPos);

You can use a uniform for the position of the camera. The camera position is also the translation of the inverse view matrix:

vec3 cameraPos = vec3(inverse(u_View)[3]);