Intersect a ray with a triangle in GLSL C++

Question

I am trying to make a ray intersection with a triangle in the fragment shader, if it colide i will paint a black dot in the texture, if not colide i will paint the texture color. But it do not have effect, I do not know nothing more to do to resolve it.

It is the Shader of the ground and have the coords of a geometry coming from vertex, this geometry will be draw after the ground shader. The color is coming too from the vertex shader, and the light point is a vec3 with a point in the space, I want to create a ray from the fragment position to the light point and see if it colides with the geometry that I have created in the code. After I will need to see if the intersection point in the texture is alpha or not, but it will be the next problem, now i need to see the shadow of the geometry in the ground.

#version 330 core

#define INTERSECT_EPSILON 0.0001

out vec4 FragColor;

in vec2 TexCoord;

in  vec3 geometryP;
in  vec3 lampP;
in  vec3 colorP;
in  vec3 imagePos;

//texture samplers
uniform sampler2D groundTexture;
uniform sampler2D treeTexture;

struct Ray
{
    vec3 Origin;
    vec3 Direction;
};


float dot(vec3 firstPoint, vec3 secondPoint)
{
    return (firstPoint.x * secondPoint.x + firstPoint.y * secondPoint.y + firstPoint.z * secondPoint.z);
}

vec3 cross(vec3 firstPoint, vec3 secondPoint)
{
    vec3 crossResult;

    crossResult.x =  firstPoint.y*secondPoint.z - firstPoint.z*secondPoint.y;
    crossResult.y  = firstPoint.z*secondPoint.x - firstPoint.x*secondPoint.z; 
    crossResult.z  = firstPoint.x*secondPoint.y - firstPoint.y*secondPoint.x;


    return crossResult;
}

bool IntersectTriangle(Ray ray, vec3 p0, vec3 p1, vec3 p2)
{

    float hit; 
    vec3 barycentricCoord;
    vec3 triangleNormal;

    vec3 e0 = p1 - p0;
    vec3 e1 = p0 - p2;
    triangleNormal = cross(e1 , e0);

    float valueDot = 1.0 / dot( triangleNormal, ray.Direction );

    vec3 e2 = ( valueDot ) * ( p0 - ray.Origin );
    vec3 i  = cross(ray.Direction , e2);

    barycentricCoord.y = dot( i, e1 );
    barycentricCoord.z = dot( i, e0 );
    barycentricCoord.x = 1.0 - (barycentricCoord.z + barycentricCoord.y);
    hit   = dot( triangleNormal, e2 );


    return  (hit > INTERSECT_EPSILON) && (barycentricCoord.x > 0 && barycentricCoord.y >0 && barycentricCoord.z > 0);
}

void main()
{
    vec3 firstPlane[3];
    firstPlane[0] = geometryP + vec3(-0.2, -0.2, 0.0);
    firstPlane[1] = geometryP + vec3(0.2, -0.2, 0.0);
    firstPlane[2] = geometryP + vec3(0.2,  0.5, 0.0);

    Ray ray1;
    ray1.Origin = imagePos;
    ray1.Direction = lampP;

    bool intersect = IntersectTriangle(ray1, firstPlane[0], firstPlane[1], firstPlane[2]);

    vec3 secondPlane[3];

    secondPlane[0] = geometryP + vec3(0.2, -0.2, 0.0);
    secondPlane[1] = geometryP + vec3(-0.2,  0.5, 0.0);
    secondPlane[2] = geometryP + vec3(0.2,  0.5, 0.0);

    if(!intersect)
    {
        intersect = IntersectTriangle(ray1, secondPlane[0], secondPlane[1], secondPlane[2]);
    }

    if(!intersect)
        FragColor = mix(texture(groundTexture, TexCoord), texture(treeTexture, TexCoord), 0.2);
    else
        FragColor = vec4(colorP, 0.0);
}

Someone can help me in this?

Edit: Result of the rays, I do not have tested the intersection with the tree texture alpha, the tree is a geometry shader, the ground is two triangles, and the shadow is made in the ground shader with the intersection calculation:

Answer 1

First of all note, that dot and cross are built-in glsl functions.

Write a GLSL function that evaluates if a point is inside a triangle in 3 dimensional space:

float PointInOrOn( vec3 P1, vec3 P2, vec3 A, vec3 B )
{
    vec3 CP1 = cross(B - A, P1 - A)
    vec3 CP2 = cross(B - A, P2 - A)
    return step(0.0, dot(CP1, CP2));
}

bool PointInTriangle( vec3 px, vec3 p0, vec3 p1, vec3 p2 )
{
    return 
        PointInOrOn(px, p0, p1, p2) *
        PointInOrOn(px, p1, p2, p0) *
        PointInOrOn(px, p2, p0, p1);
}

And another function that intersects a plane (which is defined by 3 points, by a ray:

struct Ray
{
    vec3 Origin;
    vec3 Direction;
};

vec3 IntersectPlane(Ray ray, vec3 p0, vec3 p1, vec3 p2)
{
    vec3 D = ray.Direction;
    vec3 N = cross(p1-p0, p2-p0);
    vec3 X = ray.Origin + D * dot(p0 - ray.Origin, N) / dot(D, N);

    return X;
}

Find the intersection point and evaluate if it is in the triangle:

bool IntersectTriangle(Ray ray, vec3 p0, vec3 p1, vec3 p2)
{
    vec3 X = IntersectPlane(ray, p0, p1, p2);
    return PointInTriangle(X, p0, p1, p2);
}

See the following explanation.

---

Intersection of a ray and a triangle primitive

The ray is defined by a point R0 and a direction D.
The plane is defined by a triangle with the three points PA, PB, and PC.

The normal vector of the plane can be calculated by the cross product of 2 legs of the triangle:

N  =  cross(PC-PA, PB-PA)

The normal distance n of the point R0 to the plane is:

n  =  | R0 - PA | * cos(alpha)  =  dot(PA - R0, N)

It follows that the distance d of the intersection point X to the origin of the ray R0 is:

d  =  n / cos(beta)  =  n / dot(D, N)

The intersection point X is:

X  =  R0 + D * d  =  R0 + D * dot(PA - R0, N) / dot(D, N)

Note, it is not necessary to normalize N and D, because D * dot(PA - R0, N) / dot(D, N) is equal to normalze(D) * dot(PA - R0, normalze(N)) / dot(normalze(D), normalze(N)).

---

To find out, if a point is inside a triangle, has to be tested, if the line from a corner point to the intersection point is between the to legs which are connect to the corner point. The triangle is defined by the points A, B, C and the point to be tested is P:

bool PointInOrOn( P1, P2, A, B )
{
    CP1 = cross( B - A, P1 - A )
    CP2 = cross( B - A, P2 - A )
    return dot( CP1, CP2 ) >= 0
}

bool PointInOrOnTriangle( P, A, B, C )
{
    return PointInOrOn( P, A, B, C ) &&
           PointInOrOn( P, B, C, A ) &&
           PointInOrOn( P, C, A, B );
}