OpenGL - GLSL Geometry Shader - gl_Position to gl_FragCoord

Question

How can I convert gl_Position to gl_FragCoord in order to check if the gl_FragCoord is between 2 points?

See what I have done so far below "it glitch, there is a bit of red". I think I am badly converting gl_Position to gl_FragCoord.

Can someone help me with my problem? FYI: I am new with GLSL.

Geometry Shader:

#version 330 core

layout (triangles) in;
layout (triangle_strip, max_vertices = 3) out;

out vec3 color;

out vec2 v1;
out vec2 v2;
out vec2 v3;

vec2 halfScreenSize;

in DATA
{
    vec3 color;
    vec2 screenSize;
    mat4 projection;
} data_in[];

vec2 ToFragCoord(vec2 position)
{    
    return position * halfScreenSize;
}

void main()
{
    halfScreenSize = vec2(data_in[0].screenSize.x / 2, data_in[0].screenSize.y / 2);

    v1 = ToFragCoord((data_in[0].projection * gl_in[0].gl_Position).xy);
    v2 = ToFragCoord((data_in[1].projection * gl_in[1].gl_Position).xy);
    v3 = ToFragCoord((data_in[2].projection * gl_in[2].gl_Position).xy);

    gl_Position = data_in[0].projection * gl_in[0].gl_Position;
    color = data_in[0].color;
    EmitVertex();

    gl_Position = data_in[1].projection * gl_in[1].gl_Position;
    color = data_in[1].color;
    EmitVertex();

    gl_Position = data_in[2].projection * gl_in[2].gl_Position;
    color = data_in[2].color;
    EmitVertex();

    EndPrimitive();
}

Fragment:

#version 330 core

in vec3 color;

in vec2 v1;
in vec2 v2;
in vec2 v3;

bool Check(vec2 a, vec2 b, vec2 c)
{    
    float len = distance(a,c) + distance(b,c);

    if(len <= distance(a,b) + 1.0f 
    && len >= distance(a,b) - 1.0f)
    {
        return true;
    }

    return false;
}

void main()
{
    if(Check(v1, v2, gl_FragCoord.xy)
    || Check(v2, v3, gl_FragCoord.xy))
    {
        gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);
    }
    else
    {
        gl_FragColor = vec4(1.0, 1.0, 1.0, 1.0);
    }
}

Answer 1

The clip space coordinate is a Homogeneous coordinate. You must do a perspective derivation to get a Cartesian normalized device space coordinate.

Normalized device coordinates are in the range [-1.0, 1.0]. The conversion of a normalized device coordinate into the range [0.0, 1.0] is therefore x' = (x + 1) / 2.

gl_FragCoord contains window relative x and y coordinate and the depth in the z coordinate. So you have to multiply the x and y component of the coordinate in the range [0.0, 1.0] with the resolution of the window.

vec4 vh1 = data_in[0].projection * gl_in[0].gl_Position;
v1 = (vh1.xy / vh1.w + 1.0) / 2.0 * screenSize.xy;

Note: vh1 = data_in[0].projection * gl_in[0].gl_Position is only necessary if gl_in[0].gl_Position is not already transformed with the projection matrix in the vertex shader. If it is already transformed, it is simply vh1 = gl_in[0].gl_position;.