how do i normalize a circle drawn in glsl so the resolution doesn't stretch?

Question

i'm attempting to use glsl to create a circle around a particular sprite. however, the resolution is rectangular (800x600) which leads to a slightly stretched circle:

i've tried the answers here, but i can't figure out how to properly apply them to my particular shader. what do i need to do

#define SHADER_NAME SPOTLIGHT_FS
precision highp float;

uniform sampler2D uMainSampler;
varying vec2 outTexCoord;
// coord is the non-normalized position if the sprite that the circle follows
uniform vec2 coord;
vec2 px = vec2(1.0/800.0, 1.0/600.0);

float getDistance(vec2 pixelCoord, vec2 playerCoord) {
    // pixelCoord is normalized, but playerCoord is passed in as-is
    return distance(pixelCoord, playerCoord*px);
}

void main( void )
{
    if (getDistance(outTexCoord, coord) > 0.125) {
        gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
    }
    else {
        gl_FragColor = texture2D(uMainSampler, outTexCoord);
    }
}

Answer 1

You must consider the aspect ratio of the viewport. For example:

vec2 px = vec2(1.0/800.0, 1.0/600.0);
float aspect = 800.0 / 600.0; 

float getDistance(vec2 pixelCoord, vec2 playerCoord) {
    vec2 p = playerCoord * px * vec2(1.0, 1.0/aspect);
    // pixelCoord is normalized, but playerCoord is passed in as-is
    return distance(pixelCoord, p);
}

Alternatively, you can calculate the distance in pixel units:

vec2 resolution = vec2(800.0, 600.0);
float radius = 100.0; // radius in pixel

float getDistance(vec2 pixelCoord, vec2 playerCoord) {
    return distance(pixelCoord*resolution, playerCoord);
}

void main( void )
{
    if (getDistance(outTexCoord, coord) > radius) {
        gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
    }
    else {
        gl_FragColor = texture2D(uMainSampler, outTexCoord);
    }
}