how to get z depth using glreadpixels()

Question

can anyone tell me how to use

GL11.glreadpixels()

in lwjgl to get the z depth of the ray casted by mouse? i can get x,y and of the view before transforming it to ray

float x = Mouse.getX();
float y = Mouse.getY();

but i don't know how to use glreadpixels as when i use it it doesn't give any significance both calculateMousePoint and calculate MouseRay give the same result

public static float getZDepth(int x, int y)
{
   ByteBuffer zdepth = allocBytes(SIZE_FLOAT);
   GL11.glReadPixels(x, y, 1, 1, GL11.GL_DEPTH_COMPONENT, GL11.GL_FLOAT, zdepth);
   return ( (float) (zdepth.getFloat(0)));
}
private Vector3f calculateMouseRay() {
    float mouseX = Mouse.getX();
    float mouseY = Mouse.getY();
    Vector2f normalizedCoords = getNormalisedDeviceCoordinates(mouseX, mouseY);
    Vector4f clipCoords = new Vector4f(normalizedCoords.x, normalizedCoords.y, -1.0f, 1.0f);
    Vector4f eyeCoords = toEyeCoords(clipCoords);
    Vector3f worldRay = toWorldCoords(eyeCoords);
    return worldRay;
}

private Vector3f calculateMousePoint() {
    float mouseX = Mouse.getX();
    float mouseY = Mouse.getY();
    float mouseZ = getZDepth((int)mouseX,(int) mouseY);

    Vector2f normalizedCoords = getNormalisedDeviceCoordinates(mouseX, mouseY);
    Vector4f clipCoords = new Vector4f(normalizedCoords.x, normalizedCoords.y, mouseZ, 1.0f);
    Vector4f eyeCoords = toEyeCoords2(clipCoords);
    Vector3f worldRay = toWorldCoords(eyeCoords);
    return worldRay;
}
private Vector3f toWorldCoords(Vector4f eyeCoords) {
    Matrix4f invertedView = Matrix4f.invert(viewMatrix, null);
    Vector4f rayWorld = Matrix4f.transform(invertedView, eyeCoords, null);
    Vector3f mouseRay = new Vector3f(rayWorld.x, rayWorld.y, rayWorld.z);
    mouseRay.normalise();
    return mouseRay;
}

private Vector4f toEyeCoords(Vector4f clipCoords) {
    Matrix4f invertedProjection = Matrix4f.invert(projectionMatrix, null);
    Vector4f eyeCoords = Matrix4f.transform(invertedProjection, clipCoords, null);
    return new Vector4f(eyeCoords.x, eyeCoords.y, -1f, 0f);
}
private Vector4f toEyeCoords2(Vector4f clipCoords) {
    Matrix4f invertedProjection = Matrix4f.invert(projectionMatrix, null);
    Vector4f eyeCoords = Matrix4f.transform(invertedProjection, clipCoords, null);
    return new Vector4f(eyeCoords.x, eyeCoords.y, eyeCoords.z, 0f);
}
private Vector2f getNormalisedDeviceCoordinates(float mouseX, float mouseY) {
    float x = (2.0f * mouseX) / Display.getWidth() - 1f;
    float y = (2.0f * mouseY) / Display.getHeight() - 1f;
    return new Vector2f(x, y);
}

Answer 1

From the documentation, x,y and _width, height stand for the area to shoot. type is the type of data, then data is the result.

Last, the most important here is the format parameter : You can select what you want to retreive. For you, it will be GL_DEPTH_COMPONENT :

float zmouse;
GL11.glReadnPixels(xmouse, ymouse, 1, 1, GL11.GL_DEPTH_COMPONENT, GL11.GL_FLOAT, zmouse)

You have the Z depth, now you have to convert it into the good space. Actually, it's the clip space and I think you would get the camera space one. So you have to multiply the "mouse" point by the invert of projection and view matrix, something like realPoint = inverse(projection * view * model) * (xmouse, ymousen, zmouse).

Finally, realPoint is the point in the 3D space.

With your code as example, this should do the work :

public static float getZDepth(int x, int y)
{
   ByteBuffer zdepth = allocBytes(SIZE_FLOAT);
   GL11.glReadPixels(x, y, 1, 1, GL11.GL_DEPTH_COMPONENT, GL11.GL_FLOAT, zdepth);
   return ( (float) (zdepth.getFloat(0)));
}

private Vector3f calculateMousePoint(Vector3f point) {
    float x = Mouse.getX();
    float y = Mouse.getY();
    float z = getZDepth((int)mouseX,(int) mouseY);
    return project(new Vector3f(x,y,z), new Vector4f(0,0,Display.getWidth(), Display.getHeight()));
}
private Vector3f calculateFarPoint(Vector3f point) {
    float x = Mouse.getX();
    float y = Mouse.getY();
    return project(new Vector3f(x,y,1.), new Vector4f(0,0,Display.getWidth(), Display.getHeight()));
}
// Code translated from GLM_GTC_matrix_transform 
//(https://glm.g-truc.net/0.9.2/api/a00245.html#gac38d611231b15799a0c06c54ff1ede43)
private Vector3f project(Vector3f point, Vector4f viewport)
{
    Matrix4f Inverse = Matrix4f.invert(projectionMatrix * viewMatrix)
    Vector4f tmp = new Vector4f(point.x, point.y, point.z, 1.f);
    tmp.x = (tmp.x - viewport.x) / viewport.z;
    tmp.y = (tmp.y - viewport.y) / viewport.w;
    tmp = tmp * 2.0 - 1.0;
    Vector4f obj = Inverse * tmp;
    obj /= obj.w;
    return new Vector3f(obj.x, obj.y, obj.z);
}