How to make a proper planar reflection using ogl with a perspective camera?

I'm trying to reflect a WebGL scene in a plane with ogl, as a mirror would do. I made a naive implementation of the three.js reflector but all I got is a distorted image that doesn't match what an actual reflection would look like.

I get the general idea of how the reflector works by rendering the scene from the virtual reflected point of view of the original camera, adjusting the projection matrix of the virtual camera to render only what projects onto the plane, and later mapping the resulting texture to the plane. But I don't have a complete understanding of all the involved computations (particularly for the two last steps I described).

All I did is copy line by line the three.js reflector and change function names or references, but I can't tell where I'm wrong or what I missed. I would really appreciate any help !

// objects/Reflector.js

import { Transform, Mesh, Plane, Program, RenderTarget, Camera, Vec3, Vec4, Quat, Mat4 } from 'ogl'
import { dot } from 'ogl/src/math/functions/Vec4Func'

import vertex from '../shaders/reflector.vert'
import fragment from '../shaders/reflector.frag'

import { setFromNormalAndCoplanarPoint, transformMat4 } from '../math/PlaneFunc'
import { getRotationMatrix } from '../math/Mat4Func'
import { reflect } from '../math/Vec3Func'

export class Reflector extends Mesh {
  constructor(gl, {
    scene,
    camera,
    renderSize = 512,
    clipBias = 0
  }) {
    const renderTarget = new RenderTarget(gl, {
      width: renderSize,
      height: renderSize
    })

    super(gl, {
      geometry: new Plane(gl),
      program: new Program(gl, {
        vertex,
        fragment,
        uniforms: {
          textureMatrix: { value: new Mat4() },
          diffuseMap: { value: renderTarget.texture }
        }
      })
    })

    this.viewCamera = camera
    this.clipBias = clipBias
    this.reflectionCamera = new Camera(gl)
    this.reflectionPlane = new Vec4()
    this.reflectionWorldMatrixInverse = new Mat4()
    this.reflectionQuaternion = new Quat()
    this.worldPosition = new Vec3()
    this.normal = new Vec3()
    this.view = new Vec3()
    this.lookAtPosition = new Vec3()
    this.rotationMatrix = new Mat4()
    this.target = new Vec3()
    this.textureMatrix = this.program.uniforms.textureMatrix.value

    this.renderParameters = {
      scene,
      target: renderTarget,
      camera: this.reflectionCamera
    }
  }

  update() {
    this.worldMatrix.getTranslation(this.worldPosition)
    getRotationMatrix(this.rotationMatrix, this.worldMatrix)

    this.normal
      .set(0, 0, 1)
      .applyMatrix4(this.rotationMatrix)

    this.view.sub(this.worldPosition, this.viewCamera.worldPosition)

    if (this.view.dot(this.normal) > 0) {
      return
    }

    reflect(this.view, this.view, this.normal)
      .negate()
      .add(this.worldPosition)

    getRotationMatrix(this.rotationMatrix, this.viewCamera.worldMatrix)

    this.lookAtPosition.set(0, 0, -1)
    this.lookAtPosition.applyMatrix4(this.rotationMatrix)
    this.lookAtPosition.add(this.viewCamera.worldPosition)

    this.target.sub(this.worldPosition, this.lookAtPosition)
    reflect(this.target, this.target, this.normal)
      .negate()
      .add(this.worldPosition)

    this.reflectionCamera.position.copy(this.view)

    this.reflectionCamera.up
      .set(0, 1, 0)
      .applyMatrix4(this.rotationMatrix)

    reflect(this.reflectionCamera.up, this.reflectionCamera.up, this.normal)
    this.reflectionCamera.lookAt(this.target)

    this.reflectionCamera.perspective({
      near: this.viewCamera.near,
      far: this.viewCamera.far,
      fov: this.viewCamera.fov,
      aspect: 1
    })

    this.reflectionCamera.worldMatrixNeedsUpdate = true
    this.reflectionCamera.updateMatrixWorld()
    this.reflectionWorldMatrixInverse.inverse(this.reflectionCamera.worldMatrix)
    this.reflectionCamera.updateFrustum()

    this.textureMatrix.set(
      0.5, 0.0, 0.0, 0.5,
      0.0, 0.5, 0.0, 0.5,
      0.0, 0.0, 0.5, 0.5,
      0.0, 0.0, 0.0, 1.0
    )

    this.textureMatrix
      .multiply(this.reflectionCamera.projectionMatrix)
      .multiply(this.reflectionWorldMatrixInverse)
      .multiply(this.worldMatrix)

    setFromNormalAndCoplanarPoint(this.reflectionPlane, this.normal, this.worldPosition)
    transformMat4(this.reflectionPlane, this.reflectionPlane, this.reflectionWorldMatrixInverse)

    const projectionMatrix = this.reflectionCamera.projectionMatrix

    this.reflectionQuaternion.set(
      (Math.sign(this.reflectionPlane.x) + projectionMatrix[8]) / projectionMatrix[0],
      (Math.sign(this.reflectionPlane.y) + projectionMatrix[9]) / projectionMatrix[5],
      -1,
      (1 + projectionMatrix[10]) / projectionMatrix[14]
    )

    const f = 2 / dot(this.reflectionPlane, this.reflectionQuaternion)

    this.reflectionPlane.x *= f
    this.reflectionPlane.y *= f
    this.reflectionPlane.z *= f
    this.reflectionPlane.w *= f

    projectionMatrix[2] = this.reflectionPlane.x
    projectionMatrix[6] = this.reflectionPlane.y
    projectionMatrix[10] = this.reflectionPlane.z + 1 - this.clipBias
    projectionMatrix[14] = this.reflectionPlane.w

    this.helper.position.copy(this.reflectionCamera.position)
    this.helper.rotation.copy(this.reflectionCamera.rotation)

    this.visible = false

    this.gl.renderer.render(this.renderParameters)

    this.visible = true
  }
}

// shaders/reflector.vert

precision highp float;

attribute vec3 position;

uniform mat4 modelViewMatrix;
uniform mat4 projectionMatrix;
uniform mat4 textureMatrix;

varying vec4 vUv;

void main() {
  vUv = textureMatrix * vec4(position, 1.);

  gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.);
}

// shaders/reflector.frag

precision highp float;

uniform sampler2D diffuseMap;

varying vec4 vUv;

void main() {
  vec3 diffuse = texture2DProj(diffuseMap, vUv).rgb;

  gl_FragColor = vec4(diffuse, 1.);
}

Here are some math functions available in three.js that I had to implement myself:

// math/Mat4Func.js

import { length } from 'ogl/src/math/functions/Vec3Func'

export function getRotationMatrix(out, m) {
  const sX = 1 / length(m.slice(0, 3))
  const sY = 1 / length(m.slice(4, 7))
  const sZ = 1 / length(m.slice(8, 11))

  out[0] = m[0] * sX
  out[1] = m[1] * sX
  out[2] = m[2] * sX
  out[3] = 0

  out[4] = m[4] * sY
  out[5] = m[5] * sY
  out[6] = m[6] * sY
  out[7] = 0

  out[8] = m[8] * sZ
  out[9] = m[9] * sZ
  out[10] = m[10] * sZ
  out[11] = 0

  out[12] = 0
  out[13] = 0
  out[14] = 0
  out[15] = 1

  return out
}

// math/PlaneFunc.js

import { normalFromMat4 } from 'ogl/src/math/functions/Mat3Func'

import {
  dot,
  normalize,
  transformMat4 as transformMat4Vec3,
  transformMat3 as transformMat3Vec3
} from 'ogl/src/math/functions/Vec3Func'

const normal = []
const normalMatrix = []
const coplanarPoint = []

export function transformMat4(out, p, m) {
  normalFromMat4(normalMatrix, m)
  getCoplanarPoint(coplanarPoint, p)
  transformMat4Vec3(coplanarPoint, coplanarPoint, m)
  transformMat3Vec3(normal, p, normalMatrix)
  normalize(normal, normal)
  setFromNormalAndCoplanarPoint(out, normal, coplanarPoint)

  return out
}

export function getCoplanarPoint(out, p) {
  out[0] = p[0] * -p[3]
  out[1] = p[1] * -p[3]
  out[2] = p[2] * -p[3]

  return out
}

export function setFromNormalAndCoplanarPoint(out, n, c) {
  out[0] = n[0]
  out[1] = n[1]
  out[2] = n[2]
  out[3] = -dot(c, n)

  return out
}

// math/Vec3Func.js

import { dot } from 'ogl/src/math/functions/Vec3Func'

export function reflect(out, a, b) {
  const f = 2 * dot(a, b)

  out[0] = a[0] - b[0] * f
  out[1] = a[1] - b[1] * f
  out[2] = a[2] - b[2] * f

  return out
}

Solution

Mr.Coder's comment reminded me of this post and made me feel like I should post the solution I came up with 2 years ago... I guess it could help some people.

Here is a working example of the planar reflector, this is a code I wrote some times ago now so it is not super clean and would need some refactoring as well as a typescript rewrite, but it is a good starting point if you are stuck on this issue as I was when I made this post:

import {
  RenderTarget,
  Camera,
  Plane,
  Program,
  Mesh,
  Vec2,
  Vec3,
  Vec4,
  Mat3,
  Mat4
} from 'ogl'

import { transformMat3, normalize } from 'ogl/src/math/functions/Vec3Func'
import { scale as scaleVec4 } from 'ogl/src/math/functions/Vec4Func'

export class Reflector extends Mesh {
  constructor(gl, {
    scene,
    camera,
    clipBias = 0,
    width = 1,
    height = 1,
    renderWidth = 512,
    renderHeight = 512
  }) {
    const renderTarget = new RenderTarget(gl, {
      width: renderWidth,
      height: renderHeight
    })
    const textureMatrix = new Mat4()

    super(gl, {
      geometry: new Plane(gl, { width, height }),
      program: new Program(gl, {
        uniforms: {
          reflectionMap: {
            value: renderTarget.texture
          },
          reflectionMatrix: {
            value: textureMatrix
          }
        },
        vertex: `#version 300 es
          precision highp float;

          uniform mat4 projectionMatrix;
          uniform mat4 modelViewMatrix;
          uniform mat4 reflectionMatrix;

          in vec3 position;
          in vec2 uv;

          out vec4 vReflectionUv;

          void main() {
            vReflectionUv = reflectionMatrix * vec4(position, 1.0);
            gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.);
          }
        `,
        fragment: `#version 300 es
          precision highp float;

          uniform sampler2D reflectionMap;

          in vec4 vReflectionUv;

          out vec4 FragColor;

          void main() {
            vec4 reflection = textureProj(reflectionMap, vReflectionUv);
            FragColor = reflection + vec4(0.1, 0.1, 0.1, 0.);
          }
        `
      })
    })

    this.camera = camera
    this.clipBias = clipBias
    this.renderTarget = renderTarget
    this.textureMatrix = textureMatrix
    this.resolution = new Vec2(renderWidth, renderHeight)
    this.virtualCamera = new Camera(gl)
    this.normal = new Vec3()
    this.view = new Vec3()
    this.plane = new Vec4()
    this.q = new Vec4()
    this.referencePoint = new Vec3()
    this.lookAtPosition = new Vec3()
    this.targetPosition = new Vec3()
    this.normalMatrix = new Mat3()
    this.rotationMatrix = new Mat4()

    this.renderParameters = {
      scene,
      camera: this.virtualCamera,
      target: this.renderTarget
    }
  }

  update() {
    this.camera.updateMatrixWorld()
    this.updateMatrixWorld()

    extractRotationMatrix(this.worldMatrix, this.rotationMatrix)

    this.normal
      .set(0, 0, 1)
      .applyMatrix4(this.rotationMatrix)

    this.view.sub(this.position, this.camera.worldPosition)

    if (this.view.dot(this.normal) > 0) {
      return
    }

    const distance = this.view.len()

    reflect(this.view, this.normal).negate()

    this.view.add(this.position)

    extractRotationMatrix(this.camera.worldMatrix, this.rotationMatrix)

    this.lookAtPosition
      .set(0, 0, -1)
      .applyMatrix4(this.rotationMatrix)
      .add(this.camera.worldPosition)

    reflect(
      this.targetPosition.sub(this.position, this.lookAtPosition),
      this.normal
    )
      .negate()
      .add(this.position)

    this.virtualCamera.position.copy(this.view)

    reflect(
      this.virtualCamera.up.set(0, 1, 0).applyMatrix4(this.rotationMatrix),
      this.normal
    )

    this.virtualCamera.lookAt(this.targetPosition)
    this.virtualCamera.near = this.camera.near
    this.virtualCamera.far = this.camera.far
    this.virtualCamera.fov = this.camera.fov

    this.virtualCamera.updateMatrixWorld(true)
    this.virtualCamera.projectionMatrix.copy(this.camera.projectionMatrix)

    this.textureMatrix.set(
      0.5, 0.0, 0.0, 0.0,
      0.0, 0.5, 0.0, 0.0,
      0.0, 0.0, 0.5, 0.0,
      0.5, 0.5, 0.5, 1.0
    )

    this.textureMatrix
      .multiply(this.virtualCamera.projectionMatrix)
      .multiply(this.virtualCamera.viewMatrix)
      .multiply(this.worldMatrix)

    this.plane
      .set(
        this.normal.x,
        this.normal.y,
        this.normal.z,
        -this.position.dot(this.normal)
      )

    applyPlaneMatrix4(
      this.plane,
      this.virtualCamera.viewMatrix,
      this.normalMatrix,
      this.referencePoint
    )

    const projectionMatrix = this.virtualCamera.projectionMatrix

    this.q.x = (Math.sign(this.plane.x) + projectionMatrix[8]) / projectionMatrix[0]
    this.q.y = (Math.sign(this.plane.y) + projectionMatrix[9]) / projectionMatrix[5]
    this.q.z = -1
    this.q.w = (1 + projectionMatrix[10]) / projectionMatrix[14]

    scaleVec4(this.plane, this.plane, 2 / this.plane.dot(this.q))

    projectionMatrix[2] = this.plane.x
    projectionMatrix[6] = this.plane.y
    projectionMatrix[10] = this.plane.z + 1 - this.clipBias
    projectionMatrix[14] = this.plane.w

    this.visible = false
    this.gl.renderer.render(this.renderParameters)
    this.visible = true
  }

  setSize(width, height) {
    this.resolution.set(width, height)
    this.renderTarget.setSize(width, height)
  }

  remove() {
    this.gl.deleteFramebuffer(this.renderTarget.buffer)

    for (const texture of this.renderTarget.textures) {
      this.gl.deleteTexture(texture)
    }
  }
}

function extractRotationMatrix(matrix, output) {
  const sX = 1 / Math.hypot(matrix[0], matrix[1], matrix[2])
  const sY = 1 / Math.hypot(matrix[4], matrix[5], matrix[6])
  const sZ = 1 / Math.hypot(matrix[8], matrix[9], matrix[10])

  output[0] = matrix[0] * sX
  output[1] = matrix[1] * sX
  output[2] = matrix[2] * sX
  output[3] = 0

  output[4] = matrix[4] * sY
  output[5] = matrix[5] * sY
  output[6] = matrix[6] * sY
  output[7] = 0

  output[8] = matrix[8] * sZ
  output[9] = matrix[9] * sZ
  output[10] = matrix[10] * sZ
  output[11] = 0

  output[12] = 0
  output[13] = 0
  output[14] = 0
  output[15] = 1

  return output
}

function reflect(vector, normal, output = vector) {
  const f = 2 * vector.dot(normal)
  output[0] -= normal[0] * f
  output[1] -= normal[1] * f
  output[2] -= normal[2] * f

  return output
}

function applyPlaneMatrix4(
  plane,
  matrix,
  normalMatrix,
  referencePoint,
  output = plane
) {
  normalMatrix.getNormalMatrix(matrix)
  referencePoint.copy(plane).scale(-plane.w).applyMatrix4(matrix)
  normalize(plane, transformMat3(plane, plane, normalMatrix))
  plane.w = -referencePoint.dot(plane)

  return output;
}

The main problem was the initialization of the texture matrix. In the old code, it was made like that:

this.textureMatrix.set(
  0.5, 0.0, 0.0, 0.5,
  0.0, 0.5, 0.0, 0.5,
  0.0, 0.0, 0.5, 0.5,
  0.0, 0.0, 0.0, 1.0
)

But it should have been made this way from the beginning:

this.textureMatrix.set(
  0.5, 0.0, 0.0, 0.0,
  0.0, 0.5, 0.0, 0.0,
  0.0, 0.0, 0.5, 0.0,
  0.5, 0.5, 0.5, 1.0
)

I made this mistake because columns and rows are reversed in threejs compared to ogl !