What is the equation for bilinear filtering between 2 mip levels based on texture coordinate derivatives

Tagging as OpenGL since the question should be OpenGL/WebGL agnostic

I'm trying to implement shader based texture filtering. I had a previous question and an answer that helped me fix a bug in selecting a mip level. Effectively that example was doing "NEAREST" between mip levels.

Now I want to do bilinear filtering between the levels (NEAREST_MIPMAP_LINEAR).

Taking the previous example, it computes a mipLevel

It then does

 outColor = colors[int(mipLevel)]

Where colors is an array of 8 vec4 colors that match the colors of the mip levels of a corresponding texture. The example draws on the left using a texture and on the right using this color table and adjusting texture coordinates to force mip level selection.

According to the spec section 3.8.10.4, the formula to interpolate between mip levels is

 τ = [1 − frac(λ)]τ1 + frac(λ)τ2.

Where λ is mipLevel from above and τ1 and τ2 are the colors from the mip levels. That seems like it just translates to this

 t1 = colors[int(mipLevel)];
 t2 = colors[int(mipLevel)];

 outColor = mix(t1, t2, fract(mipLevel));

But when I try it the output doesn't match actual filtering by the hardware. Below, if it was working, the 2 sides should change color in sync.

What am I missing?

html, body {
  margin: 0;
  font-family: monospace;
  height: 100%;
}
canvas {
  display: block;
  width: 100%;
  height: 100%;
}

<canvas id="c"></canvas>

<script type="module">
import * as twgl from 'https://twgljs.org/dist/5.x/twgl-full.module.js';

const vs = `#version 300 es
uniform mat4 u_worldViewProjection;
uniform mat3 u_texMat;

out vec2 v_texCoord;

const vec2 position[6] = vec2[6](
  vec2(0, 0),
  vec2(1, 0),
  vec2(0, 1),
  vec2(0, 1),
  vec2(1, 0),
  vec2(1, 1));

void main() {
  vec2 p = position[gl_VertexID];
  v_texCoord = (u_texMat * vec3(p, 1)).xy;
  gl_Position = u_worldViewProjection * vec4(p, 0, 1);
}
`;
const fsTex = `#version 300 es
precision highp float;

in vec2 v_texCoord;

uniform sampler2D u_tex;

out vec4 outColor;

void main() {
  outColor = texture(u_tex, v_texCoord);
}
`;
const fsMipLevel = `#version 300 es
precision highp float;

in vec2 v_texCoord;

out vec4 outColor;

const vec4 colors[8] = vec4[8](
  vec4(  1,   0,   0, 1), // 0: red
  vec4(  1,   1,   0, 1), // 1: yellow
  vec4(  0,   1,   0, 1), // 2: green
  vec4(  0,   1,   1, 1), // 3: cyan
  vec4(  0,   0,   1, 1), // 4: blue
  vec4(  1,   0,   1, 1), // 5: magenta
  vec4(0.5, 0.5, 0.5, 1), // 6: gray
  vec4(  1,   1,   1, 1));// 7: white

void main() {
  vec2 size = vec2(128.0, 128.0); // size of the texture in your example
  vec2 dx = dFdx(v_texCoord * size);
  vec2 dy = dFdy(v_texCoord * size);
  float deltaMaxSq = max(dot(dx, dx), dot(dy, dy));
  float mipLevel = 0.5 * log2(deltaMaxSq) + 0.5;
  
  vec4 t1 = colors[int(mipLevel)];
  vec4 t2 = colors[int(mipLevel + 1.0)];
  
  outColor = mix(t1, t2, fract(mipLevel));
}
`;

const colors = [
  '#F00',
  '#FF0',
  '#0F0',
  '#0FF',
  '#00F',
  '#F0F',
  '#888',
  '#FFF',
];


function createMips(colors) {
  const ctx = document.createElement('canvas').getContext('2d');
  const numMips = colors.length;
  return colors.map((color, i) => {
    const size = 2 ** (numMips - i - 1);
    ctx.canvas.width = size;
    ctx.canvas.height = size;
    ctx.fillStyle = color;
    ctx.fillRect(0, 0, size, size);
    return ctx.getImageData(0, 0, size, size);
  });
}

function main() {
  const m4 = twgl.m4;
  const gl = document.getElementById("c").getContext("webgl2");
  if (!gl) {
    alert("Sorry, this example requires WebGL 2.0");  // eslint-disable-line
    return;
  }
  
  const texProgramInfo = twgl.createProgramInfo(gl, [vs, fsTex]);
  const mipProgramInfo = twgl.createProgramInfo(gl, [vs, fsMipLevel]);

  const texImage = gl.createTexture();
  gl.bindTexture(gl.TEXTURE_2D, texImage);
  const data = createMips(colors);
  data.forEach(({width, height, data}, level) => {
    gl.texImage2D(gl.TEXTURE_2D, level, gl.RGBA8, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, data);
  });
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST_MIPMAP_LINEAR);

  const lerp = (a, b, t) => a + (b - a) * t;

  function render(time) {
    time *= 0.001;
    twgl.resizeCanvasToDisplaySize(gl.canvas);
    gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
    gl.clearColor(0.3, 0.3, 0.3, 1);
    gl.clear(gl.COLOR_BUFFER_BIT);

    const uniforms = {};

    const s = lerp(1, 128, Math.sin(time) * 0.5 + 0.5);
    uniforms.u_texMat = [
      s, 0, 0,
      0, s, 0,
      0, 0, 1,
    ];
    uniforms.u_worldViewProjection = m4.translation([-1.01, -0.5, 0]);

    gl.useProgram(texProgramInfo.program);
    twgl.setUniforms(texProgramInfo, uniforms);
    gl.drawArrays(gl.TRIANGLES, 0, 6);

    uniforms.u_worldViewProjection = m4.translation([0.01, -0.5, 0]);

    gl.useProgram(mipProgramInfo.program);
    twgl.setUniforms(mipProgramInfo, uniforms);
    gl.drawArrays(gl.TRIANGLES, 0, 6);

    requestAnimationFrame(render);
  }
  requestAnimationFrame(render);
}

main();
</script>

Solution

For the "NEAREST" filtering you have to round the index (as the name "NEAREST" suggests):

c[round(l)]

However, with "LINEAR" filtering, you must interpolate between the color with the next smaller index and the next larger index:

mix(c[int(l)], c[int(l)+1], fract(l))

"NEAREST":

float mipLevel = 0.5 * log2(deltaMaxSq) + 0.5;
outColor = colors[int(mipLevel)];

"LINEAR" (you have to remove the + 0.5):

float mipLevel = 0.5 * log2(deltaMaxSq);
vec4 t1 = colors[int(mipLevel)];
vec4 t2 = colors[int(mipLevel + 1.0)];
outColor = mix(t1, t2, fract(mipLevel));

html, body {
  margin: 0;
  font-family: monospace;
  height: 100%;
}
canvas {
  display: block;
  width: 100%;
  height: 100%;
}

<canvas id="c"></canvas>

<script type="module">
import * as twgl from 'https://twgljs.org/dist/5.x/twgl-full.module.js';

const vs = `#version 300 es
uniform mat4 u_worldViewProjection;
uniform mat3 u_texMat;

out vec2 v_texCoord;

const vec2 position[6] = vec2[6](
  vec2(0, 0),
  vec2(1, 0),
  vec2(0, 1),
  vec2(0, 1),
  vec2(1, 0),
  vec2(1, 1));

void main() {
  vec2 p = position[gl_VertexID];
  v_texCoord = (u_texMat * vec3(p, 1)).xy;
  gl_Position = u_worldViewProjection * vec4(p, 0, 1);
}
`;
const fsTex = `#version 300 es
precision highp float;

in vec2 v_texCoord;

uniform sampler2D u_tex;

out vec4 outColor;

void main() {
  outColor = texture(u_tex, v_texCoord);
}
`;
const fsMipLevel = `#version 300 es
precision highp float;

in vec2 v_texCoord;

out vec4 outColor;

const vec4 colors[8] = vec4[8](
  vec4(  1,   0,   0, 1), // 0: red
  vec4(  1,   1,   0, 1), // 1: yellow
  vec4(  0,   1,   0, 1), // 2: green
  vec4(  0,   1,   1, 1), // 3: cyan
  vec4(  0,   0,   1, 1), // 4: blue
  vec4(  1,   0,   1, 1), // 5: magenta
  vec4(0.5, 0.5, 0.5, 1), // 6: gray
  vec4(  1,   1,   1, 1));// 7: white

void main() {
  vec2 size = vec2(128.0, 128.0); // size of the texture in your example
  vec2 dx = dFdx(v_texCoord * size);
  vec2 dy = dFdy(v_texCoord * size);
  float deltaMaxSq = max(dot(dx, dx), dot(dy, dy));
  float mipLevel = 0.5 * log2(deltaMaxSq);
  
  vec4 t1 = colors[int(mipLevel)];
  vec4 t2 = colors[int(mipLevel + 1.0)];
  
  outColor = mix(t1, t2, fract(mipLevel));
}
`;

const colors = [
  '#F00',
  '#FF0',
  '#0F0',
  '#0FF',
  '#00F',
  '#F0F',
  '#888',
  '#FFF',
];


function createMips(colors) {
  const ctx = document.createElement('canvas').getContext('2d');
  const numMips = colors.length;
  return colors.map((color, i) => {
    const size = 2 ** (numMips - i - 1);
    ctx.canvas.width = size;
    ctx.canvas.height = size;
    ctx.fillStyle = color;
    ctx.fillRect(0, 0, size, size);
    return ctx.getImageData(0, 0, size, size);
  });
}

function main() {
  const m4 = twgl.m4;
  const gl = document.getElementById("c").getContext("webgl2");
  if (!gl) {
    alert("Sorry, this example requires WebGL 2.0");  // eslint-disable-line
    return;
  }
  
  const texProgramInfo = twgl.createProgramInfo(gl, [vs, fsTex]);
  const mipProgramInfo = twgl.createProgramInfo(gl, [vs, fsMipLevel]);

  const texImage = gl.createTexture();
  gl.bindTexture(gl.TEXTURE_2D, texImage);
  const data = createMips(colors);
  data.forEach(({width, height, data}, level) => {
    gl.texImage2D(gl.TEXTURE_2D, level, gl.RGBA8, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, data);
  });
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST_MIPMAP_LINEAR);

  const lerp = (a, b, t) => a + (b - a) * t;

  function render(time) {
    time *= 0.001;
    twgl.resizeCanvasToDisplaySize(gl.canvas);
    gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
    gl.clearColor(0.3, 0.3, 0.3, 1);
    gl.clear(gl.COLOR_BUFFER_BIT);

    const uniforms = {};

    const s = lerp(1, 128, Math.sin(time) * 0.5 + 0.5);
    uniforms.u_texMat = [
      s, 0, 0,
      0, s, 0,
      0, 0, 1,
    ];
    uniforms.u_worldViewProjection = m4.translation([-1.01, -0.5, 0]);

    gl.useProgram(texProgramInfo.program);
    twgl.setUniforms(texProgramInfo, uniforms);
    gl.drawArrays(gl.TRIANGLES, 0, 6);

    uniforms.u_worldViewProjection = m4.translation([0.01, -0.5, 0]);

    gl.useProgram(mipProgramInfo.program);
    twgl.setUniforms(mipProgramInfo, uniforms);
    gl.drawArrays(gl.TRIANGLES, 0, 6);

    requestAnimationFrame(render);
  }
  requestAnimationFrame(render);
}

main();
</script>