Scene rendering issues in three.js + imgui-js project

I have a project that is using Three.js and imgui-js. I’m trying to update to the latest version of each but I'm running into issues (they were last updated around February of 2020).

To initialize the two libraries I am calling:

  await ImGui.default();
  ImGui.CreateContext();
  ImGui_Impl.Init(canvas);
  ImGui.StyleColorsDark();
  
  const clear_color = new ImGui.ImVec4(0.3, 0.3, 0.3, 1.00);
  const renderer = new THREE.WebGLRenderer({ canvas: canvas });
  renderer.shadowMap.enabled = true;
  renderer.shadowMap.type = THREE.PCFSoftShadowMap;
  renderer.gammaFactor = 2.2;
  renderer.physicallyCorrectLights = true;
  renderer.outputEncoding = THREE.sRGBEncoding;

I then add some objects to the three scene:

  const scene = new THREE.Scene();
  
  let floorDim = 1000;
  var grid = new THREE.GridHelper(floorDim, 200, 'orange', 'white');
  grid.name = "grid";
  grid.material.opacity = 1.0;
  grid.material.transparent = true;
  grid.material.color.convertGammaToLinear(2.2);
  grid.position.set(0, 2, 0);
  scene.add(grid);
  
  var subgrid = new THREE.GridHelper(floorDim, 600, 'grey', 'grey');
  subgrid.name = "subgrid";
  subgrid.material.opacity = 1.0;
  subgrid.material.color.convertGammaToLinear(2.2);
  subgrid.material.transparent = true;
  subgrid.position.set(0, 1, 0);
  scene.add(subgrid);
  
  const light = new THREE.DirectionalLight(0xffffff, 0.8);
    light.position.set(0, 100, -50);
    light.lookAt(new THREE.Vector3(0, 0, 0));
    scene.add(light);

    const box_1_mesh = new THREE.Mesh(new THREE.BoxGeometry(5, 5, 5), new THREE.MeshLambertMaterial({ color:0x970000 }));
  box_1_mesh.position.set(0, 5, 0);
  scene.add(box_1_mesh);

    const box_2_mesh = new THREE.Mesh(new THREE.BoxGeometry(5, 5, 5), new THREE.MeshLambertMaterial({ color:0x333000 }));
  box_2_mesh.position.set(-2, 6, 7);
  scene.add(box_2_mesh);
  
  const camera = new THREE.PerspectiveCamera(50, canvas.width / canvas.height, 2, 10000);
    camera.position.set(0, 10, -25);
  camera.updateProjectionMatrix();
  camera.lookAt(0, 0, 0);
    scene.add(camera);

And call an update loop that looks like:

  function _loop(time) {
    ImGui_Impl.NewFrame(time);
    ImGui.NewFrame();

    ImGui.SetNextWindowPos(new ImGui.ImVec2(20, 20), ImGui.Cond.FirstUseEver);
    ImGui.SetNextWindowSize(new ImGui.ImVec2(294, 140), ImGui.Cond.FirstUseEver);
    ImGui.Begin("Visblility Toggles:");
    ImGui.Checkbox("box_1 (near)", (value = box_1_mesh.visible) => box_1_mesh.visible = value);
    ImGui.Checkbox("box_2 (distant)", (value = box_2_mesh.visible) => box_2_mesh.visible = value);
    ImGui.Checkbox("grid", (value = grid.visible) => grid.visible = value);
    ImGui.Checkbox("subgrid", (value = subgrid.visible) => subgrid.visible = value);
    
    ImGui.End();
    ImGui.EndFrame();
    ImGui.Render();
    
    renderer.setClearColor(new THREE.Color(clear_color.x, clear_color.y, clear_color.z), clear_color.w);
    renderer.setSize(canvas.width, canvas.height);
    camera.aspect = canvas.width / canvas.height;
    camera.updateProjectionMatrix();
    renderer.render(scene, camera);

    ImGui_Impl.RenderDrawData(ImGui.GetDrawData());
    renderer.state.reset();

    window.requestAnimationFrame(_loop);
  }

After updating to the latest versions of these libraries, I am getting issues where objects in the scene aren't rendering as expected.

For example, I am adding 2 GridHelpers to the scene, but for some reason only one gets displayed. Also, if the visibility of the different Three objects is toggled in the scene, for example toggling the display of one of the grids off/on, when toggled back on the object doesn't display again (and sometimes other objects disappear).

Another issue occurring is that more complex meshs' materials will have an unexpected transparency, but with the simple boxes I am adding in this example, that issue doesn’t appear to be occurring. Although, it might be relevant that the look of the boxes is changing based on whether the ImGui_Impl.RenderDrawData call is made.

I have found that if I comment out the call to ImGui_Impl.RenderDrawData(ImGui.GetDrawData()), the scene displays as expected with both grid helpers showing up so I suspect that there might be some internal conflict that is now occurring between the two libraries usages of webGl. Although it's definitely possible that I have set things up incorrectly or I am missing a call that is required.

To help demonstrate the issue, I’ve created a snippet that replicates the issue: https://codepen.io/bpeake/pen/KKNNQre.

(async function() {
  await ImGui.default();
  const canvas = document.getElementById("output");
  const devicePixelRatio = window.devicePixelRatio || 1;
  canvas.width = canvas.scrollWidth * devicePixelRatio;
  canvas.height = canvas.scrollHeight * devicePixelRatio;
  window.addEventListener("resize", () => {
    const devicePixelRatio = window.devicePixelRatio || 1;
    canvas.width = canvas.scrollWidth * devicePixelRatio;
    canvas.height = canvas.scrollHeight * devicePixelRatio;
  });

  ImGui.CreateContext();
  ImGui_Impl.Init(canvas);
  ImGui.StyleColorsDark();
  
  const clear_color = new ImGui.ImVec4(0.3, 0.3, 0.3, 1.00);
  const renderer = new THREE.WebGLRenderer({ canvas: canvas });
  renderer.shadowMap.enabled = true;
  renderer.shadowMap.type = THREE.PCFSoftShadowMap;
  renderer.gammaFactor = 2.2;
  renderer.physicallyCorrectLights = true;
  renderer.outputEncoding = THREE.sRGBEncoding;
  
  const scene = new THREE.Scene();
  
  let floorDim = 1000;
  var grid = new THREE.GridHelper(floorDim, 200, 'orange', 'white');
  grid.name = "grid";
  grid.material.opacity = 1.0;
  grid.material.transparent = true;
  grid.material.color.convertGammaToLinear(2.2);
  grid.position.set(0, 2, 0);
  scene.add(grid);
  
  var subgrid = new THREE.GridHelper(floorDim, 600, 'grey', 'grey');
  subgrid.name = "subgrid";
  subgrid.material.opacity = 1.0;
  subgrid.material.color.convertGammaToLinear(2.2);
  subgrid.material.transparent = true;
  subgrid.position.set(0, 1, 0);
  scene.add(subgrid);
  
  const light = new THREE.DirectionalLight(0xffffff, 0.8);
    light.position.set(0, 100, -50);
    light.lookAt(new THREE.Vector3(0, 0, 0));
    scene.add(light);

    const box_1_mesh = new THREE.Mesh(new THREE.BoxGeometry(5, 5, 5), new THREE.MeshLambertMaterial({ color:0x970000 }));
  box_1_mesh.position.set(0, 5, 0);
  scene.add(box_1_mesh);

    const box_2_mesh = new THREE.Mesh(new THREE.BoxGeometry(5, 5, 5), new THREE.MeshLambertMaterial({ color:0x333000 }));
  box_2_mesh.position.set(-2, 6, 7);
  scene.add(box_2_mesh);
  
  const camera = new THREE.PerspectiveCamera(50, canvas.width / canvas.height, 2, 10000);
    camera.position.set(0, 10, -25);
  camera.updateProjectionMatrix();
  camera.lookAt(0, 0, 0);
    scene.add(camera);
  
  window.requestAnimationFrame(_loop);
  function _loop(time) {
    ImGui_Impl.NewFrame(time);
    ImGui.NewFrame();

    ImGui.SetNextWindowPos(new ImGui.ImVec2(20, 20), ImGui.Cond.FirstUseEver);
    ImGui.SetNextWindowSize(new ImGui.ImVec2(294, 140), ImGui.Cond.FirstUseEver);
    ImGui.Begin("Visblility Toggles:");
    ImGui.Checkbox("box_1 (near)", (value = box_1_mesh.visible) => box_1_mesh.visible = value);
    ImGui.Checkbox("box_2 (distant)", (value = box_2_mesh.visible) => box_2_mesh.visible = value);
    ImGui.Checkbox("grid", (value = grid.visible) => grid.visible = value);
    ImGui.Checkbox("subgrid", (value = subgrid.visible) => subgrid.visible = value);
    
    ImGui.End();
    ImGui.EndFrame();
    ImGui.Render();
    
    renderer.setClearColor(new THREE.Color(clear_color.x, clear_color.y, clear_color.z), clear_color.w);
    renderer.setSize(canvas.width, canvas.height);
    camera.aspect = canvas.width / canvas.height;
    camera.updateProjectionMatrix();
    renderer.render(scene, camera);

    ImGui_Impl.RenderDrawData(ImGui.GetDrawData());
    renderer.state.reset();

    window.requestAnimationFrame(_loop);
  }

})();

#output {
  position: absolute;
  top: 0px;
  right: 0px;
  width: 100%;
  height: 100%;
  z-index: 1;
}

<script>
// emu localStorage otherwise ImGUI fails because S.O blocks localStorage
{
  const localStorageStorage = new Map();
  const localStorage = {
    getItem(k) { return localStorageStorage.get(k); },
    setItem(k,v) { return localStorageStorage.set(k, v); },
  };
  Object.defineProperty(window, 'localStorage', {
    get() { return localStorage; }
  });
}
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r125/three.min.js"></script>
<script src="https://flyover.github.io/imgui-js/dist/imgui.umd.js"></script>
<script src="https://flyover.github.io/imgui-js/dist/imgui_impl.umd.js"></script>
<script src="https://flyover.github.io/nanovg-js/dist/nanovg.umd.js"></script>
<canvas tabindex="0" id="output"></canvas>

Is there anything wrong with my setup of imgui-js or three-js? Any ideas as to what I might do so that the ImGui_Impl.RenderDrawData call does not impact the rendering of my three.js scene?

Solution

The issue is imgui-js is trashing the attribute state.

You might want to consider running imgui-js in another canvas overlayed on top of the three.js canvas, each with their own WebGL context. Then they don't have to worry about each other.

A quick hack is this

    const gl = renderer.getContext();
    window.vao = window.vao || gl.createVertexArray();
    const oldVao = gl.getParameter(gl.VERTEX_ARRAY_BINDING);
    gl.bindVertexArray(vao);
    
    ImGui_Impl.RenderDrawData(ImGui.GetDrawData());
    
    gl.bindVertexArray(oldVao);

But that hack only works in WebGL2. To fully save and restore the missing state is not hard but handling WebGL1, and extensions, and WebGL2 is probably 50-80 lines of code.

I filed an issue here with an example PR

(async function() {
  await ImGui.default();
  const canvas = document.getElementById("output");
  const devicePixelRatio = window.devicePixelRatio || 1;
  canvas.width = canvas.scrollWidth * devicePixelRatio;
  canvas.height = canvas.scrollHeight * devicePixelRatio;
  window.addEventListener("resize", () => {
    const devicePixelRatio = window.devicePixelRatio || 1;
    canvas.width = canvas.scrollWidth * devicePixelRatio;
    canvas.height = canvas.scrollHeight * devicePixelRatio;
  });

  ImGui.CreateContext();
  ImGui_Impl.Init(canvas);
  ImGui.StyleColorsDark();

  const clear_color = new ImGui.ImVec4(0.3, 0.3, 0.3, 1.00);
  const renderer = new THREE.WebGLRenderer({ canvas: canvas });
  renderer.shadowMap.enabled = true;
  renderer.shadowMap.type = THREE.PCFSoftShadowMap;
  renderer.gammaFactor = 2.2;
  renderer.physicallyCorrectLights = true;
  renderer.outputEncoding = THREE.sRGBEncoding;
  
  const scene = new THREE.Scene();
  
  let floorDim = 1000;
  var grid = new THREE.GridHelper(floorDim, 200, 'orange', 'white');
  grid.name = "grid";
  grid.material.opacity = 1.0;
  grid.material.transparent = true;
  grid.material.color.convertGammaToLinear(2.2);
  grid.position.set(0, 2, 0);
  scene.add(grid);
  
  var subgrid = new THREE.GridHelper(floorDim, 600, 'grey', 'grey');
  subgrid.name = "subgrid";
  subgrid.material.opacity = 1.0;
  subgrid.material.color.convertGammaToLinear(2.2);
  subgrid.material.transparent = true;
  subgrid.position.set(0, 1, 0);
  scene.add(subgrid);
  
  const light = new THREE.DirectionalLight(0xffffff, 0.8);
    light.position.set(0, 100, -50);
    light.lookAt(new THREE.Vector3(0, 0, 0));
    scene.add(light);

    const box_1_mesh = new THREE.Mesh(new THREE.BoxGeometry(5, 5, 5), new THREE.MeshLambertMaterial({ color:0x970000 }));
  box_1_mesh.position.set(0, 5, 0);
  scene.add(box_1_mesh);

    const box_2_mesh = new THREE.Mesh(new THREE.BoxGeometry(5, 5, 5), new THREE.MeshLambertMaterial({ color:0x333000 }));
  box_2_mesh.position.set(-2, 6, 7);
  scene.add(box_2_mesh);
  
  const camera = new THREE.PerspectiveCamera(50, canvas.width / canvas.height, 2, 10000);
    camera.position.set(0, 10, -25);
  camera.updateProjectionMatrix();
  camera.lookAt(0, 0, 0);
    scene.add(camera);

  setTimeout(() => {
    grid.visible = false;
  }, 1000);
  setTimeout(() => {
    grid.visible = true;
  }, 2000);
  setTimeout(() => {
    console.log(grid.visible);
  }, 2500);

  window.requestAnimationFrame(_loop);
  function _loop(time) {
    ImGui_Impl.NewFrame(time);
    ImGui.NewFrame();

    ImGui.SetNextWindowPos(new ImGui.ImVec2(20, 20), ImGui.Cond.FirstUseEver);
    ImGui.SetNextWindowSize(new ImGui.ImVec2(294, 140), ImGui.Cond.FirstUseEver);
    ImGui.Begin("Visblility Toggles:");
    ImGui.Checkbox("box_1 (near)", (value = box_1_mesh.visible) => box_1_mesh.visible = value);
    ImGui.Checkbox("box_2 (distant)", (value = box_2_mesh.visible) => box_2_mesh.visible = value);
    ImGui.Checkbox("grid", (value = grid.visible) => grid.visible = value);
    ImGui.Checkbox("subgrid", (value = subgrid.visible) => subgrid.visible = value);
    
    
    ImGui.End();
    ImGui.EndFrame();
    ImGui.Render();

    renderer.setClearColor(new THREE.Color(clear_color.x, clear_color.y, clear_color.z), clear_color.w);
    renderer.setSize(canvas.width, canvas.height);
    camera.aspect = canvas.width / canvas.height;
    camera.updateProjectionMatrix();
    renderer.render(scene, camera);

    const gl = renderer.getContext();
    window.vao = window.vao || gl.createVertexArray();
    const oldVao = gl.getParameter(gl.VERTEX_ARRAY_BINDING);
    gl.bindVertexArray(vao);
    
    ImGui_Impl.RenderDrawData(ImGui.GetDrawData());
    
    gl.bindVertexArray(oldVao);

    renderer.state.reset();

    window.requestAnimationFrame(_loop);
  }

})();

#output {
  position: absolute;
  top: 0px;
  right: 0px;
  width: 100%;
  height: 100%;
  z-index: 1;
}

<script>
// emu localStorage otherwise ImGUI fails because S.O blocks localStorage
{
  const localStorageStorage = new Map();
  const localStorage = {
    getItem(k) { return localStorageStorage.get(k); },
    setItem(k,v) { return localStorageStorage.set(k, v); },
  };
  Object.defineProperty(window, 'localStorage', {
    get() { return localStorage; }
  });
}
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r125/three.js"></script>
<script src="https://flyover.github.io/imgui-js/dist/imgui.umd.js"></script>
<script src="https://flyover.github.io/imgui-js/dist/imgui_impl.umd.js"></script>
<script src="https://flyover.github.io/nanovg-js/dist/nanovg.umd.js"></script>
<canvas tabindex="0" id="output"></canvas>