Three JS rotate group with same orientation as camera for View Cube/Orientation Cube

Question

I'm trying to make a blender-esk orientation view that looks like this using Three JS:

My plan is to take the camera rotation and rotate 3 vector points (representing the X,Y and Z bubbles in the picture). Then once I have those points I can use the X and Y coordinates to draw circles on a canvas and draw lines to connect them to the center. I can then use the Z direction to determine what should be drawn on top of what.

Currently I'm using a 2nd three.js scene to visualize the position of these X, Y and Z points as I try different ways to rotate the group of vertices to have it match the camera's orientation. I'm struggling with this part and I've tried a bunch of things that haven't worked.

Here is a JS Fiddle for where I'm currently at:

https://jsfiddle.net/j9mcL0x4/4/ (doesn't load in Brave but works in Chrome)

As you pan the camera around, it moves the 3 cubes in the 2nd scene but I cannot get it to rotate correctly.

The X, Y and Z points are represented by a group of Vector3's as such:

this.ref = [
  [new THREE.Vector3(1,0,0), 0xFF0000],
  [new THREE.Vector3(0,1,0), 0x00FF00],
  [new THREE.Vector3(0,0,1), 0x0000FF],
]

I then create each cube and add it to a group:

this.group = new THREE.Group();
for(var pos of this.ref) {
  var geometry = new THREE.BoxGeometry(.25,.25,.25);
  var material = new THREE.MeshBasicMaterial({
    color: pos[1]
  });
  var cube = new THREE.Mesh(geometry, material);
  cube.position.copy(pos[0]);
  this.group.add(cube);
}

this.scene.add(this.group);

Then in my animate function, I'm trying to calculate the rotation for the group to get it to be in the same orientation as the main view:

var quaternion = new THREE.Quaternion();
camera.getWorldQuaternion( quaternion );

let rotation = new THREE.Euler();
rotation.setFromQuaternion(quaternion);

var dir = new THREE.Vector3();
dir.subVectors( camera.position, controls.target ).normalize();

orientationHelper.group.rotation.set(dir.x, dir.y, dir.z);
orientationHelper.animate();

What I have is completely wrong but here are some things I've tried:

• Calculate a point in front of the camera, then use that to make the group look in that direction using the lookAt function
• I'm using orbit controls which has a target that the camera is focused on. I've tried subtracting the camera position vector and the target vector to get the look direction but this failed to work (what's currently in the jsfiddle)
• I've tried using just the camera local rotation (and tried negating it)

Note I could rotate the camera in the 2nd scene to match the camera in the main view but I really want to rotate the vectors themselves. This is so I can take those points and project them onto a basic canvas to draw 6 circles and 3 lines. I feel like this will be much easier than trying to use sprites in 3d and Three JS cannot easily draw thick lines without creating rectangular meshes for it.

Here are some examples of what I want the output to look like:

Update

This problem was solved by Rabbid76 and if anyone is interested in using a Blender Style orientation cube for their project, you can find the full source code here:

https://github.com/jrj2211/three-orientation-gizmo/

Or download it with npm:

https://www.npmjs.com/package/three-orientation-gizmo

Answer 1

First of all in view space, the X-axis points to the right, the Y-axis points is up and the Z-axis points out of the canvas.
So you have to choose an initial position for the OrientationHelper camera on the positive Z-axis:

class OrientationHelper {
    constructor(canvas) {
        // [...]

        this.camera = new THREE.PerspectiveCamera(75, size / size, 0.1, 1000);
        this.camera.position.set(0, 0, 2);

        // [...]

In the following I recommend to set the orientation of the group in the OrientationHelper scene by a matrix:

class OrientationHelper {
    // [...]

    setCameraRotation(rotation) {
        this.group.setRotationFromMatrix(rotation);
    }

    // [...]

The view matrix is the inverse matrix, of that matrix, that defines the position and orientation of the camera. You want to visualize the orientation of the camera. You want to visualize the orientation of the view matrix.
Create a rotation matrix from the camera.rotation. Compute the Inverse matrix and set the orientation of the OrientationHelper by the computed matrix:

let rotMat = new THREE.Matrix4().makeRotationFromEuler(camera.rotation);
let invRotMat = new THREE.Matrix4().getInverse(rotMat); 

orientationHelper.setCameraRotation(invRotMat);
orientationHelper.update();

See the example:

class OrientationHelper {
  constructor(canvas) {
    this.canvas = canvas;
    
    this.ref = [
      [new THREE.Vector3(1,0,0), 0xFF0000],
      [new THREE.Vector3(0,1,0), 0x00FF00],
      [new THREE.Vector3(0,0,1), 0x0000FF],
    ]

    var size = 200;
    this.scene = new THREE.Scene();
    this.camera = new THREE.PerspectiveCamera(75, size / size, 0.1, 1000);
    this.camera.position.set(0, 0, 2);

    this.renderer = new THREE.WebGLRenderer({alpha: true});
    this.renderer.setSize(size, size);
    canvas.appendChild(this.renderer.domElement);
    
    var controls = new THREE.OrbitControls(this.camera, this.renderer.domElement);
    
    var geometry = new THREE.SphereGeometry( .1, 32, 32 );
    var material = new THREE.MeshBasicMaterial( {color: 0xffffff} );
    var sphere = new THREE.Mesh( geometry, material );
    this.scene.add( sphere );

    this.group = new THREE.Group();
    for(var pos of this.ref) {
      var geometry = new THREE.BoxGeometry(.25,.25,.25);
      var material = new THREE.MeshBasicMaterial({
        color: pos[1]
      });
      var cube = new THREE.Mesh(geometry, material);
      cube.position.copy(pos[0]);
      this.group.add(cube);
    }

    this.scene.add(this.group);
  }

  setCameraRotation(rotation) {
    this.group.setRotationFromMatrix(rotation);
  }
  
  update() {
  	this.renderer.render(this.scene, this.camera);
  }
}

var scene = new THREE.Scene();
var camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);

var renderer = new THREE.WebGLRenderer();
renderer.setSize(window.innerWidth, window.innerHeight);
document.body.appendChild(renderer.domElement);

var controls = new THREE.OrbitControls(camera, renderer.domElement);
camera.position.set(0, 3, 3);
controls.update();

//controls.autoRotate = true;

var size = 10;
var divisions = 10;

var gridHelper = new THREE.GridHelper(size, divisions);
scene.add(gridHelper);

var axesHelper = new THREE.AxesHelper(5);
scene.add(axesHelper);

 // material
  var material = new THREE.MeshBasicMaterial({
    color: 0xffffff,
    vertexColors: THREE.FaceColors
  });

var geometry = new THREE.BoxGeometry();

 // colors
  red = new THREE.Color(1, 0, 0);
  green = new THREE.Color(0, 1, 0);
  blue = new THREE.Color(0, 0, 1);
  var colors = [red, green, blue];
  
  for (var i = 0; i < 3; i++) {
    geometry.faces[4 * i].color = colors[i];
    geometry.faces[4 * i + 1].color = colors[i];
    geometry.faces[4 * i + 2].color = colors[i];
    geometry.faces[4 * i + 3].color = colors[i];
  }
  
var cube = new THREE.Mesh(geometry, material);
scene.add(cube);

// Orientation
var orientationHelper = new OrientationHelper(document.getElementById("orientation-helper"));

function animate() {
  requestAnimationFrame(animate);
  controls.update();

  let rotMat = new THREE.Matrix4().makeRotationFromEuler(camera.rotation);
  let invRotMat = new THREE.Matrix4().getInverse(rotMat); 
  orientationHelper.setCameraRotation(invRotMat);
	orientationHelper.update();
  
  renderer.render(scene, camera);
}

animate();

body { margin: 0; }
canvas { display: block; }
#orientation-helper { position: absolute; top: 10px; left: 10px; background: #505050; }

<script src="https://rawcdn.githack.com/mrdoob/three.js/r113/build/three.js"></script>
<script src="https://rawcdn.githack.com/mrdoob/three.js/r113/examples/js/controls/OrbitControls.js"></script>
<div id='orientation-helper'></div>