Proper way to handle camera rotations

Question

Let's start by considering 2 type of camera rotations:

Camera rotating around a point (Orbit):

def rotate_around_target(self, target, delta):
    right = (self.target - self.eye).cross(self.up).normalize()
    amount = (right * delta.y + self.up * delta.x)
    self.target = target
    self.up = self.original_up
    self.eye = (
        mat4.rotatez(amount.z) *
        mat4.rotatey(amount.y) *
        mat4.rotatex(amount.x) *
        vec3(self.eye)
    )

Camera rotating the target (FPS)

def rotate_target(self, delta):
    right = (self.target - self.eye).cross(self.up).normalize()
    self.target = (
        mat4.translate(self.eye) *
        mat4().rotate(delta.y, right) *
        mat4().rotate(delta.x, self.up) *
        mat4.translate(-self.eye) *
        self.target
    )

And then just an update function where the projection/view matrices are calculated out of the eye/target/up camera vectors:

def update(self, aspect):
    self.view = mat4.lookat(self.eye, self.target, self.up)
    self.projection = mat4.perspective_fovx(
        self.fov, aspect, self.near, self.far
    )

Problem with these rotation functions appears when the camera view direction becomes parallel to the up axis (z-up over here)... at that point the camera behaves in a really nasty way so I'll have glitches such as:

So my question is, how can I adjust the above code so the camera will make full rotations without the end result looking weird at certain edge points (camera axis flipping around :/)?

I'd like to have the same behaviour than many DCC packages out there (3dsmax, maya, ...) where they make full rotations without presenting any strange behaviour.

EDIT:

For those who want to give it a shot to the maths I've decided to create a really minimalistic version that's able to reproduce the explained problems:

import math
from ctypes import c_void_p

import numpy as np
from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *

import glm


class Camera():

    def __init__(
        self,
        eye=None, target=None, up=None,
        fov=None, near=0.1, far=100000
    ):
        self.eye = eye or glm.vec3(0, 0, 1)
        self.target = target or glm.vec3(0, 0, 0)
        self.up = up or glm.vec3(0, 1, 0)
        self.original_up = glm.vec3(self.up)
        self.fov = fov or glm.radians(45)
        self.near = near
        self.far = far

    def update(self, aspect):
        self.view = glm.lookAt(
            self.eye, self.target, self.up
        )
        self.projection = glm.perspective(
            self.fov, aspect, self.near, self.far
        )

    def rotate_target(self, delta):
        right = glm.normalize(glm.cross(self.target - self.eye, self.up))
        M = glm.mat4(1)
        M = glm.translate(M, self.eye)
        M = glm.rotate(M, delta.y, right)
        M = glm.rotate(M, delta.x, self.up)
        M = glm.translate(M, -self.eye)
        self.target = glm.vec3(M * glm.vec4(self.target, 1.0))

    def rotate_around_target(self, target, delta):
        right = glm.normalize(glm.cross(self.target - self.eye, self.up))
        amount = (right * delta.y + self.up * delta.x)
        M = glm.mat4(1)
        M = glm.rotate(M, amount.z, glm.vec3(0, 0, 1))
        M = glm.rotate(M, amount.y, glm.vec3(0, 1, 0))
        M = glm.rotate(M, amount.x, glm.vec3(1, 0, 0))
        self.eye = glm.vec3(M * glm.vec4(self.eye, 1.0))
        self.target = target
        self.up = self.original_up

    def rotate_around_origin(self, delta):
        return self.rotate_around_target(glm.vec3(0), delta)


class GlutController():

    FPS = 0
    ORBIT = 1

    def __init__(self, camera, velocity=100, velocity_wheel=100):
        self.velocity = velocity
        self.velocity_wheel = velocity_wheel
        self.camera = camera

    def glut_mouse(self, button, state, x, y):
        self.mouse_last_pos = glm.vec2(x, y)
        self.mouse_down_pos = glm.vec2(x, y)

        if button == GLUT_LEFT_BUTTON:
            self.mode = self.FPS
        elif button == GLUT_RIGHT_BUTTON:
            self.mode = self.ORBIT

    def glut_motion(self, x, y):
        pos = glm.vec2(x, y)
        move = self.mouse_last_pos - pos
        self.mouse_last_pos = pos

        if self.mode == self.FPS:
            self.camera.rotate_target(move * 0.005)
        elif self.mode == self.ORBIT:
            self.camera.rotate_around_origin(move * 0.005)


class MyWindow:

    def __init__(self, w, h):
        self.width = w
        self.height = h

        glutInit()
        glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGB | GLUT_DEPTH)
        glutInitWindowSize(w, h)
        glutCreateWindow('OpenGL Window')

        self.startup()

        glutReshapeFunc(self.reshape)
        glutDisplayFunc(self.display)
        glutMouseFunc(self.controller.glut_mouse)
        glutMotionFunc(self.controller.glut_motion)
        glutIdleFunc(self.idle_func)

    def startup(self):
        glEnable(GL_DEPTH_TEST)

        aspect = self.width / self.height
        self.camera = Camera(
            eye=glm.vec3(10, 10, 10),
            target=glm.vec3(0, 0, 0),
            up=glm.vec3(0, 1, 0)
        )
        self.model = glm.mat4(1)
        self.controller = GlutController(self.camera)

    def run(self):
        glutMainLoop()

    def idle_func(self):
        glutPostRedisplay()

    def reshape(self, w, h):
        glViewport(0, 0, w, h)
        self.width = w
        self.height = h

    def display(self):
        self.camera.update(self.width / self.height)

        glClearColor(0.2, 0.3, 0.3, 1.0)
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        gluPerspective(glm.degrees(self.camera.fov), self.width / self.height, self.camera.near, self.camera.far)
        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()
        e = self.camera.eye
        t = self.camera.target
        u = self.camera.up
        gluLookAt(e.x, e.y, e.z, t.x, t.y, t.z, u.x, u.y, u.z)
        glColor3f(1, 1, 1)
        glBegin(GL_LINES)
        for i in range(-5, 6):
            if i == 0:
                continue
            glVertex3f(-5, 0, i)
            glVertex3f(5, 0, i)
            glVertex3f(i, 0, -5)
            glVertex3f(i, 0, 5)
        glEnd()

        glBegin(GL_LINES)
        glColor3f(1, 0, 0)
        glVertex3f(-5, 0, 0)
        glVertex3f(5, 0, 0)
        glColor3f(0, 1, 0)
        glVertex3f(0, -5, 0)
        glVertex3f(0, 5, 0)
        glColor3f(0, 0, 1)
        glVertex3f(0, 0, -5)
        glVertex3f(0, 0, 5)
        glEnd()

        glutSwapBuffers()


if __name__ == '__main__':
    window = MyWindow(800, 600)
    window.run()

In order to run it you'll need to install pyopengl and pyglm

Answer 1

I recommend to do a rotation around a pivot in view space

You have to know the view matrix (V). Since the view matrix is encoded in self.eye, self.target and self.up, it has to be computed by lookAt:

V = glm.lookAt(self.eye, self.target, self.up)

Compute the pivot in view space, the rotation angle and the rotation axis. The axis is in this case the right rotated direction, where the y axis has to be flipped:

pivot = glm.vec3(V * glm.vec4(target.x, target.y, target.z, 1))
axis  = glm.vec3(-delta.y, -delta.x, 0)
angle = glm.length(delta)

Set up the rotation matrix R and calculate the ration matrix around the pivot RP. Finally transform the view matrix (V) by the rotation matrix. The result is the new view matrix NV:

R  = glm.rotate( glm.mat4(1), angle, axis )
RP = glm.translate(glm.mat4(1), pivot) * R * glm.translate(glm.mat4(1), -pivot)
NV = RP * V

Decode the self.eye, self.target and self.up from the new view matrix NV:

C = glm.inverse(NV)
targetDist  = glm.length(self.target - self.eye)
self.eye    = glm.vec3(C[3])
self.target = self.eye - glm.vec3(C[2]) * targetDist 
self.up     = glm.vec3(C[1])

Full coding of the method rotate_around_target_view:

def rotate_around_target_view(self, target, delta):

    V = glm.lookAt(self.eye, self.target, self.up)

    pivot = glm.vec3(V * glm.vec4(target.x, target.y, target.z, 1))
    axis  = glm.vec3(-delta.y, -delta.x, 0)
    angle = glm.length(delta)

    R  = glm.rotate( glm.mat4(1), angle, axis )
    RP = glm.translate(glm.mat4(1), pivot) * R * glm.translate(glm.mat4(1), -pivot)
    NV = RP * V

    C = glm.inverse(NV)
    targetDist  = glm.length(self.target - self.eye)
    self.eye    = glm.vec3(C[3])
    self.target = self.eye - glm.vec3(C[2]) * targetDist 
    self.up     = glm.vec3(C[1])

Finally it can be rotated around the origin of the world and the the eye position or even any other point.

def rotate_around_origin(self, delta):
    return self.rotate_around_target_view(glm.vec3(0), delta)

def rotate_target(self, delta):
    return self.rotate_around_target_view(self.eye, delta)

---

Alternatively the rotation can be performed in world space on the model. The solution is very similar. The rotation is done in world space, so the pivot hasn't to be transforms to view space and The rotation is applied before the view matrix (NV = V * RP):

def rotate_around_target_world(self, target, delta):

    V = glm.lookAt(self.eye, self.target, self.up)

    pivot = target
    axis  = glm.vec3(-delta.y, -delta.x, 0)
    angle = glm.length(delta)

    R  = glm.rotate( glm.mat4(1), angle, axis )
    RP = glm.translate(glm.mat4(1), pivot) * R * glm.translate(glm.mat4(1), -pivot)
    NV = V * RP

    C = glm.inverse(NV)
    targetDist  = glm.length(self.target - self.eye)
    self.eye    = glm.vec3(C[3])
    self.target = self.eye - glm.vec3(C[2]) * targetDist 
    self.up     = glm.vec3(C[1]) 

def rotate_around_origin(self, delta):
    return self.rotate_around_target_world(glm.vec3(0), delta)

---

Of course both solutions can be combined. By dragging vertical (up and down), the view can be rotated on its horizontal axis. And by dragging horizontal (left and right) the model (world) can be rotated around its (up) axis:

def rotate_around_target(self, target, delta):
    if abs(delta.x) > 0:
        self.rotate_around_target_world(target, glm.vec3(delta.x, 0.0, 0.0))
    if abs(delta.y) > 0:    
        self.rotate_around_target_view(target, glm.vec3(0.0, delta.y, 0.0))

---

I order to achieve a minimal invasive approach, considering the original code of the question, I'll make the following suggestion:

• After the manipulation the target of the view should be the input parameter targetof the function rotate_around_target.

• A horizontal mouse movement should rotate the view around the up vector of the world

• a vertical mouse movement should tilt the view around current horizontal axis

I came up to the following approach:

1. Calculate the current line of sight (los), up vector (up) and horizontla axis (right)

2. Upright the up vector, by projecting the up vector to a plane which is given by the original up vector and the current line of sight. This is don by Gram–Schmidt orthogonalization.

3. Tilt around the current horizontal axis. This means los and up is rotated around the right axis.

4. Rotate around the up vector. los and right is rotated around up.

5. Calculate set the up and calculate the eye and target position, where the target is set by the input parameter target:

def rotate_around_target(self, target, delta):

    # get directions
    los    = self.target - self.eye
    losLen = glm.length(los)
    right  = glm.normalize(glm.cross(los, self.up))
    up     = glm.cross(right, los)

    # upright up vector (Gram–Schmidt orthogonalization)
    fix_right = glm.normalize(glm.cross(los, self.original_up))
    UPdotX    = glm.dot(fix_right, up)
    up        = glm.normalize(up - UPdotX * fix_right)
    right     = glm.normalize(glm.cross(los, up))
    los       = glm.cross(up, right)

    # tilt around horizontal axis
    RHor = glm.rotate(glm.mat4(1), delta.y, right)
    up   = glm.vec3(RHor * glm.vec4(up, 0.0))
    los  = glm.vec3(RHor * glm.vec4(los, 0.0))

    # rotate around up vector
    RUp   = glm.rotate(glm.mat4(1), delta.x, up)
    right = glm.vec3(RUp * glm.vec4(right, 0.0))
    los   = glm.vec3(RUp * glm.vec4(los, 0.0))

    # set eye, target and up
    self.eye    = target - los * losLen 
    self.target = target
    self.up     = up