Creating quaternion from direction and up vectors for rotation matrix

Question

How to calculate rotation matrix based on cone direction? I have a spotlight defined like this:

I'm creating vertices array for a cone to draw spot light shape and then I'm creating a buffer (line loop mode):

private static float[] createConeVerticesArray(float radius,
                                               float height,
                                               int segments)
{
    Array<Float> verticesArray = new Array<>();

    float angle = 2 * MathUtils.PI / segments;
    float cos = MathUtils.cos(angle);
    float sin = MathUtils.sin(angle);
    float cx = radius, cy = 0;

    for(int i = 0; i < segments; i++)
    {
        verticesArray.add(cx);
        verticesArray.add(cy);
        verticesArray.add(height);

        verticesArray.add(0f);
        verticesArray.add(0f);
        verticesArray.add(0f);

        verticesArray.add(cx);
        verticesArray.add(cy);
        verticesArray.add(height);

        float temp = cx;
        cx = cos * cx - sin * cy;
        cy = sin * temp + cos * cy;

        verticesArray.add(cx);
        verticesArray.add(cy);
        verticesArray.add(height);
    }
    verticesArray.add(cx);
    verticesArray.add(cy);
    verticesArray.add(height);

    cx = radius;
    cy = 0;
    verticesArray.add(cx);
    verticesArray.add(cy);
    verticesArray.add(height);

    float[] result = new float[verticesArray.size];
    for(int i = 0; i < verticesArray.size; i++)
    {
        result[i] = verticesArray.get(i);
    }

    return result;
}

Next step I'm creating three spot lights like this which represents 3 rotations around each axis:

this.spotLights.add(new CSpotLight(
            new Color(1f, 0f, 0f, 1f), //Color
            new Vector3(0f, 0f, 5000f), //Position
            1f, //Intensity
            new Vector3(0f, 0f, -1f), //Direction
            15f, //Inner angle
            30f, //Outer angle
            4000f, //Radius
            1f)); //Attenuation
    this.spotLights.add(new CSpotLight(
            new Color(1f, 0f, 0f, 1f),
            new Vector3(0f, 5000f, 0f),
            1f,
            new Vector3(0f, -1f, 0f),
            15f,
            30f,
            4000f,
            1f));
    this.spotLights.add(new CSpotLight(
            new Color(1f, 0f, 0f, 1f),
            new Vector3(5000f, 0f, 0f),
            1f,
            new Vector3(-1f, 0f, 0f),
            15f,
            30f,
            4000f,
            1f));

And then I'm calculating quaternion for each spot light to create rotation matrix for model matrix (just for vertex shader purpose):

private static Quaternion quaternionLookRotation(Vector3 direction, Vector3 up)
{
    Vector3 vector = Pools.obtain(Vector3.class).set(direction).nor();
    Vector3 vector2 = Pools.obtain(Vector3.class).set(up).crs(vector).nor();
    Vector3 vector3 = Pools.obtain(Vector3.class).set(vector).crs(vector2);

    float m00 = vector2.x;
    float m01 = vector2.y;
    float m02 = vector2.z;
    float m10 = vector3.x;
    float m11 = vector3.y;
    float m12 = vector3.z;
    float m20 = vector.x;
    float m21 = vector.y;
    float m22 = vector.z;

    Pools.free(vector);
    Pools.free(vector2);
    Pools.free(vector3);

    float num8 = (m00 + m11) + m22;
    Quaternion quaternion = Pools.obtain(Quaternion.class);
    if (num8 > 0f)
    {
        float num = (float) Math.sqrt(num8 + 1f);
        quaternion.w = num * 0.5f;
        num = 0.5f / num;
        quaternion.x = (m12 - m21) * num;
        quaternion.y = (m20 - m02) * num;
        quaternion.z = (m01 - m10) * num;
        return quaternion;
    }
    if ((m00 >= m11) && (m00 >= m22))
    {
        float num7 = (float) Math.sqrt(((1f + m00) - m11) - m22);
        float num4 = 0.5f / num7;
        quaternion.x = 0.5f * num7;
        quaternion.y = (m01 + m10) * num4;
        quaternion.z = (m02 + m20) * num4;
        quaternion.w = (m12 - m21) * num4;
        return quaternion;
    }
    if (m11 > m22)
    {
        float num6 = (float) Math.sqrt(((1f + m11) - m00) - m22);
        float num3 = 0.5f / num6;
        quaternion.x = (m10+ m01) * num3;
        quaternion.y = 0.5f * num6;
        quaternion.z = (m21 + m12) * num3;
        quaternion.w = (m20 - m02) * num3;
        return quaternion;
    }
    float num5 = (float) Math.sqrt(((1f + m22) - m00) - m11);
    float num2 = 0.5f / num5;
    quaternion.x = (m20 + m02) * num2;
    quaternion.y = (m21 + m12) * num2;
    quaternion.z = 0.5f * num5;
    quaternion.w = (m01 - m10) * num2;
    return quaternion;
}

this.rotMatrix.set(quaternionLookRotation(
                spotLight.getDirection(),
                new Vector3(0, 1, 0)));

Two rotations from three works perfectly. The problem appears only in the third rotation axis which should faced point (0, 0, 0):

Answer 1

Here is my final code which works fine in any cases:

private static Quaternion rotationBetweenTwoVectors(Vector3 vec1, Vector3 vec2)
{
    Quaternion result = Pools.obtain(Quaternion.class);
    Vector3 u = Pools.obtain(Vector3.class).set(vec1).nor();
    Vector3 v = Pools.obtain(Vector3.class).set(vec2).nor();
    float dot = u.dot(v);

    if(dot < -0.999999)
    {
        Vector3 tmp1 = Pools.obtain(Vector3.class).set(Vector3.X).crs(u);
        if(tmp1.len() < 0.000001)
        {
            tmp1.set(Vector3.Y).crs(u);
        }
        tmp1.nor();
        result.setFromAxisRad(tmp1, MathUtils.PI).nor();
        Pools.free(tmp1);
    }
    else if(dot > 0.999999)
    {
        result.idt();
    }
    else
    {
        result.setFromCross(u, v);
    }
    Pools.free(u);
    Pools.free(v);

    return result;
}

Where setFromCross calculates axis from cross product of two vectors and angle like this:

final float dot = MathUtils.clamp(Vector3.dot(x1, y1, z1, x2, y2, z2), -1f, 1f);
final float angle = (float)Math.acos(dot);

Hope this will helps somebody :)