Search code examples
iosopengl-esglsltexturesopengl-es-2.0

Display YUV(yuv420p) is not correct on IOS


Using (EAGLRenderingAPI api = kEAGLRenderingAPIOpenGLES2) to displaying YUV on the screen on iOS. But I got this problem, and how to deal with this problem thanks.

Display YUV (size-> 3268:1838) isn't correct, showing as:

enter image description here

But using the same code TO Display YUV(size-> 1280:720) is correct, showing as:

enter image description here

WHAT IS WRONG? ANY HELP thanks.

CODES:

EAGLRenderingAPI api = kEAGLRenderingAPIOpenGLES2;
_context = [[EAGLContext alloc] initWithAPI:api];
if (!_context)
{
    if([LTJNYJCommon isDebug]){NSLog(@"Failed to initialize OpenGLES 2.0 context");}
    exit(1);
}

if (![EAGLContext setCurrentContext:_context])
{
    if([LTJNYJCommon isDebug]){NSLog(@"Failed to set current OpenGLES 2.0 context");}
    exit(1);
}


    int idxU = width * height;
    int idxV = idxU + (idxU / 4);
    uint8_t *pyuvData = (uint8_t *)[data bytes];

    glClearColor(0.0, 0.0, 0.0, 1.0);
    glClear(GL_COLOR_BUFFER_BIT);
    glViewport(x, y, w, h);
    glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
    glVertexAttribPointer(_positionSlot, 3, GL_FLOAT, GL_FALSE, sizeof(LTGLVertex), 0);
    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, _indexBuffer);
    glVertexAttribPointer(_texCoordSlot, 2, GL_FLOAT, GL_FALSE, sizeof(LTGLVertex), (GLvoid *)(offsetof(LTGLVertex, TexCoord)));


        //----------------------------------------------------
        //Y texture
        //----------------------------------------------------
        if (_samplerYTexture){glDeleteTextures(1, &_samplerYTexture);}
        glActiveTexture(GL_TEXTURE0);
        glGenTextures(1, &_samplerYTexture);
        glBindTexture(GL_TEXTURE_2D, _samplerYTexture);
        glUniform1i(_samplerYUniform, 0);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, pyuvData);
        //----------------------------------------------------
        //U texture
        //----------------------------------------------------
        if (_samplerUTexture){glDeleteTextures(1, &_samplerUTexture);}
        glActiveTexture(GL_TEXTURE1);
        glGenTextures(1, &_samplerUTexture);
        glBindTexture(GL_TEXTURE_2D, _samplerUTexture);
        glUniform1i(_samplerUUniform, 1);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        if (datalength>idxU && pyuvData[idxU]) {
            glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width/2, height/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, &pyuvData[idxU]);
        }else{
            if([LTJNYJCommon isDebug]){NSLog(@"error1");}
        }
        //----------------------------------------------------
        //V texture
        //----------------------------------------------------
        if (_samplerVTexture){glDeleteTextures(1, &_samplerVTexture);}
        glActiveTexture(GL_TEXTURE2);
        glGenTextures(1, &_samplerVTexture);
        glBindTexture(GL_TEXTURE_2D, _samplerVTexture);
        glUniform1i(_samplerVUniform, 2);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
        if (datalength>idxV && pyuvData[idxV]) {
            glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width/2, height/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, &pyuvData[idxV]);
        }else{
            if([LTJNYJCommon isDebug]){NSLog(@"error2");}
        }
        //----------------------------------------------------
        //glDrawElements
        //----------------------------------------------------
        glDrawElements(GL_TRIANGLE_STRIP, sizeof(LTGLIndices)/sizeof(GLubyte), GL_UNSIGNED_BYTE, 0);

//SHADER

precision highp float;

uniform sampler2D samplerY;
uniform sampler2D samplerU;
uniform sampler2D samplerV;

varying highp vec2 TexCoordOut;

const highp mat3 yuv2rgb = mat3(1, 1, 1,
                                0, -0.39465, 2.03211,
                                1.13983, -0.58060, 0);
void main(void)
{
    highp vec3 yuv;
    yuv.x = texture2D(samplerY, TexCoordOut).r;
    yuv.y = texture2D(samplerU, TexCoordOut).r - 0.5;
    yuv.z = texture2D(samplerV, TexCoordOut).r - 0.5;

    vec3 rgb = yuv2rgb * yuv;

    gl_FragColor = vec4(rgb, 1.0);
}

//
attribute vec4 Position;
attribute vec2 TexCoordIn;

uniform mat4 Projection;
uniform mat4 Modelview;

varying vec2 TexCoordOut;

void main(void)
{
    gl_Position = Position;
    TexCoordOut = TexCoordIn;
}

Solution

  • By default OpenGL assumes that the start of each row of an image is aligned 4 bytes. This is because the GL_UNPACK_ALIGNMENT parameter by default is 4.
    Since the images have 1 (GL_LUMINANCE) channel with a sizof 1 byte, and are tightly packed the start of a row is possibly misaligned.
    Change the the GL_UNPACK_ALIGNMENT parameter to 1, before specifying the two-dimensional texture image (glTexImage2D).

    Note glPixelStorei changes a global state, so it is sufficient to set the parameter once, before the texture images are specified by glTexImage2D.

    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
    

    If you don't set the alignment correctly, then this causes a shift effect at row of the texture and finally the image buffer is accessed out of bounds.

    Note, the issue is cause by the width of the first texture which is 3268. So widht/2 is 1634. 1634 is not divisible by 4 (1634/4 = 408,5).
    In compare the width of the 2nd texture is 1280. widht/2 is 640 and that is divisible by 4.
    Actually an alignment of 2 (glPixelStorei(GL_UNPACK_ALIGNMENT, 2);) would solve the issue, too (in this special case).