Understanding this shader for YUV to RGB conversion

Question

I'm reading this shader for YUV planar conversion to RGB.

It deals with lots of YUV formats. As you can see, it first does something with the yuv values, and then do the rgb conversion in the end:

vec3 yuv;
vec4 rgba;
if(tex_format == 0 || tex_format == 1){
    if(tex_format == 0){
        yuv.r = texture2D(tex_y, textureOut).r - 0.0625;
    }else{
        yuv.r = texture2D(tex_y, textureOut).r;
    }
    yuv.g = texture2D(tex_u, textureOut).r - 0.5;
    yuv.b = texture2D(tex_v, textureOut).r - 0.5;
}else if(tex_format == 2){ // rgb
    yuv = texture2D(tex_y, textureOut).rgb;
}else if(tex_format == 3){ // gray8
    yuv.r = texture2D(tex_y, textureOut).r;
}else if(tex_format == 6){ //BGR
    yuv = texture2D(tex_y, textureOut).bgr;
}else if(tex_format == 10){//yuv420p10le yuv444p10le
    vec3 yuv_l;
    vec3 yuv_h;
    yuv_l.x = texture2D(tex_y, textureOut).r;
    yuv_h.x = texture2D(tex_y, textureOut).a;
    yuv_l.y = texture2D(tex_u, textureOut).r;
    yuv_h.y = texture2D(tex_u, textureOut).a;
    yuv_l.z = texture2D(tex_v, textureOut).r;
    yuv_h.z = texture2D(tex_v, textureOut).a;
    yuv = (yuv_l * 255.0 + yuv_h * 255.0 * 256.0) / (1023.0) - vec3(16.0 / 255.0, 0.5, 0.5);
}else if(tex_format == 8 || tex_format == 9){ //NV12 | NV21
    yuv.r = texture2D(tex_y, textureOut).r - 0.0625;
    vec4 uv = texture2D( tex_u, textureOut);
    if(tex_format == 9){ //NV21
        yuv.g = uv.a - 0.5;
        yuv.b = uv.r - 0.5;
    }else{ //NV12
        yuv.g = uv.r - 0.5;
        yuv.b = uv.a - 0.5;
    }
}else if(tex_format == 16 || tex_format == 17){ //YUV16 YUVJ16
    if(tex_format == 16){
        yuv.r = texture2D(tex_y, textureOut).r - 0.0625;
    }else{
        yuv.r = texture2D(tex_y, textureOut).r;
    }
    yuv.g = texture2D(tex_u, textureOut).r - 0.5;
    yuv.b = texture2D(tex_v, textureOut).r - 0.5;
}



if(tex_format == 0 || tex_format == 10 || tex_format == 16){//yuv | p10le | //YUV16
    rgba.r = yuv.r + 1.596 * yuv.b;
    rgba.g = yuv.r - 0.813 * yuv.b - 0.391 * yuv.g;
    rgba.b = yuv.r + 2.018 * yuv.g;
}else if(tex_format == 1 || tex_format == 17){ //yuy-jpeg || YUVJ16
    rgba.r = yuv.r + 1.402 * yuv.b;
    rgba.g = yuv.r - 0.34413 * yuv.g - 0.71414 * yuv.b;
    rgba.b = yuv.r + 1.772 * yuv.g;
}
else if(tex_format == 2){ //rgb
    rgba.rgb = yuv.rgb;
}else if(tex_format == 3){ //gray8
    rgba.r = yuv.r;
    rgba.g = yuv.r;
    rgba.b = yuv.r;
}else if(tex_format == 6){ //BGR
    rgba.r = yuv.b;
    rgba.g = yuv.g;
    rgba.b = yuv.r;
}else if(tex_format == 19){ // BGGR
    vec2 firstRed = vec2(1,1);
    rgba.r = texture2D(tex_y, textureOut).r;
    rgba.g = texture2D(tex_u, textureOut).r;
    rgba.b = texture2D(tex_v, textureOut).r;
    //        //        BGGR = 19,
    //        //        RGGB = 20 ,
    //        //        GRBG = 21 ,
    //        //        GBRG = 22 ,
}else if(tex_format == 20){ //RGGB
    vec2 firstRed = vec2(0,0);
}else if(tex_format == 21){ //GRBG
    vec2 firstRed = vec2(0,1);
}else if(tex_format == 22){ //GBRG
    vec2 firstRed = vec2(1,0);
}else if(tex_format == 23){//BGR565
    rgba.rgb = texture2D(tex_y, textureOut).bgr;
}else{ //其它
    rgba.r = yuv.r + 1.596 * yuv.b;
    rgba.g = yuv.r - 0.813 * yuv.b - 0.391 * yuv.g;
    rgba.b = yuv.r + 2.018 * yuv.g;
}
rgba.a = alpha;
if(enableHDR){
    rgba.rgb = toHDR(rgba.rgb,1.0);
}
rgba.a = alpha;
gl_FragColor = rgba;

For example, in the yuv420p10le format, it first does:

vec3 yuv_l;
vec3 yuv_h;
yuv_l.x = texture2D(tex_y, textureOut).r;
yuv_h.x = texture2D(tex_y, textureOut).a;
yuv_l.y = texture2D(tex_u, textureOut).r;
yuv_h.y = texture2D(tex_u, textureOut).a;
yuv_l.z = texture2D(tex_v, textureOut).r;
yuv_h.z = texture2D(tex_v, textureOut).a;
yuv = (yuv_l * 255.0 + yuv_h * 255.0 * 256.0) / (1023.0) - vec3(16.0 / 255.0, 0.5, 0.5);

and then, to those yuv values, it does this:

rgba.r = yuv.r + 1.596 * yuv.b;
rgba.g = yuv.r - 0.813 * yuv.b - 0.391 * yuv.g;
rgba.b = yuv.r + 2.018 * yuv.g;

I've found the second conversion above here, but I can't understand which conversion is the first one. What is it doing? What about the other ones?

Answer 1

The first conversion is a transformation that normalize the limited YUV data. In limited YUV formation, Y in limited in 16 to 240. If we use 16 to divide 256, the answer is 0.0625. So, it is a normalization step.