I work with webcodecs and have found that when I manually convert an NV12 format to an RGB format, I get slightly different values than if I simply write the VideoFrame to a canvas and read the imageData.
The ImageData from the Canvas [87, 87, 39, 255, 86, 86, 39, 255, 85, 85, 39, 255, 83, 84, 39, 255, 81, 83, 39, 255, 79, 82, 38, 255, 77, 81, 37, 255, 76, 80, 36, 255, 79, 83, ... ]
The ImageData from manuel converting the NV12-Format to the RGB-Format: [94, 101, 62, 255, 94, 101, 62, 255, 95, 102, 63, 255, 97, 103, 64, 255, 97, 103, 64, 255, 98, 104, 66, 255, 100, 106, 68, 255, 101, 108, 69, 25, ...]
Can anyone tell me how these differences come about or how the Canvas converts a YUV format to an RGB format?
Here is the code for the two different options:
this.context.drawImage(frame, 0,0, this.canvas.width, this.canvas.height);
let imageData = this.context!.getImageData(0,0,this.canvas.width,this.canvas.height);
...
let imageData = convertToImageData(videoFrame);
...
public async convertToImageData(frame) {
let buffer : Uint8ClampedArray = new Uint8ClampedArray(frame.allocationSize());
await frame.copyTo(buffer)
let imageData = await this.convertNV12ToRGB(buffer)
return imageData;
}
private convertNV12ToRGB(buffer : Uint8ClampedArray) {
// Y should be from 0-921599 (1280x720)
// Cr & Cb should be from 921600 - 1382399 interleaved
// NOTE: Solution is slow: 1280 * 720; 1 Minute -> 44 Seconds full convert.
let imageData : ImageData = this.context!.getImageData(0, 0, 1280, 720);
let pixels : Uint8ClampedArray = imageData.data;
let row : number = 0;
let col : number = 0;
const plane2Start = 1280 * 720; //Hier starten die Chroma-Werte
for (row = 0; row < 720; row++) {
const p2row = (row % 2 === 0 ? row / 2 : row / 2 - 0.5) * 1280; //Hälfte der Row
let cr = 0;
let cb = 0;
const rowOffset : number = row * 1280;
for (col = 0; col < 1280; col++) {
const indexY = rowOffset + col;
let y = buffer[indexY];
if (col % 2 === 0) {
const indexCr = plane2Start + p2row + (col);
const indexCb = indexCr + 1;
cr = buffer[indexCr];
cb = buffer[indexCb];
}
this.yuvToRgb(y, cr, cb, row, col, pixels);
}
}
return imageData;
}
private yuvToRgb(y : number, u : number, v : number, row : number, col : number, pixels : Uint8ClampedArray) {
y -= 16;
u -= 128;
v -= 128;
let r = 1.164 * y + 1.596 * v;
let g = 1.164 * y - 0.392 * u - 0.813 * v;
let b = 1.164 * y + 2.017 * u;
const index = (row * 1280 + col) * 4;
pixels[index] = r;
pixels[index + 1] = g;
pixels[index + 2] = b;
pixels[index + 3] = 255;
} ```
The conversion formula used by Canvas applies BT.709 "limited range" conversion coefficients.
Conversion YUV to RGB matrix coefficients (after subtracting [16, 128, 128]):
1.1644 -0.0000 1.7927
1.1644 -0.2132 -0.5329
1.1644 2.1124 0.0000
It looks like Canvas conversion uses some kind of approximation, because I can't get the exact results.
JavaScript pixel conversion code:
function yuvToRgb(y, u, v, row, col, pixels) {
y -= 16;
u -= 128;
v -= 128;
//let r = 1.164 * y + 1.596 * v;
//let g = 1.164 * y - 0.392 * u - 0.813 * v;
//let b = 1.164 * y + 2.017 * u;
//Use BT.709 "limited range" conversion formula:
//1.1644 -0.0000 1.7927
//1.1644 -0.2132 -0.5329
//1.1644 2.1124 0.0000
let r = 1.1644 * y + 1.7927 * v;
let g = 1.1644 * y - 0.2132 * u - 0.5329 * v;
let b = 1.1644 * y + 2.1124 * u;
const index = (row * 4 + col) * 4;
pixels[index] = Math.max(Math.min(Math.round(r), 255), 0); //Round and clip range to [0, 255]
pixels[index + 1] = Math.max(Math.min(Math.round(g), 255), 0);
pixels[index + 2] = Math.max(Math.min(Math.round(b), 255), 0);
pixels[index + 3] = 255;
}
Notes:
u applies cb, and v applies cr.Testing code:
function yuvToRgb(y, u, v, row, col, pixels) {
y -= 16;
u -= 128;
v -= 128;
//let r = 1.164 * y + 1.596 * v;
//let g = 1.164 * y - 0.392 * u - 0.813 * v;
//let b = 1.164 * y + 2.017 * u;
//Use BT.709 "limited range" conversion formula:
//1.1644 -0.0000 1.7927
//1.1644 -0.2132 -0.5329
//1.1644 2.1124 0.0000
let r = 1.1644 * y + 1.7927 * v;
let g = 1.1644 * y - 0.2132 * u - 0.5329 * v;
let b = 1.1644 * y + 2.1124 * u;
const index = (row * 4 + col) * 4;
pixels[index] = Math.max(Math.min(Math.round(r), 255), 0); //Round and clip range to [0, 255]
pixels[index + 1] = Math.max(Math.min(Math.round(g), 255), 0);
pixels[index + 2] = Math.max(Math.min(Math.round(b), 255), 0);
pixels[index + 3] = 255;
}
//read image
function convertNV12ToRGBExample() {
let NV12 = new Array(16, 81, 145, 210, 16, 170, 106, 41, 16, 170, 106, 41, 81, 177, 170, 74, 128, 128, 128, 128, 105, 118, 207, 128);
let RGBA = new Array(64); //Output
let row = 0;
let col = 0;
const plane2Start = 4 * 4;
for (row = 0; row < 4; row++) {
const p2row = (row % 2 === 0 ? row / 2 : (row-1) / 2) * 4;
let cr = 0;
let cb = 0;
const rowOffset = row * 4;
for (col = 0; col < 4; col++) {
const indexY = rowOffset + col;
let y = NV12[indexY];
if (col % 2 === 0) {
//const indexCr = plane2Start + p2row + col;
//const indexCb = indexCr + 1;
const indexCb = plane2Start + p2row + col;
const indexCr = indexCb + 1;
cb = NV12[indexCb];
cr = NV12[indexCr];
}
yuvToRgb(y, cb, cr, row, col, RGBA);
}
}
console.log(RGBA);
}
MATLAB testing code (code is incomplete):
NV12 = uint8([16,81,145,210,16,170,106,41,16,170,106,41,81,177,170,74,128,128,128,128,105,118,207,128]);
convertNV12ToRGB = uint8([0, 0, 0, 255, 75, 75, 75, 255, 150, 150, 150, 255, 225, 225, 225, 255, 241, 17, 210, 255, 163, 196, 132, 255, 104, 73, 8, 255, 29, 255, 188, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255]);
Canvas2D = uint8([0, 0, 0, 255, 76, 76, 76, 255, 150, 150, 150, 255, 226, 226, 226, 255, 0, 0, 0, 255, 179, 179, 179, 255, 105, 105, 105, 255, 29, 29, 29, 255, 0, 10, 0, 255, 161, 190, 133, 255, 105, 87, 255, 255, 29, 12, 187, 255, 58, 86, 30, 255, 169, 198, 141, 255, 179, 162, 255, 255, 68, 50, 226, 255]);
Y = reshape(NV12(1:16), [4, 4])';
U = reshape(NV12(17:2:end), [2, 2])';
V = reshape(NV12(18:2:end), [2, 2])';
U = imresize(U, [4, 4], 'nearest');
V = imresize(V, [4, 4], 'nearest');
YUV = cat(3, Y, U, V);
T = [1.1644 -0.0000 1.7927
1.1644 -0.2132 -0.5329
1.1644 2.1124 0.0000];
yuv = double(YUV);
yuv(:,:,1) = yuv(:,:,1) - 16;
yuv(:,:,2) = yuv(:,:,2) - 128;
yuv(:,:,3) = yuv(:,:,3) - 128;
RGB = zeros(size(Y,1), size(Y,2), 3);
RGB(:,:,1) = T(1,1) * yuv(:,:,1) + T(1,2) * yuv(:,:,2) + T(1,3) * yuv(:,:,3);
RGB(:,:,2) = T(2,1) * yuv(:,:,1) + T(2,2) * yuv(:,:,2) + T(2,3) * yuv(:,:,3);
RGB(:,:,3) = T(3,1) * yuv(:,:,1) + T(3,2) * yuv(:,:,2) + T(3,3) * yuv(:,:,3);
RGB = uint8(round(RGB));
RGBA = cat(3, RGB(:,:,1), RGB(:,:,2), RGB(:,:,3), uint8(ones(4)*255)); % Convert to RGBA
rgba_data = reshape(permute(RGBA, [3, 2, 1]), [1, 64]); % Convert to vector
C = permute(reshape(Canvas2D, [4, 4, 4]), [3, 2, 1]);
C = C(:, :, 1:3);