Background
I've built a little web based application that pops up windows to display your webcam(s). I wanted to add the ability to chroma key your feed and have been successful in getting several different algorithms working. The best algorithm I have found however is very resource intensive for JavaScript; single threaded application.
Question
Is there a way to offload the intensive math operations to the GPU? I've tried getting GPU.js to work but I keep getting all kinds of errors. Here is the functions I would like to have the GPU run:
let dE76 = function(a, b, c, d, e, f) {
return Math.sqrt( pow(d - a, 2) + pow(e - b, 2) + pow(f - c, 2) );
};
let rgbToLab = function(r, g, b) {
let x, y, z;
r = r / 255;
g = g / 255;
b = b / 255;
r = (r > 0.04045) ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
g = (g > 0.04045) ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
b = (b > 0.04045) ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92;
x = (r * 0.4124 + g * 0.3576 + b * 0.1805) / 0.95047;
y = (r * 0.2126 + g * 0.7152 + b * 0.0722) / 1.00000;
z = (r * 0.0193 + g * 0.1192 + b * 0.9505) / 1.08883;
x = (x > 0.008856) ? Math.pow(x, 1/3) : (7.787 * x) + 16/116;
y = (y > 0.008856) ? Math.pow(y, 1/3) : (7.787 * y) + 16/116;
z = (z > 0.008856) ? Math.pow(z, 1/3) : (7.787 * z) + 16/116;
return [ (116 * y) - 16, 500 * (x - y), 200 * (y - z) ];
};
What happens here is I send in an RGB value to rgbToLab
which gives back the LAB value that can be compared to an already stored LAB value for my green screen with dE76
. Then in my app we check the dE76
value to a threashold, say 25, and if the value is less than this I turn that pixel opacity to 0 in the video feed.
GPU.js Attempt
Here is my latest GUI.js attempt:
// Try to combine the 2 functions into a single kernel function for GPU.js
let tmp = gpu.createKernel( function( r, g, b, lab ) {
let x, y, z;
r = r / 255;
g = g / 255;
b = b / 255;
r = (r > 0.04045) ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
g = (g > 0.04045) ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
b = (b > 0.04045) ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92;
x = (r * 0.4124 + g * 0.3576 + b * 0.1805) / 0.95047;
y = (r * 0.2126 + g * 0.7152 + b * 0.0722) / 1.00000;
z = (r * 0.0193 + g * 0.1192 + b * 0.9505) / 1.08883;
x = (x > 0.008856) ? Math.pow(x, 1/3) : (7.787 * x) + 16/116;
y = (y > 0.008856) ? Math.pow(y, 1/3) : (7.787 * y) + 16/116;
z = (z > 0.008856) ? Math.pow(z, 1/3) : (7.787 * z) + 16/116;
let clab = [ (116 * y) - 16, 500 * (x - y), 200 * (y - z) ];
let d = pow(lab[0] - clab[0], 2) + pow(lab[1] - clab[1], 2) + pow(lab[2] - clab[2], 2);
return Math.sqrt( d );
} ).setOutput( [256] );
// ...
// Call the function above.
let d = tmp( r, g, b, chromaColors[c].lab );
// If the delta (d) is lower than my tolerance level set pixel opacity to 0.
if( d < tolerance ){
frame.data[ i * 4 + 3 ] = 0;
}
ERRORS:
Here are a list of errors I get trying to use GPU.js when I call my tmp function. 1) is for the code I provided above. 2) is for erasing all the code in tmp and adding only an empty return 3) is if I try and add the functions inside the tmp function; a valid JavaScript thing but not C or kernel code.
Some typos
pow should be Math.pow()
and
let x, y, z should be declare on there own
let x = 0
let y = 0
let z = 0
You cannot assign value to parameter variable. They become uniform.
Full working script
const { GPU } = require('gpu.js')
const gpu = new GPU()
const tmp = gpu.createKernel(function (r, g, b, lab) {
let x = 0
let y = 0
let z = 0
let r1 = r / 255
let g1 = g / 255
let b1 = b / 255
r1 = (r1 > 0.04045) ? Math.pow((r1 + 0.055) / 1.055, 2.4) : r1 / 12.92
g1 = (g1 > 0.04045) ? Math.pow((g1 + 0.055) / 1.055, 2.4) : g1 / 12.92
b1 = (b1 > 0.04045) ? Math.pow((b1 + 0.055) / 1.055, 2.4) : b1 / 12.92
x = (r1 * 0.4124 + g1 * 0.3576 + b1 * 0.1805) / 0.95047
y = (r1 * 0.2126 + g1 * 0.7152 + b1 * 0.0722) / 1.00000
z = (r1 * 0.0193 + g1 * 0.1192 + b1 * 0.9505) / 1.08883
x = (x > 0.008856) ? Math.pow(x, 1 / 3) : (7.787 * x) + 16 / 116
y = (y > 0.008856) ? Math.pow(y, 1 / 3) : (7.787 * y) + 16 / 116
z = (z > 0.008856) ? Math.pow(z, 1 / 3) : (7.787 * z) + 16 / 116
const clab = [(116 * y) - 16, 500 * (x - y), 200 * (y - z)]
const d = Math.pow(lab[0] - clab[0], 2) + Math.pow(lab[1] - clab[1], 2) + Math.pow(lab[2] - clab[2], 2)
return Math.sqrt(d)
}).setOutput([256])
console.log(tmp(128, 139, 117, [40.1332, 10.99816, 5.216413]))