Search code examples
javascriptbrowserchromakeygpu.js

How to pass off heavy JavaScript math operations to GPU with GPU.js


Background
I've built a little web based application that pops up windows to display your webcam(s). I wanted to add the ability to chroma key your feed and have been successful in getting several different algorithms working. The best algorithm I have found however is very resource intensive for JavaScript; single threaded application.

Question
Is there a way to offload the intensive math operations to the GPU? I've tried getting GPU.js to work but I keep getting all kinds of errors. Here is the functions I would like to have the GPU run:

let dE76 = function(a, b, c, d, e, f) {
    return Math.sqrt( pow(d - a, 2) + pow(e - b, 2) + pow(f - c, 2) );
};


let rgbToLab = function(r, g, b) {
    
    let x, y, z;

    r = r / 255;
    g = g / 255;
    b = b / 255;

    r = (r > 0.04045) ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
    g = (g > 0.04045) ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
    b = (b > 0.04045) ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92;

    x = (r * 0.4124 + g * 0.3576 + b * 0.1805) / 0.95047;
    y = (r * 0.2126 + g * 0.7152 + b * 0.0722) / 1.00000;
    z = (r * 0.0193 + g * 0.1192 + b * 0.9505) / 1.08883;

    x = (x > 0.008856) ? Math.pow(x, 1/3) : (7.787 * x) + 16/116;
    y = (y > 0.008856) ? Math.pow(y, 1/3) : (7.787 * y) + 16/116;
    z = (z > 0.008856) ? Math.pow(z, 1/3) : (7.787 * z) + 16/116;

    return [ (116 * y) - 16, 500 * (x - y), 200 * (y - z) ];
};

What happens here is I send in an RGB value to rgbToLab which gives back the LAB value that can be compared to an already stored LAB value for my green screen with dE76. Then in my app we check the dE76 value to a threashold, say 25, and if the value is less than this I turn that pixel opacity to 0 in the video feed.

GPU.js Attempt
Here is my latest GUI.js attempt:

// Try to combine the 2 functions into a single kernel function for GPU.js
let tmp = gpu.createKernel( function( r, g, b, lab ) {

  let x, y, z;

  r = r / 255;
  g = g / 255;
  b = b / 255;

  r = (r > 0.04045) ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
  g = (g > 0.04045) ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
  b = (b > 0.04045) ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92;

  x = (r * 0.4124 + g * 0.3576 + b * 0.1805) / 0.95047;
  y = (r * 0.2126 + g * 0.7152 + b * 0.0722) / 1.00000;
  z = (r * 0.0193 + g * 0.1192 + b * 0.9505) / 1.08883;

  x = (x > 0.008856) ? Math.pow(x, 1/3) : (7.787 * x) + 16/116;
  y = (y > 0.008856) ? Math.pow(y, 1/3) : (7.787 * y) + 16/116;
  z = (z > 0.008856) ? Math.pow(z, 1/3) : (7.787 * z) + 16/116;

  let clab = [ (116 * y) - 16, 500 * (x - y), 200 * (y - z) ];
  
  let d = pow(lab[0] - clab[0], 2) + pow(lab[1] - clab[1], 2) + pow(lab[2] - clab[2], 2);
  
  return Math.sqrt( d );

} ).setOutput( [256] );

// ...

// Call the function above.
let d = tmp( r, g, b, chromaColors[c].lab );

// If the delta (d) is lower than my tolerance level set pixel opacity to 0.
if( d < tolerance ){
    frame.data[ i * 4 + 3 ] = 0;
}

ERRORS:
Here are a list of errors I get trying to use GPU.js when I call my tmp function. 1) is for the code I provided above. 2) is for erasing all the code in tmp and adding only an empty return 3) is if I try and add the functions inside the tmp function; a valid JavaScript thing but not C or kernel code.

  1. Uncaught Error: Identifier is not defined
  2. Uncaught Error: Error compiling fragment shader: ERROR: 0:463: ';' : syntax error
  3. Uncaught Error: Unhandled type FunctionExpression in getDependencies

Solution

  • Some typos

    pow should be Math.pow()
    

    and

    let x, y, z should be declare on there own
    
    let x = 0
    let y = 0
    let z = 0
    

    You cannot assign value to parameter variable. They become uniform.

    Full working script

    const { GPU } = require('gpu.js')
    const gpu = new GPU()
    
    const tmp = gpu.createKernel(function (r, g, b, lab) {
      let x = 0
      let y = 0
      let z = 0
    
      let r1 = r / 255
      let g1 = g / 255
      let b1 = b / 255
    
      r1 = (r1 > 0.04045) ? Math.pow((r1 + 0.055) / 1.055, 2.4) : r1 / 12.92
      g1 = (g1 > 0.04045) ? Math.pow((g1 + 0.055) / 1.055, 2.4) : g1 / 12.92
      b1 = (b1 > 0.04045) ? Math.pow((b1 + 0.055) / 1.055, 2.4) : b1 / 12.92
    
      x = (r1 * 0.4124 + g1 * 0.3576 + b1 * 0.1805) / 0.95047
      y = (r1 * 0.2126 + g1 * 0.7152 + b1 * 0.0722) / 1.00000
      z = (r1 * 0.0193 + g1 * 0.1192 + b1 * 0.9505) / 1.08883
    
      x = (x > 0.008856) ? Math.pow(x, 1 / 3) : (7.787 * x) + 16 / 116
      y = (y > 0.008856) ? Math.pow(y, 1 / 3) : (7.787 * y) + 16 / 116
      z = (z > 0.008856) ? Math.pow(z, 1 / 3) : (7.787 * z) + 16 / 116
    
      const clab = [(116 * y) - 16, 500 * (x - y), 200 * (y - z)]
      const d = Math.pow(lab[0] - clab[0], 2) + Math.pow(lab[1] - clab[1], 2) + Math.pow(lab[2] - clab[2], 2)
      return Math.sqrt(d)
    }).setOutput([256])
    
    console.log(tmp(128, 139, 117, [40.1332, 10.99816, 5.216413]))