Processing | Program is lagging

Question

I'm new to Processing and I need to make a program that, captured the main monitor, shows on the second screen the average color and makes a spiral using another color (perceptual dominant color) get by a function.
The problem is that the program is so slow (lag, 1FPS). I think it's because it has too many things to do everytime i do a screenshot, but I have no idea how to make it faster.

Also there could be many other problems, but the main one is that.
Thank you very much!

Here's the code:

import java.awt.Robot;
import java.awt.AWTException;
import java.awt.Rectangle;
import java.awt.color.ColorSpace;

PImage screenshot; 

float a = 0;
int blockSize = 20;

int avg_c;
int per_c;


void setup() {
  fullScreen(2); // 1920x1080
  noStroke();
  frame.removeNotify();
}

void draw() { 
  screenshot();
  avg_c = extractColorFromImage(screenshot);
  per_c = extractAverageColorFromImage(screenshot);
  background(avg_c); // Average color
  spiral();
}


void screenshot() {
  try{
    Robot robot_Screenshot = new Robot();
    screenshot = new PImage(robot_Screenshot.createScreenCapture
    (new Rectangle(0, 0, displayWidth, displayHeight)));
  }
  catch (AWTException e){ }
  frame.setLocation(displayWidth/2, 0);
}

void spiral() {
  fill (per_c); 
  for (int i = blockSize; i < width; i += blockSize*2)
  {
    ellipse(i, height/2+sin(a+i)*100, blockSize+cos(a+i)*5, blockSize+cos(a+i)*5);    
    a += 0.001;
  }
}


color extractColorFromImage(PImage screenshot) { // Get average color
    screenshot.loadPixels(); 
    int r = 0, g = 0, b = 0; 
    for (int i = 0; i < screenshot.pixels.length; i++) { 
        color c = screenshot.pixels[i]; 
        r += c>>16&0xFF; 
        g += c>>8&0xFF; 
        b += c&0xFF;
    } 
    r /= screenshot.pixels.length; g /= screenshot.pixels.length; b /= screenshot.pixels.length;
    return color(r, g, b);
}

color extractAverageColorFromImage(PImage screenshot) { // Get lab average color (perceptual)
  float[] average = new float[3];
  CIELab lab = new CIELab();

  int numPixels = screenshot.pixels.length;
  for (int i = 0; i < numPixels; i++) {
    color rgb = screenshot.pixels[i];

    float[] labValues = lab.fromRGB(new float[]{red(rgb),green(rgb),blue(rgb)});

    average[0] += labValues[0];
    average[1] += labValues[1];
    average[2] += labValues[2];
  }

  average[0] /= numPixels;
  average[1] /= numPixels;
  average[2] /= numPixels;

  float[] rgb = lab.toRGB(average);

  return color(rgb[0] * 255,rgb[1] * 255, rgb[2] * 255);
}


public class CIELab extends ColorSpace {

    @Override
    public float[] fromCIEXYZ(float[] colorvalue) {
        double l = f(colorvalue[1]);
        double L = 116.0 * l - 16.0;
        double a = 500.0 * (f(colorvalue[0]) - l);
        double b = 200.0 * (l - f(colorvalue[2]));
        return new float[] {(float) L, (float) a, (float) b};
    }

    @Override
    public float[] fromRGB(float[] rgbvalue) {
        float[] xyz = CIEXYZ.fromRGB(rgbvalue);
        return fromCIEXYZ(xyz);
    }

    @Override
    public float getMaxValue(int component) {
        return 128f;
    }

    @Override
    public float getMinValue(int component) {
        return (component == 0)? 0f: -128f;
    }    

    @Override
    public String getName(int idx) {
        return String.valueOf("Lab".charAt(idx));
    }

    @Override
    public float[] toCIEXYZ(float[] colorvalue) {
        double i = (colorvalue[0] + 16.0) * (1.0 / 116.0);
        double X = fInv(i + colorvalue[1] * (1.0 / 500.0));
        double Y = fInv(i);
        double Z = fInv(i - colorvalue[2] * (1.0 / 200.0));
        return new float[] {(float) X, (float) Y, (float) Z};
    }

    @Override
    public float[] toRGB(float[] colorvalue) {
        float[] xyz = toCIEXYZ(colorvalue);
        return CIEXYZ.toRGB(xyz);
    }

    CIELab() {
        super(ColorSpace.TYPE_Lab, 3);
    }

    private double f(double x) {
        if (x > 216.0 / 24389.0) {
            return Math.cbrt(x);
        } else {
            return (841.0 / 108.0) * x + N;
        }
    }

    private double fInv(double x) {
        if (x > 6.0 / 29.0) {
            return x*x*x;
        } else {
            return (108.0 / 841.0) * (x - N);
        }
    }


    private final ColorSpace CIEXYZ =
        ColorSpace.getInstance(ColorSpace.CS_CIEXYZ);

    private final double N = 4.0 / 29.0;
}

Answer 1

There's lots that can be done, even beyond what's already been mentioned.

Iteration & Threading

After taking the screenshot, immediately iterate over every 1/N pixels (perhaps every 4 or 8) of the buffered image. Then, during this iteration, calculate the LAB value for each pixel (as you have each pixel channel directly available), and meanwhile increment the running total of each RGB channel.

This saves us from iterating over the same pixels twice and avoids unncessary conversions (BufferedImage → PImage; and composing then decomposing pixel channels from PImage pixels).

Likewise, we avoid Processing's expensive resize() call (as suggested in another answer), which is not something we want to call every frame (even though it does speed the program up, it's not an efficient method).

Now, on top of iteration change, we can wrap the iteration in a Callable to easily run the workload across multiple system threads concurrently (after all, pixel iteration is embarrassingly parallel); the example below does this with 2 threads, each screenshotting and processing half of the display's pixels.

Optimise RGB→XYZ→LAB conversion

We're not so concerned about the backwards conversion since that's only done for one value per frame

It looks like you've implemented XYZ→LAB yourself and are using the RGB→XYZ converter from java.awt.color.

As has been identified, the forward conversion XYZ→LAB uses a cbrt() which is as a bottleneck. I also imagine that the RGB→XYZ implementation makes 3 calls to Math.Pow(x, 2.4) — 3 non-integer exponents per pixel adds considerably to the computation. The solution is faster math...

Jafama

Jafama is a drop-in java.math replacement -- simply import the library and replace any Math.__() calls with FastMath.__() for a free speedup (you could go even further by trading Jafama's E-15 precision with less accurate and even faster dedicated LUT-based classes).

So at the very least, swap out Math.cbrt() for FastMath.cbrt(). Then consider implementing RGB→XYZ yourself (example), again using Jafama in place of java.math.

---

You may even find that for such a project, converting to XYZ only is a sufficient color space to work with to overcome the well known weaknesses with RGB (and therefore save yourself from the XYZ→LAB conversion).

Cache LAB Calculation

Unless most pixels are changing every frame, then consider caching the LAB value for every pixel, recalculating it only when the pixel has changed between the current the previous frames. The tradeoff here is the overhead from checking every pixel against its previous value, versus how much calculation positive checks will save. Given that the LAB calculation is much more expensive it's very worthwhile here. The example below uses this technique.

Screen Capture

No matter how well optimised the rest of the program is, a considerable bottleneck is the AWT Robot's createScreenCapture(). It will struggles to go past 30FPS on large enough displays. I can't offer any exact advice but it's worth looking at other screen capture methods in Java.

Reworked code with iteration changes & threading

This code implements what has discussed above minus any changes to the LAB calculation.

float a = 0;
int blockSize = 20;

int avg_c;
int per_c;

java.util.concurrent.ExecutorService threadPool = java.util.concurrent.Executors.newFixedThreadPool(4);

List<java.util.concurrent.Callable<Boolean>> taskList;

float[] averageLAB;
int totalR = 0, totalG = 0, totalB = 0; 

CIELab lab = new CIELab();

final int pixelStride = 8; // look at every 8th pixel


void setup() {
  size(800, 800, FX2D);
  noStroke();
  frame.removeNotify();

  taskList = new ArrayList<java.util.concurrent.Callable<Boolean>>();

  Compute thread1 = new Compute(0, 0, width, height/2);
  Compute thread2 = new Compute(0, height/2, width, height/2);
  taskList.add(thread1);
  taskList.add(thread2);
}

void draw() { 

  totalR = 0; // re init
  totalG = 0; // re init
  totalB = 0; // re init 
  averageLAB = new float[3]; // re init

  final int numPixels = (width*height)/pixelStride;

  try {
    threadPool.invokeAll(taskList); // run threads now and block until completion of all
  }
  catch (Exception e) {
    e.printStackTrace();
  }

  // calculate average LAB
  averageLAB[0]/=numPixels;
  averageLAB[1]/=numPixels;
  averageLAB[2]/=numPixels;

  final float[] rgb = lab.toRGB(averageLAB);
  per_c = color(rgb[0] * 255, rgb[1] * 255, rgb[2] * 255);

  // calculate average RGB
  totalR/=numPixels;
  totalG/=numPixels;
  totalB/=numPixels;

  avg_c = color(totalR, totalG, totalB);

  background(avg_c); // Average color
  spiral();
  fill(255, 0, 0);
  text(frameRate, 10, 20);
}

class Compute implements java.util.concurrent.Callable<Boolean> {

  private final Rectangle screenRegion;
  private Robot robot_Screenshot;
  private final int[] previousRGB;
  private float[][] previousLAB;

  Compute(int x, int y, int w, int h) {

    screenRegion = new Rectangle(x, y, w, h);

    previousRGB = new int[w*h];
    previousLAB = new float[w*h][3];

    try {
      robot_Screenshot = new Robot();
    } 
    catch (AWTException e1) {
      e1.printStackTrace();
    }
  }

  @Override
    public Boolean call() {

    BufferedImage rawScreenshot = robot_Screenshot.createScreenCapture(screenRegion);  

    int[] ssPixels = new int[rawScreenshot.getWidth()*rawScreenshot.getHeight()]; // screenshot pixels

    rawScreenshot.getRGB(0, 0, rawScreenshot.getWidth(), rawScreenshot.getHeight(), ssPixels, 0, rawScreenshot.getWidth()); // copy buffer to int[] array

    for (int pixel = 0; pixel < ssPixels.length; pixel+=pixelStride) {

      // get invididual colour channels
      final int pixelColor = ssPixels[pixel];
      final int R = pixelColor >> 16 & 0xFF;
      final int G = pixelColor >> 8 & 0xFF;
      final int B = pixelColor & 0xFF;

      if (pixelColor != previousRGB[pixel]) { // if pixel has changed recalculate LAB value
        float[] labValues = lab.fromRGB(new float[]{R/255f, G/255f, B/255f}); // note that I've fixed this; beforehand you were missing the /255, so it was always white.
        previousLAB[pixel] = labValues;
      }

      averageLAB[0] += previousLAB[pixel][0];
      averageLAB[1] += previousLAB[pixel][1];
      averageLAB[2] += previousLAB[pixel][2];

      totalR+=R;
      totalG+=G;
      totalB+=B;

      previousRGB[pixel] = pixelColor; // cache last result
    }
    return true;
  }
}

800x800px; pixelStride = 4; fairly static screen background