I need to do a computation intensive image pixel manipulation in a browser using JavaScript. Pixel manipulation includes histogram equalization, converting to grayscale using lightness (L* of CIE Lab color space).
To not block the UI thread, I want to perform this pixel manipulation in a dedicated web worker.
The processed image is not going to be directly drawn to the canvas only once. The image can be zoomed and panned, some other shapes will be drawn on top of it. Thus, I can't use canvas.transferControlToOffscreen().
I came up with a solution to create an OffsceenCanvas using the constructor, draw the image and get the ImageData. Then send the ImageData as a transferable object to the worker worker.postMessage(imageData, [imageData.data.buffer]) and after processing it send it back from the worker to the main thread self.postMessage(imageData, [imageData.data.buffer]).
Is it an optimal solution or there are other more efficient patterns?
main.js
const canvas = document.getElementById('canvas');
const zoom = 1, offsetX = 0, offsetY = 0;
const offscreen, offscreenCtx;
const draw = () => {
if (!offscreen) {
return;
}
const ctx = canvas.getContext('2d');
ctx.save();
// simplified zooming and panning
ctx.scale(zoom, zoom);
ctx.translate(offsetX, offsetY);
ctx.drawImage(offscreen, 0, 0);
ctx.restore();
};
canvas.addEventListener('wheel', e => {
e.preventDefault();
if (e.deltaY > 0) {
zoom *= 1.1;
} else {
zoom /= 1.1;
}
draw();
});
const worker = new Worker(new URL('./worker.js', import.meta.url), {
type: 'module',
});
worker.addEventListener('message', message => {
const imageData = message.data;
offscreenCtx.putImageData(imageData, 0, 0);
draw();
});
const image = new Image();
image.addEventListener('load', () => {
offscreen = new OffscreenCanvas(image.width, image.height);
offscreenCtx = canvas.getContext('2d');
offscreenCtx.drawImage(image, 0, 0);
const imageData = offscreenCtx.getImageData(0, 0, image.width, image.height);
// clear the offscreen canvas to make sure the unprocessed image can't be sh
offscreenCtx.clearRect(0, 0, image.width, image.height);
worker.postMessage(imageData, [imageData.data.buffer]);
});
document.getElementById('image-select').addEventListener('change', e) => {
const files = e.target.files;
if (files && files.length > 0) {
const file = files[0];
if (/image\/.*/.test(file.type)) {
image.src = URL.createObjectURL(file);
}
}
});
worker.js
self.addEventListener('message', message => {
const imageData = message.data;
pixelManipulation(imageData);
self.postMessage(imageData, [imageData.data.buffer]);
self.close();
});
The image can be zoomed and panned, some other shapes will be drawn on top of it. Thus, I can't use canvas.transferControlToOffscreen().
Not sure how you come to that conclusion here. You can very well draw zoomed or panned images over an OffcreenCanvas that lives in a worker and renders over a placeholder canvas.
The only issue would be handling events, but you could still do this from the main thread and send only the drawing params to you worker, or I wrote an EventPort prototype some time ago that helps for this kind of scenario.
Now, if you really don't want all your code to live in the Worker thread, you can still move most of what you are doing there.
Do not use an HTMLImageElement to load and decode your image, instead use createImageBitmap(Blob) from the worker thread directly, this means that you'll have to fetch() your image yourself, but you'll save processing.
[Note]
We unfortunately still don't have an ImageBitmap#getImageData() method, but a potentially better performing path than through an OffscreenCanvas is to use a VideoFrame and then copy its data into a buffer, though it's still experimental (only in Chromium browsers) and we still don't have a way to enforce an RGBA format, so I'll keep it hidden for now, but it may still be useful.
async function readImageData(url) {
const resp = await fetch(url);
if (!resp.ok) { throw "Network Error"; }
const blob = await resp.blob();
const bmp = await createImageBitmap(blob);
const { width, height } = bmp;
if ("VideoFrame" in window) {
const frame = new VideoFrame(bmp, { timestamp: 0 })
bmp.close();
const { format } = frame;
if (format !== "RGBA" || format !== "RGBX") {
console.warn("This image would require a special parsing logic for: ", format);
}
const arr = new Uint8Array(frame.allocationSize());
frame.copyTo(arr);
frame.close();
return { width, height, data: arr, format: frame.format };
}
console.warn("using legacy canvas2d.getImageData()");
const canvas = new OffscreenCanvas(bmp.width, bmp.height);
const ctx = canvas.getContext("2d", { willReadFrequently: true });
ctx.drawImage(bmp, 0, 0);
bmp.close();
return ctx.getImageData(0, 0, width, height);
}
readImageData("https://upload.wikimedia.org/wikipedia/commons/thumb/9/9d/Elakha.jpg/320px-Elakha.jpg")
.then(console.log)
.catch(console.error);So we have the decoding of the image in the worker, the extraction of the pixels data, you have the processing, now remains the conversion of the ImageData to bitmap, that you do when you call putImageData().
Zooming and panning an ImageData implies a two step rasterization, where you first rasterize untransformed and then transform that bitmap.
So instead of passing the ImageData to you main thread, use once more createImageBitmap() passing the ImageData object, and send the resulting ImageBitmap to the rendering thread instead. You'll be able to do your transform from there directly:
const workerContent = document.querySelector("[type=worker]").textContent;
const workerURL = URL.createObjectURL(new Blob([workerContent], { type: "text/javascript" }));
const worker = new Worker(workerURL);
worker.onmessage = ({data: bmp}) => {
const canvas = document.querySelector("canvas");
// Rendering rotated 90deg
canvas.width = bmp.height;
canvas.height = bmp.width;
const ctx = canvas.getContext("2d");
ctx.translate(canvas.width/2, canvas.height/2);
ctx.rotate(Math.PI/2);
ctx.drawImage(bmp, -bmp.width/2, -bmp.height/2);
};
worker.postMessage("https://upload.wikimedia.org/wikipedia/commons/thumb/9/9d/Elakha.jpg/320px-Elakha.jpg");<script type="worker">
function process({ data }) {
for (let i = 0; i<data.length; i+=4) {
const r = data[i];
data[i] = data[i+1];
data[i+1] = r;
}
};
onmessage = async ({ data: url }) => {
const resp = await fetch(url);
if (!resp.ok) { throw "Network Error"; }
const blob = await resp.blob();
const source = await createImageBitmap(blob);
const { width, height } = source;
// Using legacy context2d.getImageData() for now.
const canvas = new OffscreenCanvas(width, height);
const ctx = canvas.getContext("2d", { willReadFrequently: true });
ctx.drawImage(source, 0, 0);
source.close(); // free memory, we don't need it anymore
const imageData = ctx.getImageData(0, 0, width, height);
process(imageData);
const bmp = await createImageBitmap(imageData);
// Transferring the ImageBitmap auto closes it.
self.postMessage(bmp, [bmp]);
}
</script>
<canvas></canvas>Finally, DO NOT CREATE ONE SHOT WORKERS, write your code in ways it's reusable. Starting a worker is an heavy operation, keeping it alive is fine in comparison.
So if you need to process multiple images or even if you need to do other works in another thread, keep that worker alive and make it handle all these jobs.