What I want is the following:
div, recording display time as precisely as possible (also, any image may be displayed multiple time)div (this should be fairly easy if I have point 2)For more context: this is a response time experiment where online participants see the pictures and make key responses to them (and their response time is measured). (Also, there is at least 200-500 ms between displays, so it is no problem if the preparation takes a little time before each display.)
I already have a full working code for this, which I put together from various previous answers: could someone verify that my code makes sense, and that there is nothing else I could do to make it more efficient?
One particular thing that confuses me is what "appendChild" exactly does in the browser. (In other words: I get that the element will be added to the page, but I don't know what that means for the browser.) Currently I'm appending the image just a little (100 ms) before I actually display it, which I then achieve by setting opacity from 0 to 1 (which is presumed to be the most optimal method for precise timing). Then I remove the image (empty the div) before displaying the next image. But I wonder if there is any point to doing it like that. For example, what if I appended all images already in the "preloading phase" and just set their opacity to 0 whenever they are not needed? How does that affect performance? Or otherwise, what if (though I'd rather not do this) I append them right before the opacity change? Does "appending" require any time, or can it in any way affect the timing of the upcoming opacity change? (EDIT: Now I have the answer to the main question, but it would still be nice to get an explanation about this point.)
The obvious problem is that I cannot really measure the display time precisely (without external hardware), so I have to rely on what "seems to make sense".
In any case, below is my code.
var preload = ['https://www.gstatic.com/webp/gallery/1.jpg', 'https://www.gstatic.com/webp/gallery3/1.png', 'https://www.gstatic.com/webp/gallery/4.jpg', 'https://www.gstatic.com/webp/gallery3/5.png'];
var promises = [];
var images = {};
// the function below preloads images; its completion is detected by the function at the end of this script
for (var i = 0; i < preload.length; i++) {
(function(url, promise) {
var filename = url.split('/').pop().split('#')[0].split('?')[0];
images[filename] = new Image();
images[filename].id = filename; // i need an id to later change its opacity
images[filename].style.opacity = 0;
images[filename].style.willChange = 'opacity';
images[filename].style['max-height'] = '15%';
images[filename].style['max-width'] = '15%';
images[filename].onload = function() {
promise.resolve();
};
images[filename].src = url;
})(preload[i], promises[i] = $.Deferred());
}
// the function below does the actual display
function image_display(img_name, current_div) {
window.warmup_needed = true;
document.getElementById(current_div).innerHTML = ''; // remove previous images
document.getElementById(current_div).appendChild(images[img_name]); // append new image (invisible; opacity == 0)
chromeWorkaroundLoop(); // part of the timing mechanism
setTimeout(function() {
document.getElementById(img_name).style.opacity = 1; // HERE i make the image visible
requestPostAnimationFrame(function() {
window.stim_start = now(); // HERE i catch the time of display (image painted on screen)
warmup_needed = false;
});
}, 100); // time needed for raF timing "warmup"
}
// below are functions for precise timing; see https://stackoverflow.com/questions/50895206/
function monkeyPatchRequestPostAnimationFrame() {
const channel = new MessageChannel();
const callbacks = [];
let timestamp = 0;
let called = false;
channel.port2.onmessage = e => {
called = false;
const toCall = callbacks.slice();
callbacks.length = 0;
toCall.forEach(fn => {
try {
fn(timestamp);
} catch (e) {}
});
};
window.requestPostAnimationFrame = function(callback) {
if (typeof callback !== 'function') {
throw new TypeError('Argument 1 is not callable');
}
callbacks.push(callback);
if (!called) {
requestAnimationFrame((time) => {
timestamp = time;
channel.port1.postMessage('');
});
called = true;
}
};
}
if (typeof requestPostAnimationFrame !== 'function') {
monkeyPatchRequestPostAnimationFrame();
}
function chromeWorkaroundLoop() {
if (warmup_needed) {
requestAnimationFrame(chromeWorkaroundLoop);
}
}
// below i execute some example displays after preloading is complete
$.when.apply($, promises).done(function() {
console.log("All images ready!");
// now i can display images
// e.g.:
image_display('1.jpg', 'my_div');
// then, e.g. 1 sec later another one
setTimeout(function() {
image_display('5.png', 'my_div');
}, 1000);
});<script src="https://code.jquery.com/jquery-3.4.1.slim.min.js"></script>
<div id='my_div'></div>One problem is that officially, onload only tells us about the network status (+ metadata decoding).
There may still be some other operations that the browser has to perform before being able to render it on the screen, like fully decoding it to a bitmap that can be moved to the GPU so it finally gets presented to the screen.
This can be seen at least in Firefox by drawing your images on a canvas using the synchronous drawImage method; with big images this drawImage() call can take a few ms even after your code ran, showing that they didn't yet completely decoded it:
var preload = ['https://upload.wikimedia.org/wikipedia/commons/c/cf/Black_hole_-_Messier_87.jpg'];
var promises = [];
var images = {};
// the function below preloads images; its completion is detected by the function at the end of this script
for (var i = 0; i < preload.length; i++) {
(function(url, promise) {
var filename = url.split('/').pop().split('#')[0].split('?')[0];
images[filename] = new Image();
images[filename].id = filename; // i need an id to later change its opacity
images[filename].style.opacity = 0;
images[filename].style.willChange = 'opacity';
images[filename].style['max-height'] = '15%';
images[filename].style['max-width'] = '15%';
images[filename].onload = function() {
promise.resolve();
};
images[filename].onerror = promise.reject;
images[filename].src = url;
})(preload[i] + '?r='+Math.random(), promises[i] = $.Deferred());
}
// the function below does the actual display
function image_display(img_name, current_div) {
window.warmup_needed = true;
document.getElementById(current_div).innerHTML = ''; // remove previous images
document.getElementById(current_div).appendChild(images[img_name]); // append new image (invisible; opacity == 0)
chromeWorkaroundLoop(); // part of the timing mechanism
setTimeout(function() {
document.getElementById(img_name).style.opacity = 1;
const c = document.createElement('canvas');
c.width = document.getElementById(img_name).width
c.height = document.getElementById(img_name).height;
console.time('paint1'); c.getContext("2d").drawImage(document.getElementById(img_name),0,0);
console.timeEnd('paint1');
console.time('paint2'); c.getContext("2d").drawImage(document.getElementById(img_name),0,0);
console.timeEnd('paint2');
// HERE i make the image visible
requestPostAnimationFrame(function() {
window.stim_start = now(); // HERE i catch the time of display (image painted on screen)
warmup_needed = false;
});
}, 100); // time needed for raF timing "warmup"
}
// below are functions for precise timing; see https://stackoverflow.com/questions/50895206/
function monkeyPatchRequestPostAnimationFrame() {
const channel = new MessageChannel();
const callbacks = [];
let timestamp = 0;
let called = false;
channel.port2.onmessage = e => {
called = false;
const toCall = callbacks.slice();
callbacks.length = 0;
toCall.forEach(fn => {
try {
fn(timestamp);
} catch (e) {}
});
};
window.requestPostAnimationFrame = function(callback) {
if (typeof callback !== 'function') {
throw new TypeError('Argument 1 is not callable');
}
callbacks.push(callback);
if (!called) {
requestAnimationFrame((time) => {
timestamp = time;
channel.port1.postMessage('');
});
called = true;
}
};
}
if (typeof requestPostAnimationFrame !== 'function') {
monkeyPatchRequestPostAnimationFrame();
}
function chromeWorkaroundLoop() {
if (warmup_needed) {
requestAnimationFrame(chromeWorkaroundLoop);
}
}
// below i execute some example displays after preloading is complete
$.when.apply($, promises).done(function() {
console.log("All images ready!");
// now i can display images
// e.g.:
image_display('Black_hole_-_Messier_87.jpg', 'my_div');
}).catch(console.error);<h4>run this snippet in Firefox</h4>
<script src="https://code.jquery.com/jquery-3.4.1.slim.min.js"></script>
<div id='my_div'></div>On my FF I get
All images ready!
paint1: 665.000ms
paint2: 0.000ms
This means that with this huge image, the screen would be empty for 650ms after you start measuring my response time.
There is a .decode() method that has been added to the HTMLImageElement interface, and which should get us closer to the painting time, but if you read correctly the introduction of this answer, we already found a better solution:
CanvasRenderingContext2D.drawImage() is synchronous.
Instead of presenting different <img>, risking a complete page reflow, triggering of CSS transitions or what else that could delay the display of your images, stay at low level, use a single visible DOM element, and use synchronous painting methods. In other words, use an HTMLCanvasElement.
var canvas = document.getElementById('mycanvas');
var ctx = canvas.getContext('2d');
// modified version of image_display function
function image_display(img_name, current_div) {
window.warmup_needed = true;
chromeWorkaroundLoop(); // part of the timing mechanism
setTimeout(function() {
// clear previous
ctx.clearRect( 0, 0, canvas.width, canvas.height );
// request the painting to canvas
// synchronous decoding + conversion to bitmap
var img = images[img_name];
// probably better to use a fixed sized canvas
// but if you wish you could also change its width and height attrs
var ratio = img.naturalHeight / img.naturalWidth;
ctx.drawImage( img, 0, 0, canvas.width, canvas.width * ratio );
// at this point it's been painted to the canvas,
// remains only passing it to the screen
requestPostAnimationFrame(function() {
window.stim_start = performance.now();
warmup_needed = false;
});
}, 100); // time needed for raF timing "warmup"
}
// preloading is kept the same
var preload = ['https://www.gstatic.com/webp/gallery/1.jpg', 'https://www.gstatic.com/webp/gallery3/1.png', 'https://www.gstatic.com/webp/gallery/4.jpg', 'https://www.gstatic.com/webp/gallery3/5.png'];
var promises = [];
var images = {};
// the function below preloads images; its completion is detected by the function at the end of this script
for (var i = 0; i < preload.length; i++) {
(function(url, promise) {
var filename = url.split('/').pop().split('#')[0].split('?')[0];
images[filename] = new Image();
images[filename].id = filename; // i need an id to later change its opacity
images[filename].style.opacity = 0;
images[filename].style.willChange = 'opacity';
images[filename].style['max-height'] = '15%';
images[filename].style['max-width'] = '15%';
images[filename].onload = function() {
promise.resolve();
};
images[filename].src = url;
})(preload[i], promises[i] = $.Deferred());
}
// below are functions for precise timing; see https://stackoverflow.com/questions/50895206/
function monkeyPatchRequestPostAnimationFrame() {
const channel = new MessageChannel();
const callbacks = [];
let timestamp = 0;
let called = false;
channel.port2.onmessage = e => {
called = false;
const toCall = callbacks.slice();
callbacks.length = 0;
toCall.forEach(fn => {
try {
fn(timestamp);
} catch (e) {}
});
};
window.requestPostAnimationFrame = function(callback) {
if (typeof callback !== 'function') {
throw new TypeError('Argument 1 is not callable');
}
callbacks.push(callback);
if (!called) {
requestAnimationFrame((time) => {
timestamp = time;
channel.port1.postMessage('');
});
called = true;
}
};
}
if (typeof requestPostAnimationFrame !== 'function') {
monkeyPatchRequestPostAnimationFrame();
}
function chromeWorkaroundLoop() {
if (warmup_needed) {
requestAnimationFrame(chromeWorkaroundLoop);
}
}
// below i execute some example displays after preloading is complete
$.when.apply($, promises).done(function() {
console.log("All images ready!");
// now i can display images
// e.g.:
image_display('1.jpg', 'my_div');
// then, e.g. 1 sec later another one
setTimeout(function() {
image_display('5.png', 'my_div');
}, 1000);
});canvas {
max-width: 15%;
max-height: 15%
}<script src="https://code.jquery.com/jquery-3.4.1.slim.min.js"></script>
<canvas id="mycanvas" width="500" height="500"></canvas>Now beware, a web browser is far from being the perfect tool for what you are trying to do. They prefer UI responsiveness and low memory usage over response time accuracy.