I'm working on a subtitle creation tool and would like to add a feature that displays the next 5 or so seconds of the audio waveform as the video plays. I believe having a visual preview of the upcoming audio will allow the user to more accurately place in points for subtitles.
I found this method using an analyzer but it limits the audio samples to 2048 (about 1/24th of a second) which is too small to useful a preview.
I also found this method by catching "audioprocess" events but is still limited to about 16k of samples or 1/3 of a second. Again, not long enough to be useful.
I did a quick test to see how long it would take to visualize 5 seconds worth of samples it's averaging in the low 20ms. So I think it's doable to process it in realtime provided I can access the buffer. I might have to lower the FPS but I think even at 15fps it should still be useful to the user.
Is there a method where I can access say the next 240,000 samples (5 seconds of audio) as the video is playing?
<!DOCTYPE html>
<html>
<body id="body">
<canvas id="canvas" width="1000" height="200"></canvas>
<script>
var count=0;
var samples=new Int32Array(48000*5);
var CANVAS_HEIGHT=200;
var canvas = document.getElementById('canvas');
var ctx = canvas.getContext('2d');
function processSamples() {
var start = new Date().getTime();
ctx.clearRect(0, 0, 1000, 200);
for (var i=0; i < samples.length; i++) {
samples[i]=parseInt(Math.random() * CANVAS_HEIGHT);
}
// We can't display each sample on it's own line... So take the average sample and display it
// 1000 pixels for 5 second preview. So 200px per second. 48,000 samples per second / 200px = 240 samples per pixel
var sample;
for (var i=0; i < samples.length/240; i++) {
sample=0;
for (var j=0; j < 240; j++) {
sample += samples[i*240+j];
}
sample=parseInt(sample/240);
ctx.fillRect(i, 0, 1, sample);
}
if (count < 10)
setTimeout(processSamples, 1000);
count++;
var end = new Date().getTime();
var time = end - start;
console.log('Execution time: ' + time);
}
window.onload=processSamples;
</script>
</body>
</html>
It appears at this time no way to extend the buffer to obtain a larger sample size during playback. However, I was able to generate a static waveform of the entire file via this method. I modified it here to use the File API.
You load the file into an arrayBuffer and then pass the buffer a audioContext.decodeAudioData(arrayBuffer, callbackFunction)
var audioContext = new AudioContext();
function drawBuffer( width, height, context, buffer ) {
var data = buffer.getChannelData( 0 );
var step = Math.ceil( data.length / width );
var amp = height / 2;
for(var i=0; i < width; i++){
var min = 1.0;
var max = -1.0;
for (var j=0; j<step; j++) {
var datum = data[(i*step)+j];
if (datum < min)
min = datum;
if (datum > max)
max = datum;
}
context.fillRect(i,(1+min)*amp,1,Math.max(1,(max-min)*amp));
}
}
function initAudio() {
var file = document.getElementById('fileItem').files[0];
var reader = new FileReader();
reader.onerror = function(e) {
console.log("Error reading file");
console.log(e);
}
reader.onload = function(e) {
console.log("loading");
var arrayBuffer = e.target.result;
audioContext.decodeAudioData( arrayBuffer,
function(buffer) {
console.log("drawing");
var canvas = document.getElementById("canvas");
drawBuffer( canvas.width, canvas.height, canvas.getContext('2d'), buffer );
} );
}
reader.readAsArrayBuffer(file);
}body {
width: 100%;
height: 100%;
}
#canvas {
position: absolute;
top: 0;
left: 0;
width: 800px;
height: 200px;
background-color: blue;
}
#fileItem {
position: absolute;
top: 205px;
left: 0px;
}
#button {
position: absolute;
top: 205px;
left: 250px;<body>
<canvas id="canvas"></canvas>
<input id="fileItem" type="file"/>
<button id="button" onclick="initAudio();">Draw Waveform</button>
</body>