I want to do an audio input vizualizer with Phaser 3, I’m trying to get the mic input to a shader but I can’t find a way to make it work.
I have a basic understanding on shaders and I can work with textures that are images but I really don’t understand how to provide a sound. I checked a working example made in three.js: three.js webaudio - visualizer and I already managed to get the sound input from the microphone as an Uint8Array of 1024 numbers.
Here’s the shader I’m using:
// simplesound.gsl.js
#ifdef GL_ES
precision highp float;
#endif
precision mediump float;
uniform vec2 resolution;
uniform sampler2D iChannel0;
varying vec2 fragCoord;
void main() {
vec2 uv = fragCoord.xy / resolution.xy;
vec2 mu = texture2D(iChannel0, uv).rg;
float y = uv.y - mu.x;
y = smoothstep(0., 0.02, abs(y - 0.1));
gl_FragColor = vec4(y);
}
And here's my scene code, trying to make it work:
import Phaser from 'phaser';
// This will provide the array mentioned above with code that will use `navigator.getUserMedia`.
import { setupAudioContext } from '../audiostream';
export default class MainScene2 extends Phaser.Scene {
constructor() {
super({ key: 'MainScene2' });
}
preload() {
this.load.glsl('simplesound', '/static/simplesound.glsl.js');
}
create() {
this.shader = this.add.shader('simplesound', 400, 300, 800, 600);
// When the user presses the 'g' key we will start listening for mic input
const GKey = this.input.keyboard.addKey('G');
GKey.on('down', () => {
setupAudioContext((array) => {
// this array is the array mentioned above, in the three.js example they do something like creating
// a texture from this input and providing that texture to the shader uniform. I tried different things but
// nothing worked :(
//
// I tried using this.shader.setChannel0 and this.shader.setUniform but nothing seems to work as well.
});
});
}
}
I've been trying to make this work for a while, but couldn't get anything :(
For a possible solution without shaders, by using only using phaser and javascript could look like this (really no shader, but I'm also really interested in how a shader version, would look like).
In this demo I'm using data from an audio file. So that it works for your UseCase, you just would have to plug the microphone data into the data variable.
Demo:
(comments in the code, are made to highlight the main idea)
Click and wait some seconds. btw.: I added some screenshake, to give the demo more juice.
document.body.style = 'margin:0;';
var data = [];
var playing = -1;
var audioContext = new (window.AudioContext || window.webkitAudioContext)();
var analyser = audioContext.createAnalyser();
var buffer;
var source;
var url = 'https://labs.phaser.io/assets/audio/Rossini - William Tell Overture (8 Bits Version)/left.ogg'
// START Audio part for Demo
function loadAudio() {
var request = new XMLHttpRequest();
request.open('GET', url, true);
request.responseType = 'arraybuffer';
request.onload = function() {
audioContext.decodeAudioData(request.response, function(buf) {
buffer = buf;
playAudio();
});
};
request.send();
}
function playAudio() {
source = audioContext.createBufferSource();
source.buffer = buffer;
source.connect(audioContext.destination);
source.connect(analyser);
source.start(0);
}
// END Audio part for Demo
var config = {
type: Phaser.AUTO,
width: 536,
height: 183,
scene: {
create,
update
},
banner: false
};
var game = new Phaser.Game(config);
// this would be the varibale that should be updated from the audio source
var markers;
var createRandomData = true;
function create () {
// Start create Marker texture
// this could be remove if you want to load an actual image
let g = this.make.graphics({x: 0, y: 0, add: false});
g.lineStyle(10, 0xffffff);
g.beginPath();
g.moveTo(0, 0);
g.lineTo(50, 0);
g.strokePath();
g.generateTexture('marker', 30, 10);
// End create Marker texture
// Create the markers
// the repeat property sets how many markers you want to display, if you want all 1024 => that would be your value
markers = this.add.group({ key: 'marker', repeat: 50,
setXY: { x: 10, y: 10, stepX: 35 }, setOrigin: { x: 0, y: 0}});
this.add.rectangle(10, 10, 180, 20, 0).setOrigin(0);
let label = this.add.text( 10, 10, 'Click to start music', {color: 'red', fontSize:'20px', fontStyle:'bold'} )
// start and stop the playback of music
this.input.on('pointerdown', function () {
switch (playing) {
case -1:
loadAudio();
playing = 1;
label.setText('Click to stop music');
break;
case 0:
playAudio();
playing = 1;
label.setText('Click to stop music');
break;
case 1:
source.stop();
playing = 0;
label.setText('Click to start music');
break;
}
});
}
function update(){
if (markers){
// here we update the y-position of the marker in depending on the value of the data.
// ( min y = 10 and max y ~ 245)
markers.children.iterate(function (child, idx) {
child.y = 10 + (config.height - 20) / 255 * data[idx];
// you could even add some camera shake, for more effect
if(idx < 3 && data[idx] > 253){
this.cameras.main.shake(30);
}
}, this);
// if the analyser is valid and updates the data variable
// this part could some where else, I just wanted to keep the code concise
if(analyser){
var spectrums = new Uint8Array(analyser.frequencyBinCount);
analyser.getByteFrequencyData(spectrums);
// convert data to a plain array and updating the data variable
data = [].slice.call(spectrums);
}
}
}<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/phaser.js"></script>Basically this application "only" alters the Y-positions of each marker, based on the value returned from the audio array, which is loaded from the audio file.
Disclaimer: this is rough demo code, and could use some cleanup/improvement, if it should be used in production.