Everybody dance now
with Web Audio API


Jaume Sanchez @thespite
http://www.clicktorelease.com

Audio on the browser


Flash and other plugins



HTML5 Audio Element


Hybrid approach
http://www.schillmania.com/projects/soundmanager2/

Web Audio API

(and a bit of Mozilla Audio Data API)


The Web Audio API allows developers to manipulate

and play audio assets on a Web page or application.


"Web Audio API is to HTML5 <audio>

what canvas is to <img>"


Mozilla Audio Data API tried with a

JavaScript-centric approach.



How to use


var context = new AudioContext();


Usually you'll want one AudioContext (AC) per page.

Once there's an AC, you can start
adding nodes and connecting them.

Nodes

(nodes everywhere)

Modular routing. 

Basic audio operations are performed with audio nodes.

Audio nodes are linked together to form audio routing graphs.

Audio nodes are linked via their inputs and outputs. 

This modular design allows the flexibility to create complex
audio functions with dynamic effects.

Routing graphs



Sources

We need some element to input data into the system.
(something has to make noise)


                    createBufferSource
                    createMediaElementSource
                    createMediaStreamSource
                    createOscillator

     var context = new AudioContext();
     var source = context.createBufferSource();
     source.buffer = buffer;
     source.start(); 

Destinations

Any other node, or the mixer.

The graph usually ends with context.destination

     var context = new AudioContext();
     var source = context.createBufferSource(); 
     source.buffer = buffer;
     source.connect( context.destination );
     source.start(); 

but the combinations are endless

     oscillator.connect( gain );
     gain.connect( convolver );
     convolver.connect( notchFilter );
     notchFilter.connect( context.destination );

Buffer Source

Creates an AudioBufferSourceNode interface.

                    context.createBufferSource();

                    buffer attribute: IEEE 32 bit float ranging -1 to 1
                    playbackRate attribute: AudioParam
                    loop/loopStart/loopEnd attributes
                    start() and stop() methods
                    onended event


Buffer Source demo


var context = new AudioContext();
var request = new XMLHttpRequest();
request.open( 'GET', 'sound.mp3', true );
request.responseType = 'arraybuffer';
request.onload = function() {
context.decodeAudioData( request.response, function( buffer ) {

var source = context.createBufferSource(); 
source.buffer = buffer;
source.connect( context.destination );
source.start();
}, function() {
console.log( 'error' );
} );

};
request.send();

Load Sound Demo
Load Sound reusing AudioBuffer Demo

Analyser

Creates an AnalyserNode to provide real-time
frequency and time-domain analysis information 

                    context.createAnalyser()

                    fftSize attribute: size of FFT - power of two
                    getFloatFrequencyData: copies frequency-domain data
                    getByteFrequencyData: copies frequency-domain data
                    getByteTimeDomainData: copies time-domain data



Load Sound and show FFT Demo
getUserMedia Demo

AudioParam


float value (and defaultValue)

Allows scheduling of value changes

setValueAtTime
linearRampToValueAtTime
exponentialRampToValueAtTime

setTargetAtTime

setValueCurveAtTime

cancelScheduledValues

Gain

Create a GainNode to control the volume.

                    context.createGain();

                    gain attribute: AudioParam

     var context = new AudioContext();
     var source = context.createBufferSource(); 
     source.buffer = buffer;
     var volume = context.createGain();
     volume.gain.value = .1;
     source.connect( volume );
     volume.connect( context.destination );
     source.start(); 

Delay

Creates a DelayNode to add delay.

                    context.createDelay( maxDelayTime );

                    delayTime attribute AudioParam

     var context = new AudioContext();
     var source = context.createBufferSource(); 
     source.buffer = buffer;
     var delay = context.createDelay( 100 );
     delay.delayTime.value = 3;
     
     source.connect( delay );
     delay.connect( context.destination );
     source.start(); 

Delay + Gain = Reverb

Creating a loop between a gain and a delay node
creates a reverb effect on the stream.

     var context = new AudioContext();
     var a = document.createElement( 'audio' );
     a.src = 'track.wav';
     a.loop = false;
     var audioSource = context.createMediaElementSource( a );
     var delayNode = context.createDelay( 1 );
     var gainNode = context.createGain();
     gainNode.gain.value = .5;
     delayNode.delayTime.value = .1;
     audioSource.connect( delayNode );
     delayNode.connect( gainNode );
     gainNode.connect( delayNode );
     delayNode.connect( context.destination );
     a.play();

Gain + Delay Demo

Oscillator Source

Creates an OscillatorNode.

                    context.createOscillator();

                    type attribute:
                                        SINE, SQUARE, SAWTOOTH, 
                                        TRIANGLE, CUSTOM
                    setWaveTable() method for CUSTOM type
                    frequency attribute AudioParam

     var context = new AudioContext();
     var oscillator = context.createOscillator();
     oscillator.frequency.value = 440;
     oscillator.type = 4;
     oscillator.connect(context.destination);
     oscillator.start();

Filters!

Creates a BiquadFilterNode that provides several
default stock second order filters.

                    context.createBiquadFilter();

                    type attribute
                                        LOWPASS, HIGHPASS, BANDPASS,
                                        LOWSHELF, HIGHSHELF, PEAKING,
                                        NOTCH, ALLPASS
                    frequency attribute AudioParam
                    Q attribute AudioParam
                    gain attribute AudioParam

Oscillator and Biquad Filter Demo

Convolution (euh!)

Creates a ConvolverNode for linear convolution effects.

                    context.createConvolver();

                    buffer attribute: AudioBuffer with impulse response.








Convolution Demo

Panner

Creates a PannerNode for spatialisation.

                    context.createPanner();

                    Defines:
                    - position, velocity, direction of sample
                    - distance model
                    - directional sound cone

Tied to AudioListener (context.listener) for
spatial sound and doppler shift effect.


Other


                    createChannelMerger
                    createChannelSplitter

                    createDynamicsCompressor

                    createPeriodicWave
                    createWaveShaper
                    createJavaScriptNode

All together


There's a main ambient sound loop with gain.
The camera has a sound loop with gain.

Each cube has:
- a basic sound loop with gain
- a proximity sound loop with gain
- an activation sound with gain
- a deactivation sound with gain
- a sound analyser
- a panner

All sounds are merged in a convolver, a filter, a gain
and a dynamics compressor


Awesome.

Now what?


WebRTC getUserMedia

Web MIDI API

Procedural sound generation

Real-time effects

References


The spec

Getting Started with Web Audio API | HTML5 Rocks

Mixing Positional Audio and WebGL | HTML5 Rocks

Thanks!







Questions?
Let's talk!

Jaume Sanchez @thespite
http://www.clicktorelease.com
https://github.com/spite

Barcelona Future JS 2014

By Jaume Sánchez

Barcelona Future JS 2014

  • 4,551