Taming asynchronous JavaScript

https://slides.com/eiriklv/taming-async-talk

https://github.com/eiriklv/taming-async-talk

twitter @eiriklv

The Challenge

Asynchronous JavaScript can

be hard to reason about
be hard to maintain
be hard to error-handle
turn into a complete mess if you're not disciplined and keep your head straight

3 AM mental breakdown

(working on your billion dollar idea side project)

step(result, function(err, result) {
  step(result, function(err, result) {
    step(result, function(err, result) {
      step(result, function(err, result) {
        step(result, function(err, result) {
          step(result, function(err, result) {
            step(result, function(err, result) {
              step(result, function(err, result) {
                step(result, function(err, result) {
                  step(result, function(err, result) {
                    step(result, function(err, result) {
                      step(result, function(err, result) {
                        step(result, function(err, result) {
                          step(result, function(err, result) {

                            console.log('woopwoop!');
                            
                          })
                        })
                      })
                    })
                  })
                })
              })
            })
          })
        })
      })
    })
  })
})

Example Case

We have a contrived express.js route that:

Reads content from a file
Processes the content in a 3-step sequence
Responds with the result or an error reason

The synchronous way

app.get('/', function(req, res) {  
  let inputFile = 'input.txt';

  try {
    let inputData = fs.readFileSync(inputFile);
    let processedData1 = process1Sync(inputData);
    let processedData2 = process2Sync(processedData1);
    let result = process3Sync(processedData2);
    
    res.status(200).send(result);

  } catch (err) {
    res.status(500).send(err);
  }
});

The synchronous way

(synchronous blocking function)

function process1Sync(input) {
  var start = Date.now();
  var len = 100;

  while (Date.now() < (start + len)) {}

  return input + ' - processed';
};

The synchronous way

familiar semantics
easy to reason about
nice error handling using try/catch
blocks execution = no concurrency :(

The asynchronous way

continuation passing

(nesting callbacks)

The asynchronous way

(continuation passing)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  fs.readFile(inputFile, function(err, inputData) {
    if (err) return res.status(500).send(err);

    process1(inputData, function(err, processedData1) {
      if (err) return res.status(500).send(err);

      process2(processedData1, function(err, processedData2) {
        if (err) return res.status(500).send(err);

        process3(processedData2, function(err, result) {
          if (err) return res.status(500).send(err);
          
          res.status(200).send(result);
        });
      });
    });
  });
});

The asynchronous way

(asynchronous non-blocking function)

function process1(input, callback) {
  let output = input + ' - processed';
  setTimeout(callback.bind(null, null, input), 100);
};

The asynchronous way

(continuation passing)

hard to read / reason about
repeated error handling
will most likely be hard to extend and maintain

The asynchronous way

named continuation passing

(separating each step of the flow)

The asynchronous way

(named continuation passing)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  fs.readFile(inputFile, onReadFile);

  function onReadFile(err, inputData) {
    if (err) return res.status(500).send(err);
    process1(inputData, onProcess1);
  }

  function onProcess1(err, processedData1) {
    if (err) return res.status(500).send(err);
    process2(processedData1, onProcess2);
  }

  function onProcess2(err, processedData2) {
    if (err) return res.status(500).send(err);
    process3(processedData2, onDone);
  }

  function onDone(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  }
});

The asynchronous way

(named continuation passing)

still hard to reason about
(what is called from where?)
still repeating ourselves in error handling
big chance of growing out of hand
inefficient (function declarations)
v8 optimization issues (?)

no more pyramid of doom

The asynchronous way

async.js

The asynchronous way

(async.js - waterfall)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  async.waterfall([
    function(callback) {
      fs.readFile(inputFile, function(err, inputData) {
        callback(err, inputData);
      });
    },
    function(inputData, callback) {
      process1(inputData, function(err, processedData1) {
        callback(err, processedData1);
      });
    },
    function(processedData1, callback) {
      process2(processedData1, function(err, processedData2) {
        callback(err, processedData2);
      });
    },
    function(processedData2, callback) {
      process3(processedData2, function(err, processedData3) {
        callback(err, processedData3);
      });
    }
  ], function(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  });
});

The asynchronous way

(async.js - waterfall)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  async.waterfall([
    fs.readFile.bind(fs, inputFile),
    process1,
    process2,
    process3
  ], function(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  });
});

The asynchronous way

(async.js - waterfall)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  let flow = [
    fs.readFile.bind(fs, inputFile),
    process1,
    process2,
    process3
  ];

  let done = function(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  };

  async.waterfall(flow, done);
});

The asynchronous way

(async.js - composition)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  let flow = async.compose(
    process3,
    process2,
    process1,
    fs.readFile.bind(fs, inputFile)
  );

  let done = function(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  };

  flow(done);
});

The asynchronous way

(async.js)

DRY error handling
easier to read and reason about

semantically very different from the synchronous example

The asynchronous way

Promises

The asynchronous way

(ES6 Promises)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  promisify(fs.readFile.bind(fs))(inputFile)
    .then(promisify(process1))
    .then(promisify(process2))
    .then(promisify(process3))
    .then(function(result) {
      res.status(200).send(result);
    })
    .catch(function(err) {
      res.status(500).send(err);
    });
});

The asynchronous way

(converting callbacks to promises)

function promisify(fn) {
  return function() {
    let self = this;
    let args = Array.prototype.slice.call(arguments);

    return new Promise(function(resolve, reject) {
      let callback = function(err, result) {
        if (err) return reject(err);
        resolve(result);
      };

      fn.apply(self, args.concat(callback));
    });
  };
}

The asynchronous way

(bluebird Promises)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  fs.readFileAsync(inputFile)
    .then(Promise.promisify(process1))
    .then(Promise.promisify(process2))
    .then(Promise.promisify(process3))
    .then(function(result) {
      res.status(200).send(result);
    })
    .catch(function(err) {
      res.status(500).send(err);
    });
});

const fs = Promise.promisifyAll(require('fs'));
// fs.readFile -> fs.readFileAsync
// fs.writeFile -> fs.writeFileAsync
// fs.appendFile -> fs.appendFileAsync
// ...

The asynchronous way

(converting callback API's with bluebird)

The asynchronous way

(Promises)

semantics might feel a bit unfamiliar at first
need to convert callback API's to Promises
(promisify)

no indentation nesting
error handling resembles try/catch

The asynchronous way

highland.js (Streams)

The asynchronous way

(highland.js - mapping)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  let data = hl([inputFile]);

  data
    .flatMap(hl.wrapCallback(fs.readFile.bind(fs)))
    .flatMap(hl.wrapCallback(process1))
    .flatMap(hl.wrapCallback(process2))
    .flatMap(hl.wrapCallback(process3))
    .stopOnError(function(err) {
      res.status(500).send(err);
    })
    .apply(function(result) {
      res.status(200).send(result);
    });
});

The asynchronous way

(highland.js + async.js)

data
  .flatMap(hl.wrapCallback(
    async.compose(
      process3,
      process2,
      process1,
      fs.readFile.bind(fs)
    )
  ))
  .stopOnError(handleError)
  .apply(handleSuccess);

The asynchronous way

(highland.js)

no indentation nesting
decent error handling
can handle multiple values in time
great for doing streams-stuff (!)

need to convert callback API's to stream-returning ones
(wrapCallback)

semantics similar to Promises

The asynchronous way

Generators (ES6)

The asynchronous way

(ES6 Generators)

function *fooGen() {
  var x = yield 'bar';
  var y = yield 'baz';
  console.log('result: ', x + y);
}

var fooGenIterator = fooGen();

fooGenIterator.next();   // { value: 'bar', done: false }
fooGenIterator.next(10); // { value: 'baz', done: false }
fooGenIterator.next(10); // { value: undefined, done: true }

> result: 23

The asynchronous way

(ES6 Generators)

app.get('/', function(req, res) {  
  run(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield fs.readFile.bind(fs, inputFile);
      let processedData1 = yield process1.bind(null, inputData);
      let processedData2 = yield process2.bind(null, processedData1);
      let result = yield process3.bind(null, processedData2);

      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

The asynchronous way

(Generator iterator runner function)

function run(fn) {
  let gen = fn();

  function next(err, res) {
    if (err) return gen.throw(err);
    let ret = gen.next(res);
    if (ret.done) return;
    ret.value(next);
  }

  next();
};

The asynchronous way

(ES6 Generators)

app.get('/', function(req, res) {  
  run(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield curry(fs.readFile.bind(fs))(inputFile);
      let processedData1 = yield curry(process1)(inputData);
      let processedData2 = yield curry(process2)(processedData1);
      let result = yield curry(process3)(processedData2);
      
      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

The asynchronous way

(simple function currying to enable chaining)

function curry(fn) {
  let args = Array.prototype.slice.call(arguments, 1);

  return (function curryFn(prevArgs) {
    return function(arg) {
      let totalArgs = prevArgs.concat(Array.prototype.slice.call(arguments));
      if (!arg) totalArgs.push(undefined);

      if (totalArgs.length < fn.length) {
        return curryFn(totalArgs);
      } else {
        return fn.apply(null, totalArgs);
      }
    };
  }(args));
};

The asynchronous way

(ES6 Generators)

app.get('/', function(req, res) {  
  run(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield promisify(fs.readFile.bind(fs))(inputFile);
      let processedData1 = yield promisify(process1)(inputData);
      let processedData2 = yield promisify(process2)(processedData1);
      let result = yield promisify(process3)(processedData2);
      
      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

The asynchronous way

(adding support for Promises in our runner function)

function run(fn) {
  let gen = fn();

  function next(err, res) {
    if (err) return gen.throw(err);
    let ret = gen.next(res);
    if (ret.done) return;

    if (typeof ret.value.then === 'function') {
      try {
        ret.value.then(function(value) {
          next(null, value);
        }, next);
      } catch (e) {
        gen.throw(e);
      }
    } else {
      try {
        ret.value(next);
      } catch (e) {
        gen.throw(e);
      }
    }
  }

  next();
}

The asynchronous way

(using a better generator iterator runner)

const co = require('co');

The asynchronous way

(ES6 Generators)

app.get('/', function(req, res) {  
  co(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield promisify(fs.readFile.bind(fs))(inputFile);
      let processedData1 = yield promisify(process1)(inputData);
      let processedData2 = yield promisify(process2)(processedData1);
      let result = yield promisify(process3)(processedData2);

      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  }).catch(function(err) {...});
});

The asynchronous way

(ES6 Generators)

(moving the conversions out of the flow)

const readFile = promisify(fs.readFile.bind(fs));
const process1 = promisify(require('./process1'));
const process2 = promisify(require('./process2'));
const process3 = promisify(require('./process3'));

The asynchronous way

(ES6 Generators + co)

app.get('/', function(req, res) {  
  co(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield readFile(inputFile);
      let processedData1 = yield process1(inputData);
      let processedData2 = yield process2(processedData1);
      let result = yield process3(processedData2);
      
      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  }).catch(function(err) {...});
});

The asynchronous way

(using better conversions)

// shim native modules with promises
const fs = require('mz/fs');

// use .promisify(..) / .promisifyAll(..)
// to shim user-libs
const Promise = require('bluebird');

The asynchronous way

(ES6 Generators + co + mz)

app.get('/', function(req, res) {  
  co(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield fs.readFile(inputFile);
      let processedData1 = yield process1(inputData);
      let processedData2 = yield process2(processedData1);
      let result = yield process3(processedData2);
      
      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  }).catch(function(err) {...});
});

The asynchronous way

(Generators)

need to convert callback API's to thunks/promises
takes some getting used to

no indentation nesting
familiar semantics that resembles synchronous code
use try/catch for error handling

The asynchronous way

fibrous.js (fibers)

The asynchronous way

(fibers)

app.post('/process-file', function(req, res) {
  fibrous.run(function() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = fs.sync.readFile(inputFile);
      var processedData1 = process1.sync(inputData);
      var processedData2 = process2.sync(processedData1);
      var processedData3 = process3.sync(processedData2);

      fs.sync.writeFile(outputFile, processedData3);
      
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  }, function(err) {...});
});

The asynchronous way

(fibers)

feels more magical than generators - not in a good way
doesn't work with io.js (yet)
- fibers won't build
- running it with --harmony as an alternative does not allow block scope when 'use strict' (annoying..)

no indentation nesting
familiar semantics that resembles synchronous code

Start using

ES7 async await

http://jakearchibald.com/2014/es7-async-functions/

CSP (Communicating Sequencial Processes)

https://github.com/ubolonton/js-csp

https://www.youtube.com/watch?v=W2DgDNQZOwo

Generators (and io.js)

http://davidwalsh.name/es6-generators

Look out for

Taming asynchronous JavaScript (talk)

By Eirik Langholm Vullum

Taming asynchronous JavaScript (talk)

Talk for Node.js Oslo Meetup

3,515

Eirik Langholm Vullum PRO

JavaScript fanatic that loves node.js and React.