Taming asynchronous JavaScript

https://slides.com/eiriklv/taming-async-talk

https://github.com/eiriklv/taming-async-talk

twitter @eiriklv

The Challenge

Asynchronous JavaScript can

  • be hard to reason about
  • be hard to maintain
  • be hard to error-handle
  • turn into a complete mess if you're not disciplined and keep your head straight

3 AM mental breakdown

(working on your billion dollar idea side project)

step(result, function(err, result) {
  step(result, function(err, result) {
    step(result, function(err, result) {
      step(result, function(err, result) {
        step(result, function(err, result) {
          step(result, function(err, result) {
            step(result, function(err, result) {
              step(result, function(err, result) {
                step(result, function(err, result) {
                  step(result, function(err, result) {
                    step(result, function(err, result) {
                      step(result, function(err, result) {
                        step(result, function(err, result) {
                          step(result, function(err, result) {

                            console.log('woopwoop!');
                            
                          })
                        })
                      })
                    })
                  })
                })
              })
            })
          })
        })
      })
    })
  })
})

Example Case

We have a contrived express.js route that:

  1. Reads content from a file
  2. Processes the content in a 3-step sequence
  3. Responds with the result or an error reason

The synchronous way

The synchronous way

app.get('/', function(req, res) {  
  let inputFile = 'input.txt';

  try {
    let inputData = fs.readFileSync(inputFile);
    let processedData1 = process1Sync(inputData);
    let processedData2 = process2Sync(processedData1);
    let result = process3Sync(processedData2);
    
    res.status(200).send(result);

  } catch (err) {
    res.status(500).send(err);
  }
});

The synchronous way

(synchronous blocking function)

function process1Sync(input) {
  var start = Date.now();
  var len = 100;

  while (Date.now() < (start + len)) {}

  return input + ' - processed';
};

The synchronous way

  • familiar semantics
  • easy to reason about
  • nice error handling using try/catch
  • blocks execution = no concurrency :(

The asynchronous way

continuation passing

(nesting callbacks)

The asynchronous way

(continuation passing)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  fs.readFile(inputFile, function(err, inputData) {
    if (err) return res.status(500).send(err);

    process1(inputData, function(err, processedData1) {
      if (err) return res.status(500).send(err);

      process2(processedData1, function(err, processedData2) {
        if (err) return res.status(500).send(err);

        process3(processedData2, function(err, result) {
          if (err) return res.status(500).send(err);
          
          res.status(200).send(result);
        });
      });
    });
  });
});

The asynchronous way

(asynchronous non-blocking function)

function process1(input, callback) {
  let output = input + ' - processed';
  setTimeout(callback.bind(null, null, input), 100);
};

The asynchronous way

(continuation passing)

  • hard to read / reason about
  • repeated error handling
  • will most likely be hard to extend and maintain

The asynchronous way

named continuation passing

(separating each step of the flow)

The asynchronous way

(named continuation passing)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  fs.readFile(inputFile, onReadFile);

  function onReadFile(err, inputData) {
    if (err) return res.status(500).send(err);
    process1(inputData, onProcess1);
  }

  function onProcess1(err, processedData1) {
    if (err) return res.status(500).send(err);
    process2(processedData1, onProcess2);
  }

  function onProcess2(err, processedData2) {
    if (err) return res.status(500).send(err);
    process3(processedData2, onDone);
  }

  function onDone(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  }
});

The asynchronous way

(named continuation passing)

  • still hard to reason about
    (what is called from where?)
  • still repeating ourselves in error handling
  • big chance of growing out of hand
  • inefficient (function declarations)
  • v8 optimization issues (?)
  • no more pyramid of doom

The asynchronous way

async.js

The asynchronous way

(async.js - waterfall)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  async.waterfall([
    function(callback) {
      fs.readFile(inputFile, function(err, inputData) {
        callback(err, inputData);
      });
    },
    function(inputData, callback) {
      process1(inputData, function(err, processedData1) {
        callback(err, processedData1);
      });
    },
    function(processedData1, callback) {
      process2(processedData1, function(err, processedData2) {
        callback(err, processedData2);
      });
    },
    function(processedData2, callback) {
      process3(processedData2, function(err, processedData3) {
        callback(err, processedData3);
      });
    }
  ], function(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  });
});

The asynchronous way

(async.js - waterfall)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  async.waterfall([
    fs.readFile.bind(fs, inputFile),
    process1,
    process2,
    process3
  ], function(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  });
});

The asynchronous way

(async.js - waterfall)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  let flow = [
    fs.readFile.bind(fs, inputFile),
    process1,
    process2,
    process3
  ];

  let done = function(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  };

  async.waterfall(flow, done);
});

The asynchronous way

(async.js - composition)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  let flow = async.compose(
    process3,
    process2,
    process1,
    fs.readFile.bind(fs, inputFile)
  );

  let done = function(err, result) {
    if (err) return res.status(500).send(err);
    res.status(200).send(result);
  };

  flow(done);
});

The asynchronous way

(async.js)

  • DRY error handling
  • easier to read and reason about
  • semantically very different from the synchronous example

The asynchronous way

Promises

The asynchronous way

(ES6 Promises)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  promisify(fs.readFile.bind(fs))(inputFile)
    .then(promisify(process1))
    .then(promisify(process2))
    .then(promisify(process3))
    .then(function(result) {
      res.status(200).send(result);
    })
    .catch(function(err) {
      res.status(500).send(err);
    });
});

The asynchronous way

(converting callbacks to promises)

function promisify(fn) {
  return function() {
    let self = this;
    let args = Array.prototype.slice.call(arguments);

    return new Promise(function(resolve, reject) {
      let callback = function(err, result) {
        if (err) return reject(err);
        resolve(result);
      };

      fn.apply(self, args.concat(callback));
    });
  };
}

The asynchronous way

(bluebird Promises)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  fs.readFileAsync(inputFile)
    .then(Promise.promisify(process1))
    .then(Promise.promisify(process2))
    .then(Promise.promisify(process3))
    .then(function(result) {
      res.status(200).send(result);
    })
    .catch(function(err) {
      res.status(500).send(err);
    });
});
const fs = Promise.promisifyAll(require('fs'));
// fs.readFile -> fs.readFileAsync
// fs.writeFile -> fs.writeFileAsync
// fs.appendFile -> fs.appendFileAsync
// ...

The asynchronous way

(converting callback API's with bluebird)

The asynchronous way

(Promises)

  • semantics might feel a bit unfamiliar at first
  • need to convert callback API's to Promises
    (promisify)
  • no indentation nesting
  • error handling resembles try/catch

The asynchronous way

highland.js (Streams)

The asynchronous way

(highland.js - mapping)

app.get('/', function(req, res) {
  let inputFile = 'input.txt';

  let data = hl([inputFile]);

  data
    .flatMap(hl.wrapCallback(fs.readFile.bind(fs)))
    .flatMap(hl.wrapCallback(process1))
    .flatMap(hl.wrapCallback(process2))
    .flatMap(hl.wrapCallback(process3))
    .stopOnError(function(err) {
      res.status(500).send(err);
    })
    .apply(function(result) {
      res.status(200).send(result);
    });
});

The asynchronous way

(highland.js + async.js)

data
  .flatMap(hl.wrapCallback(
    async.compose(
      process3,
      process2,
      process1,
      fs.readFile.bind(fs)
    )
  ))
  .stopOnError(handleError)
  .apply(handleSuccess);

The asynchronous way

(highland.js)

  • no indentation nesting
  • decent error handling
  • can handle multiple values in time
  • great for doing streams-stuff (!)
  • need to convert callback API's to stream-returning ones
    (wrapCallback)
  • semantics similar to Promises

The asynchronous way

Generators (ES6)

The asynchronous way

(ES6 Generators)

function *fooGen() {
  var x = yield 'bar';
  var y = yield 'baz';
  console.log('result: ', x + y);
}

var fooGenIterator = fooGen();

fooGenIterator.next();   // { value: 'bar', done: false }
fooGenIterator.next(10); // { value: 'baz', done: false }
fooGenIterator.next(10); // { value: undefined, done: true }

> result: 23

The asynchronous way

(ES6 Generators)

app.get('/', function(req, res) {  
  run(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield fs.readFile.bind(fs, inputFile);
      let processedData1 = yield process1.bind(null, inputData);
      let processedData2 = yield process2.bind(null, processedData1);
      let result = yield process3.bind(null, processedData2);

      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

The asynchronous way

(Generator iterator runner function)

function run(fn) {
  let gen = fn();

  function next(err, res) {
    if (err) return gen.throw(err);
    let ret = gen.next(res);
    if (ret.done) return;
    ret.value(next);
  }

  next();
};

The asynchronous way

(ES6 Generators)

app.get('/', function(req, res) {  
  run(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield curry(fs.readFile.bind(fs))(inputFile);
      let processedData1 = yield curry(process1)(inputData);
      let processedData2 = yield curry(process2)(processedData1);
      let result = yield curry(process3)(processedData2);
      
      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

The asynchronous way

(simple function currying to enable chaining)

function curry(fn) {
  let args = Array.prototype.slice.call(arguments, 1);

  return (function curryFn(prevArgs) {
    return function(arg) {
      let totalArgs = prevArgs.concat(Array.prototype.slice.call(arguments));
      if (!arg) totalArgs.push(undefined);

      if (totalArgs.length < fn.length) {
        return curryFn(totalArgs);
      } else {
        return fn.apply(null, totalArgs);
      }
    };
  }(args));
};

The asynchronous way

(ES6 Generators)

app.get('/', function(req, res) {  
  run(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield promisify(fs.readFile.bind(fs))(inputFile);
      let processedData1 = yield promisify(process1)(inputData);
      let processedData2 = yield promisify(process2)(processedData1);
      let result = yield promisify(process3)(processedData2);
      
      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

The asynchronous way

(adding support for Promises in our runner function)

function run(fn) {
  let gen = fn();

  function next(err, res) {
    if (err) return gen.throw(err);
    let ret = gen.next(res);
    if (ret.done) return;

    if (typeof ret.value.then === 'function') {
      try {
        ret.value.then(function(value) {
          next(null, value);
        }, next);
      } catch (e) {
        gen.throw(e);
      }
    } else {
      try {
        ret.value(next);
      } catch (e) {
        gen.throw(e);
      }
    }
  }

  next();
}

The asynchronous way

(using a better generator iterator runner)

const co = require('co');

The asynchronous way

(ES6 Generators)

app.get('/', function(req, res) {  
  co(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield promisify(fs.readFile.bind(fs))(inputFile);
      let processedData1 = yield promisify(process1)(inputData);
      let processedData2 = yield promisify(process2)(processedData1);
      let result = yield promisify(process3)(processedData2);

      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  }).catch(function(err) {...});
});

The asynchronous way

(ES6 Generators)

(moving the conversions out of the flow)

const readFile = promisify(fs.readFile.bind(fs));
const process1 = promisify(require('./process1'));
const process2 = promisify(require('./process2'));
const process3 = promisify(require('./process3'));

The asynchronous way

(ES6 Generators + co)

app.get('/', function(req, res) {  
  co(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield readFile(inputFile);
      let processedData1 = yield process1(inputData);
      let processedData2 = yield process2(processedData1);
      let result = yield process3(processedData2);
      
      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  }).catch(function(err) {...});
});

The asynchronous way

(using better conversions)

// shim native modules with promises
const fs = require('mz/fs');

// use .promisify(..) / .promisifyAll(..)
// to shim user-libs
const Promise = require('bluebird');

The asynchronous way

(ES6 Generators + co + mz)

app.get('/', function(req, res) {  
  co(function *() {
    let inputFile = 'input.txt';

    try {
      let inputData = yield fs.readFile(inputFile);
      let processedData1 = yield process1(inputData);
      let processedData2 = yield process2(processedData1);
      let result = yield process3(processedData2);
      
      res.status(200).send(result);

    } catch (err) {
      res.status(500).send(err);
    }
  }).catch(function(err) {...});
});

The asynchronous way

(Generators)

  • need to convert callback API's to thunks/promises
  • takes some getting used to
  • no indentation nesting
  • familiar semantics that resembles synchronous code
  • use try/catch for error handling

The asynchronous way

fibrous.js (fibers)

The asynchronous way

(fibers)

app.post('/process-file', function(req, res) {
  fibrous.run(function() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = fs.sync.readFile(inputFile);
      var processedData1 = process1.sync(inputData);
      var processedData2 = process2.sync(processedData1);
      var processedData3 = process3.sync(processedData2);

      fs.sync.writeFile(outputFile, processedData3);
      
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  }, function(err) {...});
});

The asynchronous way

(fibers)

  • feels more magical than generators - not in a good way
  • doesn't work with io.js (yet)
    - fibers won't build
    - running it with --harmony as an alternative does not allow block scope when 'use strict' (annoying..)
  • no indentation nesting
  • familiar semantics that resembles synchronous code

Start using

ES7 async await

http://jakearchibald.com/2014/es7-async-functions/

CSP (Communicating Sequencial Processes)

https://github.com/ubolonton/js-csp

https://www.youtube.com/watch?v=W2DgDNQZOwo

Generators (and io.js)

http://davidwalsh.name/es6-generators

Look out for

Taming asynchronous JavaScript (talk)

By Eirik Langholm Vullum

Taming asynchronous JavaScript (talk)

Talk for Node.js Oslo Meetup

  • 1,113
Loading comments...

More from Eirik Langholm Vullum