Async Development


Best practices and techniques


@eschoff - @funkytek - @uhduh - @wearefractal

Consulting, training, products, professional services

contact@wearefractal.com

Open source is at github.com/wearefractal







Quick Dive

Callbacks


A callback is a function that is passed to another function as an argument

The purpose of a callback is to call back to the initiator of the task after it has finished doing something asynchronously

I/O (http, fs, db, etc.) happens in a thread pool


Application code happens in a single JS thread with an event loop


As tasks in the I/O thread pool complete, callbacks are executed in the main thread


As tasks in the main thread complete, another task is popped off and executed


Only one block of code is being executed in the main thread at any time

Event Loop

What does this accomplish?


In a typical web application most of your time is spent waiting for I/O to complete

In a blocking system, this time is spent idling

In node, this time is used to continue working

Callback Conventions


Naming: cb (short for callback)

var getUser = function(cb){
  // TODO: fetch a user from mongodb
};

Arguments: Error first, data second

getUser(function(err, user){
});

Error management: Use domains

var d = domain.create();
d.on('error', function(err){
  console.log('OMG!', err);
});

getUser(d.intercept(function(user){
  
}));

Scheduling Events


How do you tell node to execute a function asynchronously?

nextTick = Schedule before I/O tasks
setImmediate = Schedule after I/O tasks

Use setImmediate
nextTick has issues with recursion

"Callback Hell"


fs.readdir(source, function(err, files) {
  if (err) {
    console.log('Error finding files: ' + err);
  } else {
    files.forEach(function(filename, fileIndex) {
      console.log(filename);
      gm(source + filename).size(function(err, values) {
        if (err) {
          console.log('Error identifying file size: ' + err)
        } else {
          console.log(filename + ' : ' + values)
          aspect = (values.width / values.height)
          widths.forEach(function(width, widthIndex) {
            height = Math.round(width / aspect)
            console.log('resizing ' + filename + 'to ' + height + 'x' + height);
            this.resize(width, height).write(destination + 'w' + width + '_' + filename, function(err) {
              if (err) console.log('Error writing file: ' + err);
            });
          }.bind(this))
        }
      });
    });
  }
});

Debugging callback hell








Techniques


1. Keep code shallow

  • Use short returns instead of if/else
fs.readdir(source, function(err, files) {
  if (err) return console.log('Error finding files:', err);
  files.forEach(function(filename, fileIndex) {
    console.log(filename);
    gm(source + filename).size(function(err, values) {
      if (err) return console.log('Error identifying file size:', err)
      console.log(filename, ':', values)
      aspect = (values.width / values.height)
      widths.forEach(function(width, widthIndex) {
        height = Math.round(width / aspect)
        console.log('resizing', filename, 'to', width+'x'+height);
        this.resize(width, height).write(destination + 'w' + width + '_' + filename, function(err) {
          if (err) console.log('Error writing file: ' + err);
        });
      }.bind(this));
      });
    });
});

2. Modularize

  • Use named functions instead of nesting
  • Pass all errors upstream
function getImageRatio(image, cb) {
  image.size(function (err, values) {
    if (err) return cb(err);
    var aspect = (values.width / values.height);
    cb(null, aspect);
  });
}


function resizeFile(filename) {
  var image = gm(filename);

  getImageRatio(image, function (err, aspect) {
    if (err) return cb(err);

    widths.forEach(function (width, widthIndex) {
      var height = Math.round(width / aspect);
      var outputFile = filename + '-' + width;

      image.resize(width, height).write(outputFile);
    });

  });
}


fs.readdir(source, function (err, files) {
  if (err) return console.log('Error finding files:', err);
  files.forEach(resizeFile);
});

3. Use async

  • Makes iteration easy
var async = require('async');

function getImageRatio(image, cb) {
  image.size(function (err, val) {
    if (err) return cb(err);
    var aspect = (val.width / val.height);
    cb(null, aspect);
  });
}

function resizeImageWithAspect(filename, image, aspect, width, cb) {
  var height = Math.round(width / aspect);
  var outputFile = filename + '-' + width;
  image.resize(width, height).write(outputFile, cb);
}

function resizeImage(filename, cb) {
  var image = gm(filename);
  getImageRatio(image, function (err, aspect) {
    if (err) return cb(err);

    var fn = resizeImageWithAspect.bind(null, filename, image, aspect);
    async.forEach(widths, fn, cb);
  });
}


fs.readdir(source, function (err, files) {
  if (err) return console.log('Error finding files:', err);

  async.forEach(files, resizeImage, function (err) {
    if (err) return console.log('Error resizing images:', err);
    console.log('Finished!');
  });
});







async.js


What is it


async.map(['file1','file2','file3'], fs.stat, function(err, results){
    // results is now an array of stats for each file
});

async.filter(['file1','file2','file3'], fs.exists, function(results){
    // results now equals an array of the existing files
});

async.parallel([
    function(){ ... },
    function(){ ... }
], callback);

async.series([
    function(){ ... },
    function(){ ... }
]);

Series


async.each([1,2,3], calc, function(err){});

Will schedule all tasks immediately


async.eachSeries([1,2,3], calc, function(err){});

Will schedule each task one by one as they complete







Let's use it


Workshop


git clone https://github.com/wearefractal/async-node-workshop
cd async-node-workshop
npm install

Problem 1


// gets a user from the db
// and calls back with (err, user) signature
var getUser = function(name, cb) {
  // simulate async
  setTimeout(function(){
    cb(null, {name: name});
  }, 100);
};

var person = "Mary";

// Get marys user object and console.log it

Problem 1 Solution


// gets a user from the db
// and calls back with (err, user) signature
var getUser = function(name, cb) {
  // simulate async
  setTimeout(function(){
    cb(null, {name: name});
  }, 100);
};

var person = "Mary";

// Get marys user object and console.log it
getUser(person, function(err, user){
  console.log(user);
});

Problem 2


var async = require('async');
var fs = require('fs');

// gets a user from the db
// and calls back with (err, user) signature
var saveUser = function(name, cb) {
  var fileName = name + ".txt";
  var content = name + " is cool!";
  fs.writeFile(fileName, content, cb);
};


var people = ["Mary", "Todd", "Mike"];

// For each person in the people array
// Call saveUser
// Use async.each

async.each

Problem 2 Solution


var async = require('async');
var fs = require('fs');

// gets a user from the db
// and calls back with (err, user) signature
var saveUser = function(name, cb) {
  var fileName = name + ".txt";
  var content = name + " is cool!";
  fs.writeFile(fileName, content, cb);
};


var people = ["Mary", "Todd", "Mike"];

// For each person in the people array
// Call saveUser
// Use async.each

async.each(people, saveUser, function(err) {

});

Problem 3


var async = require('async');

// gets a user from the db
// and calls back with (err, user) signature
var getUser = function(name, cb) {
  // simulate async
  setTimeout(function(){
    cb(null, {name: name});
  }, 100);
};


var people = ["Mary", "Todd", "Mike"];

// Map the people array to getUser
// Using async.map
// Log the new array when you are done

async.map

Problem 3 Solution


var async = require('async');

// gets a user from the db
// and calls back with (err, user) signature
var getUser = function(name, cb) {
  // simulate async
  setTimeout(function(){
    cb(null, {name: name});
  }, 100);
};


var people = ["Mary", "Todd", "Mike"];

// Map the people array to getUser
// Using async.map
// Log the new array when you are done
async.map(people, getUser, function(err, users) {
  console.log(users);
});

Problem 4


var async = require('async');
var redis = require('redis');

var client = redis.createClient();

// gets a user from the db
var getUser = function(name, cb) {
  // simulate async
  setTimeout(function(){
    var val = name + " is cool!";
    cb(null, {name: name, val: val});
  }, 100);
};

// write user into redis
// and calls back with (err, user) signature
var writeUser = function(user, cb) {
  console.log("Writing", user.name);
  client.set(user.name, user.val, cb);
};

var people = ["Mary", "Todd", "Mike"];

// Map the people array to getUser
// Using async.map and async.each

Problem 4 Solution


var async = require('async');
var redis = require('redis');

var client = redis.createClient();

// gets a user from the db
var getUser = function(name, cb) {
  // simulate async
  setTimeout(function(){
    var val = name + " is cool!";
    cb(null, {name: name, val: val});
  }, 100);
};

// write user into redis
// and calls back with (err, user) signature
var writeUser = function(user, cb) {
  console.log("Writing", user.name);
  client.set(user.name, user.val, cb);
};

var people = ["Mary", "Todd", "Mike"];

// Map the people array to getUser
// Using async.map and async.each
async.map(people, getUser, function(err, users){
  async.each(users, writeUser, function(err){
    console.log('Done!');
  });
});

Keeping async flat


Before:

async.map(people, getUser, function(err, users){
  async.each(users, writeUser, function(err){
    console.log('Done!');
  });
});

After:

async.waterfall([
  function(done){
    async.map(people, getUser, done);
  },
  function(users, done){
    async.each(users, writeUser, done);
  }
], function(err){
  console.log('Done!');
});
  • Use async.apply to avoid function wrappers
  • Use async.parallel, async.auto, and async.waterfall to avoid nesting

Promises


Parsing Twitter:

getTweetsFor("domenic") // promise-returning async function
    .then(function (tweets) {
        var shortUrls = parseTweetsForUrls(tweets);
        var mostRecentShortUrl = shortUrls[0];
        return expandUrlUsingTwitterApi(mostRecentShortUrl); // promise-returning async function
    })
    .then(doHttpRequest) // promise-returning async function
    .then(
        function (responseBody) {
            console.log("Most recent link text:", responseBody);
        },
        function (error) {
            console.error("Error with the twitterverse:", error);
        }
    );

Streams


Parsing JSON:

inputStream
  .pipe(JSONStream.stringify())
  .pipe(outputStream);






Questions?


Async Development in Node

By Eric Schoffstall

Async Development in Node

Best practices, musings, etc.

  • 3,787
Loading comments...

More from Eric Schoffstall