async.parallel([
    function(){ ... },
    function(){ ... }
], callback);

async.series([
    function(){ ... },
    function(){ ... }
]);

async.map(['file1','file2','file3'], fs.stat, function(err, results){
    // results is now an array of stats for each file
});

async.parallel([
    function(callback){
        callback(null, 'one');
    },
    function(callback){
        setTimeout(function(){
            callback(null, 'two');
        }, 100);
    }
],
// optional callback
function(err, results){
    // results = ['one','two']
});

• it makes data dependancies readable and explicit
• sane default error handling behavior (bail out on any error)
• fewer curly braces and no long lists of named functions
• easy to switch between parallel and series for testing/debugging

• uses a "forgiving" HTML parser: many real websites don't have valid HTML...
  
• does not try to parse the whole site immediately, but waits for you to call specific selectors

var request = require('request');
var cheerio = require('cheerio');

request("http://www.meetup.com/CVJS/",function(err,r,body){
  $ = cheerio.load(body);

  var links = $(".event-item h3 a").map(function(idx,elem){
    return {
      text: $(elem).text().trim(),
      href: $(elem).attr("href")
    };
  }).toArray();

  links.forEach(function(l){
    console.log(l.href, l.text);
  });
});

// scrape.js
async.each(_.range(30),function(page,indexcb){
  request.get(domain+"/course-list/allcourses?page="+page,function(err,r,body){
    var$= cheerio.load(body);
    var links = _.uniq($(".course-title a").map(function(i,elem){
return elem.attribs.href;
}));
    async.each(links, function(link,cb){
      request.get(link,function(err,r,body){
        var$= cheerio.load(body);
        ...

series.js and scrape_callbackcounter.js

• Lightweight, fast, uses almost no memory, blah blah blah
  
  
• Runs ASCII over TCP (No HTTP overhead)
  
  
• Very low-level. Jobs are text, you define your own rules.
  
• (Personally we use JSON)
  
  
• Very easy to setup, many clients in many languages
  
  
• Persistent, if you want.
• We've been running one instance since September 2013 and it has never stopped (despite disk fulls, network downtime, etc)
  
  
• Kind of like memcached for work-queues

• We use named "tubes" for each type of job

list-tubes
OK 105
---
- default
- copySpin
- createSpin
- editSpin
- exportVideo
- pdfsupply_export
- updateMetadata
- udaverea_export

• each tube keeps a list of jobs in each state
• typical flow for a job is ready -> reserved -> *poof*

stats-tube editSpin
OK 273
---
name: editSpin
current-jobs-urgent: 0
current-jobs-ready: 1
current-jobs-reserved: 4
current-jobs-delayed: 0
current-jobs-buried: 0
total-jobs: 55137
current-using: 3
current-watching: 1
current-waiting: 1
cmd-delete: 35061
cmd-pause-tube: 0
pause: 0
pause-time-left: 0

• Each job has a timeout. If the worker that reserved the job does not checkin with a touch command before the timeout, it will be automatically moved back to the ready queue.
  
  
• Jobs can also be buried (for example if a job is repeatedly failing), to be manually examined later.
  
  
• Jobs can also be added to the queue with a delay (good for "backing off" a congested service)

function reserveWork(){
  async.auto({
    reservation: function(cb){
      beanstalkd.reserve(function(err, jobid, payload){
        cb(null,{ payload: payload, jobid: jobid });
      });
    },
    work: ['reservation',function(cb,r){
      var payload = JSON.parse(r.reservation.payload); // work work work...
      cb(null, new_payload);
    }],
    done: ['work',function(cb,r){
      beanstalkd.use("editor",function(err,tube){// next tube
        beanstalkd.put(1024, 0, 300, r.work, function(err,new_jobid){
          beanstalkd.destroy(jobid, cb);
        });
      });
    }]
  }, reserveWork);
}

• restarts jobs when they die
  
  
• has sane default behavior, easy to configure however you want
  
  
• can also be used programmatically from inside node.js