• You will be pleased to know that our radio tower on top of Carter's Mountain won "Tower of the Month"  ;D

MongoDB

 MongoClient.connect(url, function(err, db) {
  var incidents = db.collection('incidents');
  ...
        

Scraper Pattern

var dates = _.range(0,days_ago+1).map(function(i){
  return moment().subtract(i,'days').format("YYYY-MM-DD");
});

async.eachLimit(dates,5,function(date,cb){
  var url = "http://warhammer.mcc.virginia.edu/calls.php?fdate="+date;
    request(url, function(err, resp, body){
      $ = cheerio.load(body);
      ...

• Build an array of urls you want to scrape
  
  
• Pass to async.js to do them in parallel
  
  
• request() makes the actual HTTP request
  
  
• cheerio sets up "lazy parsing" so you can use selectors etc.

why upsert?

incidents.update({
  incident_id:incident.incident_id,
  unit:incident.unit_id
},incident,{upsert:true},row_cb);

2. Geocoder

4. visualize!

d3.json("data/rids.json", function(oa){
  var d3Overlay = L.d3SvgOverlay( function(selection,projection){
    selection.selectAll('#mapID circle').data(oa)
      .enter().append("circle")
      .attr("opacity",0.7)
      .attr("r", function(d){
        var endScale = Math.pow(2, 11-m.getZoom());
        return 1.0*Math.sqrt(d.incidents)*endScale;
      })
      .attr("fill", function(d){
        return svColorScale(d.averageResponseTime);
      })
      .attr("cx",function(d){
        var point = projection.latLngToLayerPoint([d.location.latitude, d.location.longitude]);
        return point.x;
      });

  }, {zoomAnimate:true});
  d3Overlay.addTo(m);
});

4. visualize!

var svColorScale = d3.scale.linear().domain([0, 30]).range(["#66FF66","#FF6666"]);
g.selectAll("scatter-dots")
  .data(calltimesByTime)
  .enter().append("svg:circle")
      .attr("cx", function (d,i) { return x(d[0]); } )
      .attr("cy", function (d) { return y(d[2]); } )
      .attr("r", 1)
      .attr("fill", function(d){
        return svColorScale(d[2]);
      })
      .attr("opacity",0.6);