Develop Monitor and Scale REST APIs in Node.js

  • StrongLoop docs - http://docs.strongloop.com/
  • CoffeeShop example app - https://github.com/strongloop/loopback-example-coffee-shop
  • Twitter - @strongloop, @strongloopHelp, @shubhrakar, @chandrikagole

Prerequisites

 

  • Node (version >= 0.10.36)
  • Python (version 0.10.27)
  • MongoDB 
    • Start mongo and connect with mongo shell
  • Git (Full install)
  • Windows - Microsoft Visual Studio(Express version works too)
  • Mac - Xcode 
  • strongloop CLI - > npm install -g strongloop

Popular Frameworks

Express

var express = require('express');
var Item = require('models').Item;
var app = express();
var itemRoute = express.Router();
 
itemRoute.param('itemId', function(req, res, next, id) {
  Item.findById(req.params.itemId, function(err, item) {
    req.item = item;
    next();
  });
});
 
itemRoute.route('/:itemId')
  .get(function(req, res, next) {
    res.json(req.item);
  })
  .put(function(req, res, next) {
    req.item.set(req.body);
    req.item.save(function(err, item) {
      res.json(item);
    });
  })
  .post(function(req, res, next) {
    var item = new Item(req.body);
    item.save(function(err, item) {
      res.json(item);
    });
  })
  .delete(function(req, res, next) {
    req.item.remove(function(err) {
      res.json({});
    });
  })
  ;
 
app.use('/api/items', itemRoute);
app.listen(8080);

Sample Code(Router using express 4.x)

Pros

  • Little learning curve, Express is nearly a standard for Node.js web application

  • Fully customizable

Cons

 

  • All end points need to be created manually, you end up doing a lot of the same code (or worse, start rolling your own libraries after a while)

  • Every end point needs to be tested (or at the very least I recommend that you hit the end points with HTTP consumer to make sure they are actually there and don’t throw 500s)

  • Refactoring becomes painful because everything needs to be updated everywhere

  • Doesn’t come with anything “standard”, have to figure out your own approach

RESTful API's with Restify

var restify = require('restify');
var Item = require('models').Item;
var app = restify.createServer()
 
app.use(function(req, res, next) {
  if (req.params.itemId) {
    Item.findById(req.params.itemId, function(err, item) {
      req.item = item;
      next();
    });
  }
  else {
    next();
  }
});
 
app.get('/api/items/:itemId', function(req, res, next) {
  res.send(200, req.item);
});
 
app.put('/api/items/:itemId', function(req, res, next) {
  req.item.set(req.body);
  req.item.save(function(err, item) {
    res.send(204, item);
  });
});
 
app.post('/api/items/:itemId', function(req, res, next) {
  var item = new Item(req.body);
  item.save(function(err, item) {
    res.send(201, item);
  });
});
 
app.delete('/api/items/:itemId', function(req, res, next) {
  req.item.remove(function(err) {
    res.send(204, {});
  });
});
 
app.listen(8080);

Sample Code(Router using restify)

Pros

  • Automatic DTrace support for all your handlers (if you’re running on a platform that supports DTrace)

  • Doesn’t have unnecessary functionality like templating and rendering

  • Built in throttling

  • Built in SPDY support

Cons

 

  • The cons are all the same with Restify as they are with Express, lots of manual labor.

var Hapi = require('hapi');
var Item = require('models').Item;
var server = Hapi.createServer('0.0.0.0', 8080);
 
server.ext('onPreHandler', function(req, next) {
  if (req.params.itemId) {
    Item.findById(req.params.itemId, function(err, item) {
      req.item = item;
      next();
    });
  }
  else {
    next();
  }
});
 
server.route([
  {
    path: '/api/items/{itemId}',
    method: 'GET',
    config: {
      handler: function(req, reply) {
        reply(req.item);
      }
    }
  },
  {
    path: '/api/items',
    method: 'PUT',
    config: {
      handler: function(req, reply) {
        req.item.set(req.body);
        req.item.save(function(err, item) {
          reply(item).code(204);
        });
      }
    }
  },
  {
    path: '/api/items',
    method: 'POST',
    config: {
      handler: function(req, reply) {
        var item = new Item(req.body);
        item.save(function(err, item) {
          reply(item).code(201);
        });
      }
    }
  },
  {
    path: '/api/items/{itemId}',
    method: 'DELETE',
    config: {
      handler: function(req, reply) {
        req.item.remove(function(err) {
          reply({}).code(204);
        });
      }
    }
  }
]);
 
server.start();

Sample Code(Router using hapi)

Pros

  • Very granular control over request handling

  • Detailed API reference with support for documentation generation

Cons

 

  • As with Express and Restifyhapi gives you great construction blocks, but you are left to your own devices figuring out how to use them.

var loopback = require('loopback');
var Item = require('./models').Item;
var app = module.exports = loopback();
 
app.model(Item);
app.use('/api', loopback.rest());
app.listen(8080);

Sample Code(Router using loopback)

Code needed

DELETE /items/{id}
GET /items
GET /items/count
GET /items/findOne
GET /items/{id}
GET /items/{id}/exists
POST /items
PUT /items
PUT /items/{id}

Endpoints automatically generated

Sample code(API explorer and REST)

var explorer = require('loopback-explorer');
app.use('/explorer', explorer(app, {basePath: '/api'}));

Code needed

API explorer automatically generated

var loopback = require('loopback');
var explorer = require('loopback-explorer');
var remoting = require('strong-remoting');
var Item = require('./models').Item;
var app = module.exports = loopback();
var rpc = remoting.create();
 
function echo(ping, callback) {
  callback(null, ping);
}
 
echo.shared = true;
echo.accepts = {arg: 'ping'};
echo.returns = {arg: 'echo'};
 
rpc.exports.system = {
  echo: echo

Code needed

$ curl "http://localhost:8080/rpc/system/echo?ping=hello"
{
  "echo": "hello"
}

Endpoints automatically generated

Pros

  • Very quick RESTful API development

  • Convention over configuration

  • Built in models ready to use

  • RPC support

  • Fully configurable when needed

  • Extensive documentation

  • Fulltime team working on the project

  • Available commercial support

Cons

 

  • Learning curve can be pretty steep because there are so many moving parts

LoopBack - An Open-Source Node.js Framework

Develop APIs from scratch with the open source Loopback.io framework.

  • Middleware API Engine

  • ORM

  • Aggregation & Mashups

  • API Explorer (Swagger)

  • Fine Grained Access Control (ACLs)

  • Data Replication

  • IsoMorphic JS and Client SDKs. 

Create a new app

$ slc loopback

     _-----_
    |       |    .--------------------------.
    |--(o)--|    |  Let's create a LoopBack |
   `---------´   |       application!       |
    ( _´U`_ )    '--------------------------'
    /___A___\    
     |  ~  |     
   __'.___.'__   
 ´   `  |° ´ Y ` 

[?] Enter a directory name where to create the project: (.) 

Connect an API to a datasource

$ cd  myproject 
$ slc loopback:datasource
$ npm install -g loopback-connector-mongodb

Install the connector

Create models

$ slc loopback:datasource
[?] Enter the data-source name: mongodb
[?] Select the connector for mongodb: MongoDB (supported by StrongLoop)
Chandrikas-MacBook-Air:demoprep chandrikagole$ slc loopback:model
[?] Enter the model name: CoffeeShop
[?] Select the data-source to attach CoffeeShop to: mongodb (mongodb)
[?] Select model's base class: PersistedModel
[?] Expose CoffeeShop via the REST API? Yes
[?] Custom plural form (used to build REST URL): 
Let's add some CoffeeShop properties now.

Enter an empty property name when done.
[?] Property name: Name
   invoke   loopback:property
[?] Property type: string
[?] Required? Yes

Let's add another CoffeeShop property.
Enter an empty property name when done.
[?] Property name: pos
   invoke   loopback:property
[?] Property type: geopoint
[?] Required? Yes

Let's add another CoffeeShop property.
Enter an empty property name when done.
[?] Property name: 
$

** CoffeeShop, Review, Customer (customer is from User base class).

** Delete User from server/models-config.json

Review & Customer 

slc loopback:model
[?] Enter the model name: Review
[?] Select the data-source to attach Review to: mongodb (mongodb)
[?] Select model's base class: PersistedModel
[?] Expose Review via the REST API? Yes
[?] Custom plural form (used to build REST URL): 
Let's add some Review properties now.
 
Enter an empty property name when done.
[?] Property name: star
   invoke   loopback:property
[?] Property type: number
[?] Required? Yes
 
Let's add another Review property.
Enter an empty property name when done.
[?] Property name: review
   invoke   loopback:property
[?] Property type: string
[?] Required? No
 
Let's add another Review property.
Enter an empty property name when done.
[?] Property name: 

API Explorer and Front End Integration

$ slc run 
supervisor running without clustering (unsupervised)
Browse your REST API at http://0.0.0.0:3000/explorer
Web server listening at: http://0.0.0.0:3000/

API Explorer 

** populate coffeeshops w/ data from coffeeshop.json

Query your data using the mongo CLI

use test
switched to db test
> show collections;
CoffeeShop
system.indexes
> db.CoffeeShop.find().limit(4)
{ "Name" : "Aesop's Tables", "pos" : { "lat" : 37.5332, "lng" : -85.7302 }, "_id" : ObjectId("543f0787f9fed2bb759fc146") }
{ "Name" : "Award Winners Cafe", "pos" : { "lat" : 41.3896, "lng" : -88.12595 }, "_id" : ObjectId("543f0787f9fed2bb759fc147") }
{ "Name" : "Acadia Cafe", "pos" : { "lat" : 44.454, "lng" : -68.04902 }, "_id" : ObjectId("543f0787f9fed2bb759fc148") }
{ "Name" : "Adams Coffee Shop", "pos" : { "lat" : 42.25639, "lng" : -71.01119 }, "_id" : ObjectId("543f0787f9fed2bb759fc149") }
> 

Model Relations

### Coffee has many Reviews
$ slc loopback:relation
[?] Select the model to create the relationship from: CoffeeShop
[?] Relation type: has many
[?] Choose a model to create a relationship with: Review
[?] Enter the property name for the relation: reviews
[?] Optionally enter a custom foreign key: coffeeshop
[?] Require a through model? No

### Customer has many Reviews
Chandrikas-MacBook-Air:demoprep chandrikagole$ slc loopback:relation
[?] Select the model to create the relationship from: Customer
[?] Relation type: has many
[?] Choose a model to create a relationship with: Review
[?] Enter the property name for the relation: reviews
[?] Optionally enter a custom foreign key: 

### Review belongs to a Customer
Chandrikas-MacBook-Air:demoprep chandrikagole$ slc loopback:relation
[?] Select the model to create the relationship from: Review
[?] Relation type: belongs to
[?] Choose a model to create a relationship with: Customer
[?] Enter the property name for the relation: customer
[?] Optionally enter a custom foreign key: review

### Review belongs to a CoffeeShop
Chandrikas-MacBook-Air:demoprep chandrikagole$ slc loopback:relation
[?] Select the model to create the relationship from: Review
[?] Relation type: belongs to
[?] Choose a model to create a relationship with: CoffeeShop
[?] Enter the property name for the relation: coffeeShop
[?] Optionally enter a custom foreign key: review

Fine Grained Access Control (ACL)

$ slc loopback:acl
[?] Select the model to apply the ACL entry to: Review
[?] Select the ACL scope: All methods and properties
[?] Select the access type: Write
[?] Select the role: Any unauthenticated user
[?] Select the permission to apply: Explicitly deny access

ACL we are trying to create - Only allow authenticated users to post a review

LoopBack Studio

A visual tool for creating and managing Node.js powered REST APIs

  • Visual ORM

  • Discovery

  • Migration

  • DEV - API Design & Composition

  • OPS – Provisioning, Scaling & Monitoring

API Design & Composition

Migration

mysql> show tables;
+------------------+
| Tables_in_coffee |
+------------------+
| Coffee           |
+------------------+
1 row in set (0.05 sec)

mysql> describe Coffee;
+------------+--------------+------+-----+---------+----------------+
| Field      | Type         | Null | Key | Default | Extra          |
+------------+--------------+------+-----+---------+----------------+
| id         | int(11)      | NO   | PRI | NULL    | auto_increment |
| CoffeeType | varchar(512) | YES  |     | NULL    |                |
| Quantity   | int(11)      | YES  |     | NULL    |                |
| Supplier   | varchar(512) | YES  |     | NULL    |                |
+------------+--------------+------+-----+---------+----------------+
4 rows in set (0.33 sec)

Discovery

Profiling using Studio

Monitoring - On-premises & Hosted

Monitor application metrics on premises using the monitoring console of your choice or with StrongLoop's hosted monitoring console, StrongOps.  

Performance Metrics using statsd

$ slc run    --metrics statsd://ec2-54-203-118-202.us-west-2.compute.amazonaws.com:8125
 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.heap.used:43596543|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.gc.heap.used:25788963|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.heap.total:94008244|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.cpu.total:2.13926|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.cpu.system:1.95049|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.cpu.user:0.18877|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.http.connection.count:0|c
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.loop.count:18|c
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.loop.minimum:0|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.loop.maximum:3|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.loop.average:0.72222|g

Metrics on the statsd server

Object Tracking

$ slc run --cluster 2   --metrics statsd://ec2-54-203-118-202.us-west-2.compute.amazonaws.com:8125
$ slc runctl objects-start <pid>

Start Object Tracking

Start a cluster

$ slc runctl objects-stop <pid>

Stop Object Tracking

7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Buffer.size:-720|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Buffer.count:-15|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.(Relocatable).size:0|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.(Relocatable).count:-1|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Object.size:-776|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Object.count:-27|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.SlowBuffer.count:-1|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.SlowBuffer.size:-32|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Timer.count:2|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Timer.size:64|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Array.count:-1|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Array.size:-32|g

Object tracking Metrics on the statsd console

CPU Profiling

$ slc run --cluster 2   --metrics statsd://ec2-54-203-118-202.us-west-2.compute.amazonaws.com:8125
$ slc runctl cpu-start <pid>

Start CPU Profiling

Start a cluster

$ slc runctl  cpu-stop 1
CPU profile written to `/private/tmp/demoprep/node.1.cpuprofile`, load into Chrome Dev Tools

Stop CPU Profiling

CPU Profiling using Google Chrome tools

Heap Snapshots

$ slc run --cluster 2   --metrics statsd://ec2-54-203-118-202.us-west-2.compute.amazonaws.com:8125
$ slc runctl  heap-snapshot  1
Heap written to `/private/tmp/demoprep/node.1.heapdump.heapsnapshot`, load into Chrome Dev Tools

Heap snapshot

Start a cluster

Open *.heapdump files with Chrome

Developer Tools for viewing and analysis.

 

What's Cooking - DI

Hosted Monitoring

Production monitoring & analytics

Fix CPU &

Memory Hotspots

StrongLoop Controller

  • OSS Controller

  • Cluster Mgmt.

  • Hot Deploy

  • Rolling Restart

  • State Mgmt.

  • Process Mgmt.

  • Deploy

$ slc run --cluster <n>

Run an application cluster

Scale and control your application

Manage the cluster using slc runctl

  • View cluster status
  • Change cluster size
  • HA
  • ​Graceful start/stop cluster
  • Fault isolation
$ slc debug

Run your app in the debugger

Debugging

$ rm -rf node_modules // Dont commit node_modules into master branch  
$ git init .
$ git commit -a -m "Initial commit" 
$ slc build 

Run your app in the debugger

Build and package the application

Log Manangement

Ensure that logs contain a timestamp and process ID from the process that generated the event.
Log one event per line.
Log as much as possible within reason.
Leave log-level-based filtering to the external log processor/viewer.
Leave aggregation of logs to the supervisor process.
If logging to a file, just log to stdout or stderr and leave the file management to the 
supervisor process.


Default Behavior
$ slc run -d --log=/dev/null

Clustered 
$ slc run --cluster n --log <filename>

example:
$ slc run --log my-app.%w.log --cluster 2

my-app.supervisor.log - Log file for the strong supervisor process.
my-app.1.log - Log file for the first worker process.
my-app.2.log - Log file for the second worker process.

Syslog
$ slc run --syslog --cluster n
$ slc pm -l 7777

Run strongloop process manager server

Deploy and manage an application

$ slc deploy http://localhost:7777 deploy  

Deploy to process manager server

What's Cooking - Mesh

Open Source mBaaS(mobile backend as a service)

Offline Sync

Push Notification

Geolocation

CoffeeShop.attachTo(oracle);
var here = new GeoPoint({lat: 10.32424, lng: 5.84978});
CoffeeShop.find( {where: {location: {near: here}}, limit:3}, function(err, nearbyShops) {
console.info(nearbyShops); // [CoffeeShop, ...]

Storage Service

Thank You !

 

@shubhrakar

Denver Training

By chandrikagole

Denver Training

  • 1,774