Develop Monitor and Scale REST APIs in Node.js
- StrongLoop docs - http://docs.strongloop.com/
- CoffeeShop example app - https://github.com/strongloop/loopback-example-coffee-shop
- Twitter - @strongloop, @strongloopHelp, @shubhrakar, @chandrikagole
Prerequisites
- Node (version >= 0.10.36)
- Python (version 0.10.27)
- MongoDB
- Start mongo and connect with mongo shell
- Git (Full install)
- Windows - Microsoft Visual Studio(Express version works too)
- Mac - Xcode
- strongloop CLI - > npm install -g strongloop
Popular Frameworks
Express
var express = require('express');
var Item = require('models').Item;
var app = express();
var itemRoute = express.Router();
itemRoute.param('itemId', function(req, res, next, id) {
Item.findById(req.params.itemId, function(err, item) {
req.item = item;
next();
});
});
itemRoute.route('/:itemId')
.get(function(req, res, next) {
res.json(req.item);
})
.put(function(req, res, next) {
req.item.set(req.body);
req.item.save(function(err, item) {
res.json(item);
});
})
.post(function(req, res, next) {
var item = new Item(req.body);
item.save(function(err, item) {
res.json(item);
});
})
.delete(function(req, res, next) {
req.item.remove(function(err) {
res.json({});
});
})
;
app.use('/api/items', itemRoute);
app.listen(8080);
Sample Code(Router using express 4.x)
Pros
-
Little learning curve, Express is nearly a standard for Node.js web application
-
Fully customizable
Cons
-
All end points need to be created manually, you end up doing a lot of the same code (or worse, start rolling your own libraries after a while)
-
Every end point needs to be tested (or at the very least I recommend that you hit the end points with HTTP consumer to make sure they are actually there and don’t throw 500s)
-
Refactoring becomes painful because everything needs to be updated everywhere
-
Doesn’t come with anything “standard”, have to figure out your own approach
RESTful API's with Restify
var restify = require('restify');
var Item = require('models').Item;
var app = restify.createServer()
app.use(function(req, res, next) {
if (req.params.itemId) {
Item.findById(req.params.itemId, function(err, item) {
req.item = item;
next();
});
}
else {
next();
}
});
app.get('/api/items/:itemId', function(req, res, next) {
res.send(200, req.item);
});
app.put('/api/items/:itemId', function(req, res, next) {
req.item.set(req.body);
req.item.save(function(err, item) {
res.send(204, item);
});
});
app.post('/api/items/:itemId', function(req, res, next) {
var item = new Item(req.body);
item.save(function(err, item) {
res.send(201, item);
});
});
app.delete('/api/items/:itemId', function(req, res, next) {
req.item.remove(function(err) {
res.send(204, {});
});
});
app.listen(8080);
Sample Code(Router using restify)
Pros
-
Automatic DTrace support for all your handlers (if you’re running on a platform that supports DTrace)
-
Doesn’t have unnecessary functionality like templating and rendering
-
Built in throttling
-
Built in SPDY support
Cons
var Hapi = require('hapi');
var Item = require('models').Item;
var server = Hapi.createServer('0.0.0.0', 8080);
server.ext('onPreHandler', function(req, next) {
if (req.params.itemId) {
Item.findById(req.params.itemId, function(err, item) {
req.item = item;
next();
});
}
else {
next();
}
});
server.route([
{
path: '/api/items/{itemId}',
method: 'GET',
config: {
handler: function(req, reply) {
reply(req.item);
}
}
},
{
path: '/api/items',
method: 'PUT',
config: {
handler: function(req, reply) {
req.item.set(req.body);
req.item.save(function(err, item) {
reply(item).code(204);
});
}
}
},
{
path: '/api/items',
method: 'POST',
config: {
handler: function(req, reply) {
var item = new Item(req.body);
item.save(function(err, item) {
reply(item).code(201);
});
}
}
},
{
path: '/api/items/{itemId}',
method: 'DELETE',
config: {
handler: function(req, reply) {
req.item.remove(function(err) {
reply({}).code(204);
});
}
}
}
]);
server.start();
Sample Code(Router using hapi)
Pros
-
Very granular control over request handling
-
Detailed API reference with support for documentation generation
Cons
var loopback = require('loopback');
var Item = require('./models').Item;
var app = module.exports = loopback();
app.model(Item);
app.use('/api', loopback.rest());
app.listen(8080);
Sample Code(Router using loopback)
Code needed
DELETE /items/{id}
GET /items
GET /items/count
GET /items/findOne
GET /items/{id}
GET /items/{id}/exists
POST /items
PUT /items
PUT /items/{id}
Endpoints automatically generated
Sample code(API explorer and REST)
var explorer = require('loopback-explorer');
app.use('/explorer', explorer(app, {basePath: '/api'}));
Code needed
API explorer automatically generated
var loopback = require('loopback');
var explorer = require('loopback-explorer');
var remoting = require('strong-remoting');
var Item = require('./models').Item;
var app = module.exports = loopback();
var rpc = remoting.create();
function echo(ping, callback) {
callback(null, ping);
}
echo.shared = true;
echo.accepts = {arg: 'ping'};
echo.returns = {arg: 'echo'};
rpc.exports.system = {
echo: echo
Code needed
$ curl "http://localhost:8080/rpc/system/echo?ping=hello"
{
"echo": "hello"
}
Endpoints automatically generated
Pros
-
Very quick RESTful API development
-
Convention over configuration
-
Built in models ready to use
-
RPC support
-
Fully configurable when needed
-
Extensive documentation
-
Fulltime team working on the project
-
Available commercial support
Cons
- Learning curve can be pretty steep because there are so many moving parts
LoopBack - An Open-Source Node.js Framework
Develop APIs from scratch with the open source Loopback.io framework.
-
Middleware API Engine
-
ORM
-
Aggregation & Mashups
-
API Explorer (Swagger)
-
Fine Grained Access Control (ACLs)
-
Data Replication
-
IsoMorphic JS and Client SDKs.
Create a new app
$ slc loopback
_-----_
| | .--------------------------.
|--(o)--| | Let's create a LoopBack |
`---------´ | application! |
( _´U`_ ) '--------------------------'
/___A___\
| ~ |
__'.___.'__
´ ` |° ´ Y `
[?] Enter a directory name where to create the project: (.)
Connect an API to a datasource
$ cd myproject
$ slc loopback:datasource
$ npm install -g loopback-connector-mongodb
Install the connector
Create models
$ slc loopback:datasource
[?] Enter the data-source name: mongodb
[?] Select the connector for mongodb: MongoDB (supported by StrongLoop)
Chandrikas-MacBook-Air:demoprep chandrikagole$ slc loopback:model
[?] Enter the model name: CoffeeShop
[?] Select the data-source to attach CoffeeShop to: mongodb (mongodb)
[?] Select model's base class: PersistedModel
[?] Expose CoffeeShop via the REST API? Yes
[?] Custom plural form (used to build REST URL):
Let's add some CoffeeShop properties now.
Enter an empty property name when done.
[?] Property name: Name
invoke loopback:property
[?] Property type: string
[?] Required? Yes
Let's add another CoffeeShop property.
Enter an empty property name when done.
[?] Property name: pos
invoke loopback:property
[?] Property type: geopoint
[?] Required? Yes
Let's add another CoffeeShop property.
Enter an empty property name when done.
[?] Property name:
$
** CoffeeShop, Review, Customer (customer is from User base class).
** Delete User from server/models-config.json
Review & Customer
slc loopback:model
[?] Enter the model name: Review
[?] Select the data-source to attach Review to: mongodb (mongodb)
[?] Select model's base class: PersistedModel
[?] Expose Review via the REST API? Yes
[?] Custom plural form (used to build REST URL):
Let's add some Review properties now.
Enter an empty property name when done.
[?] Property name: star
invoke loopback:property
[?] Property type: number
[?] Required? Yes
Let's add another Review property.
Enter an empty property name when done.
[?] Property name: review
invoke loopback:property
[?] Property type: string
[?] Required? No
Let's add another Review property.
Enter an empty property name when done.
[?] Property name:
API Explorer and Front End Integration
$ slc run
supervisor running without clustering (unsupervised)
Browse your REST API at http://0.0.0.0:3000/explorer
Web server listening at: http://0.0.0.0:3000/
API Explorer
** populate coffeeshops w/ data from coffeeshop.json
Query your data using the mongo CLI
use test
switched to db test
> show collections;
CoffeeShop
system.indexes
> db.CoffeeShop.find().limit(4)
{ "Name" : "Aesop's Tables", "pos" : { "lat" : 37.5332, "lng" : -85.7302 }, "_id" : ObjectId("543f0787f9fed2bb759fc146") }
{ "Name" : "Award Winners Cafe", "pos" : { "lat" : 41.3896, "lng" : -88.12595 }, "_id" : ObjectId("543f0787f9fed2bb759fc147") }
{ "Name" : "Acadia Cafe", "pos" : { "lat" : 44.454, "lng" : -68.04902 }, "_id" : ObjectId("543f0787f9fed2bb759fc148") }
{ "Name" : "Adams Coffee Shop", "pos" : { "lat" : 42.25639, "lng" : -71.01119 }, "_id" : ObjectId("543f0787f9fed2bb759fc149") }
>
Model Relations
### Coffee has many Reviews
$ slc loopback:relation
[?] Select the model to create the relationship from: CoffeeShop
[?] Relation type: has many
[?] Choose a model to create a relationship with: Review
[?] Enter the property name for the relation: reviews
[?] Optionally enter a custom foreign key: coffeeshop
[?] Require a through model? No
### Customer has many Reviews
Chandrikas-MacBook-Air:demoprep chandrikagole$ slc loopback:relation
[?] Select the model to create the relationship from: Customer
[?] Relation type: has many
[?] Choose a model to create a relationship with: Review
[?] Enter the property name for the relation: reviews
[?] Optionally enter a custom foreign key:
### Review belongs to a Customer
Chandrikas-MacBook-Air:demoprep chandrikagole$ slc loopback:relation
[?] Select the model to create the relationship from: Review
[?] Relation type: belongs to
[?] Choose a model to create a relationship with: Customer
[?] Enter the property name for the relation: customer
[?] Optionally enter a custom foreign key: review
### Review belongs to a CoffeeShop
Chandrikas-MacBook-Air:demoprep chandrikagole$ slc loopback:relation
[?] Select the model to create the relationship from: Review
[?] Relation type: belongs to
[?] Choose a model to create a relationship with: CoffeeShop
[?] Enter the property name for the relation: coffeeShop
[?] Optionally enter a custom foreign key: review
Fine Grained Access Control (ACL)
$ slc loopback:acl
[?] Select the model to apply the ACL entry to: Review
[?] Select the ACL scope: All methods and properties
[?] Select the access type: Write
[?] Select the role: Any unauthenticated user
[?] Select the permission to apply: Explicitly deny access
ACL we are trying to create - Only allow authenticated users to post a review
LoopBack Studio
A visual tool for creating and managing Node.js powered REST APIs
-
Visual ORM
-
Discovery
-
Migration
-
DEV - API Design & Composition
-
OPS – Provisioning, Scaling & Monitoring
API Design & Composition
Migration
mysql> show tables;
+------------------+
| Tables_in_coffee |
+------------------+
| Coffee |
+------------------+
1 row in set (0.05 sec)
mysql> describe Coffee;
+------------+--------------+------+-----+---------+----------------+
| Field | Type | Null | Key | Default | Extra |
+------------+--------------+------+-----+---------+----------------+
| id | int(11) | NO | PRI | NULL | auto_increment |
| CoffeeType | varchar(512) | YES | | NULL | |
| Quantity | int(11) | YES | | NULL | |
| Supplier | varchar(512) | YES | | NULL | |
+------------+--------------+------+-----+---------+----------------+
4 rows in set (0.33 sec)
Discovery
Profiling using Studio
Monitoring - On-premises & Hosted
Monitor application metrics on premises using the monitoring console of your choice or with StrongLoop's hosted monitoring console, StrongOps.
Performance Metrics using statsd
$ slc run --metrics statsd://ec2-54-203-118-202.us-west-2.compute.amazonaws.com:8125
Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.heap.used:43596543|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.gc.heap.used:25788963|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.heap.total:94008244|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.cpu.total:2.13926|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.cpu.system:1.95049|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.cpu.user:0.18877|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.http.connection.count:0|c
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.loop.count:18|c
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.loop.minimum:0|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.loop.maximum:3|g
7 Nov 06:29:37 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.0.loop.average:0.72222|g
Metrics on the statsd server
Object Tracking
$ slc run --cluster 2 --metrics statsd://ec2-54-203-118-202.us-west-2.compute.amazonaws.com:8125
$ slc runctl objects-start <pid>
Start Object Tracking
Start a cluster
$ slc runctl objects-stop <pid>
Stop Object Tracking
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Buffer.size:-720|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Buffer.count:-15|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.(Relocatable).size:0|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.(Relocatable).count:-1|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Object.size:-776|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Object.count:-27|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.SlowBuffer.count:-1|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.SlowBuffer.size:-32|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Timer.count:2|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Timer.size:64|g
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Array.count:-1|c
7 Nov 06:41:12 - DEBUG: demoprep.Chandrikas-MacBook-Air.local.1.object.Array.size:-32|g
Object tracking Metrics on the statsd console
CPU Profiling
$ slc run --cluster 2 --metrics statsd://ec2-54-203-118-202.us-west-2.compute.amazonaws.com:8125
$ slc runctl cpu-start <pid>
Start CPU Profiling
Start a cluster
$ slc runctl cpu-stop 1
CPU profile written to `/private/tmp/demoprep/node.1.cpuprofile`, load into Chrome Dev Tools
Stop CPU Profiling
CPU Profiling using Google Chrome tools
Heap Snapshots
$ slc run --cluster 2 --metrics statsd://ec2-54-203-118-202.us-west-2.compute.amazonaws.com:8125
$ slc runctl heap-snapshot 1
Heap written to `/private/tmp/demoprep/node.1.heapdump.heapsnapshot`, load into Chrome Dev Tools
Heap snapshot
Start a cluster
What's Cooking - DI
Hosted Monitoring
Production monitoring & analytics
Fix CPU &
Memory Hotspots
StrongLoop Controller
-
OSS Controller
-
Cluster Mgmt.
-
Hot Deploy
-
Rolling Restart
-
State Mgmt.
-
Process Mgmt.
-
Deploy
$ slc run --cluster <n>
Run an application cluster
Scale and control your application
Manage the cluster using slc runctl
- View cluster status
- Change cluster size
- HA
- Graceful start/stop cluster
- Fault isolation
$ slc debug
Run your app in the debugger
Debugging
$ rm -rf node_modules // Dont commit node_modules into master branch
$ git init .
$ git commit -a -m "Initial commit"
$ slc build
Run your app in the debugger
Build and package the application
Log Manangement
Ensure that logs contain a timestamp and process ID from the process that generated the event.
Log one event per line.
Log as much as possible within reason.
Leave log-level-based filtering to the external log processor/viewer.
Leave aggregation of logs to the supervisor process.
If logging to a file, just log to stdout or stderr and leave the file management to the
supervisor process.
Default Behavior
$ slc run -d --log=/dev/null
Clustered
$ slc run --cluster n --log <filename>
example:
$ slc run --log my-app.%w.log --cluster 2
my-app.supervisor.log - Log file for the strong supervisor process.
my-app.1.log - Log file for the first worker process.
my-app.2.log - Log file for the second worker process.
Syslog
$ slc run --syslog --cluster n
$ slc pm -l 7777
Run strongloop process manager server
Deploy and manage an application
$ slc deploy http://localhost:7777 deploy
Deploy to process manager server
What's Cooking - Mesh
Open Source mBaaS(mobile backend as a service)
Offline Sync
Push Notification
Geolocation
CoffeeShop.attachTo(oracle);
var here = new GeoPoint({lat: 10.32424, lng: 5.84978});
CoffeeShop.find( {where: {location: {near: here}}, limit:3}, function(err, nearbyShops) {
console.info(nearbyShops); // [CoffeeShop, ...]
Storage Service
Thank You !
@shubhrakar
Denver Training
By chandrikagole
Denver Training
- 1,774