Stefan Kutko, VP Engineering
Electronifie
Polyglot Developer
8 years developing financial trading systems
Eager adopter of new technology
The opportunity to build a Trading System written entirely in Node.js
(no problem)
Express
WebApp
MongoDB
Users
Accounts
Orders
Trades
Treasuries
Bonds
Instead of this...
REST API
Express
WebApp
AccountSvc
MarketSvc
RefDataSvc
Users
Accounts
Orders
Trades
Bonds
Treasuries
DB
DB
DB
REST API
?
We can use HTTP + REST but...
More Services == More Infrastructure + Config
Also...
How do other services get notified of changes?
// worker.js
var events = require("events");
var util = require('util');
function Worker () {
events.EventEmitter.call(this);
}
util.inherits(Worker, events.EventEmitter);
Worker.prototype.doStuff = function() {
// ...
this.emit('done');
};
module.exports.Worker = Worker;
// service.js
var Worker = require('./worker');
var worker = new Worker();
worker.on('done', function () {
console.log('work done');
});
worker.doStuff();
Works great intra-process, but what about inter-process?
Send/Listen
Pub/Sub
Worker Queues
With Durable or Transient messaging!
// process1.js
var bus = require('servicebus').bus({
url: process.env.RABBITMQ_URL
});
bus.send('order.create', {
userId: 1,
bondId: 000000000,
side: 'buy',
qty: 1000,
limit: 99.95
}, { ack: true });
// process2.js
var bus = require('servicebus');
var orderManager = require('./orderManager');
bus.listen('order.create', function(msg) {
console.log(msg);
orderManager.create(msg.data, function (err) {
if(err)
msg.handle.reject();
else
msg.handle.ack();
});
});
// processManager.js
var bus = require('servicebus').bus({
url: process.env.RABBITMQ_URL
});
bus.publish('broadcast', {
command: 'heartbeat'
});
bus.listen('process.manager', function(msg) {
console.log(
msg.data.id +
" is status: " +
msg.data.status
);
});
// process1.js
var bus = require('servicebus').bus({
url: process.env.RABBITMQ_URL
});
bus.subscribe('broadcast', function(msg) {
if(msg.data.command == 'heartbeat') {
bus.send('process.manager', {
id: 'process1',
status: 'online'
});
}
});
// process2.js
var bus = require('servicebus').bus({
url: process.env.RABBITMQ_URL
});
bus.subscribe('broadcast', function(msg) {
if(msg.data.command == 'heartbeat') {
bus.send('process.manager', {
id: 'process2',
status: 'online'
});
}
});
MicroSvc
DB
Command in
Persistence
Event in
Command out
Event out
Query API
(in node.js)
Best explained when compared to a CRUD architecture
(Create, Read, Update, Delete)
or
REST
The Domain:
{
userId: "trader1",
bondId: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95
}
HTTP POST /orders
Express
MongoDB
order.save()
{
_id: 1,
userId: "trader1",
bondId: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95,
status: "open",
filled: 0,
remaining: 500000
}
1 Database Save
Get back 3 additional
server-side managed fields
Update Order 1
{
_id: 1,
userId: "trader1",
bondId: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95,
status: "open",
filled: 200000,
remaining: 300000
}
{
_id: 2,
userId: "trader2",
bondId: "92343VBC7",
side: "sell",
quantity: 200000,
limit: 99.95,
status: "completed",
filled: 200000,
remaining: 0
}
Insert Order 2
{
_id: 1,
buyOrderId: 1,
sellOrderId: 2,
quantity: 200000,
price: 99.95
}
Insert Trade 1
= 1 Find + 3 Write Operations to DB
Find "buy" orders for "92343VBC7"...
Found: _id=1
_id, user, cusip, side, qty, limit, filled, remaining, status
1, trader1, "92343VBC7", buy, 500000, 99.95, 200000, 300000, "open"
2, trader2, "92343VBC7", sell, 200000, 99.95, 200000, 0, "completed"
Orders
Trades
_id, cusip, buyOrderId, sellOrderId, qty, price
1, "92343VBC7", 1, 2, 200000, 99.95
_id, cusip, user, side, qty, limit, filled, remaining, status
1, "92343VBC7", trader1, buy, 500000, 99.98, 200000, 300000, "open"
2, "92343VBC7", trader2, sell, 200000, 99.95, 200000, 0, "completed"
Orders
Trades
_id, cusip, buyOrderId, sellOrderId, qty, price
1, "92343VBC7", 1, 2, 200000, 99.95
Trader1 modifies buy order limit to $99.96
then to $99.97....
then to $99.98....
_id, user, side, qty, limit, ..., createdAt, updatedAt
1, trader1, buy, 500000, 99.98, ..., "2014-11-14 11:34:11", "2014-11-14 12:11:05"
2, trader2, sell, 200000, 99.95, ..., "2014-11-14 11:39:11", "2014-11-14 11:39:11"
Orders
Trader1 modifies buy order limit to $99.96
then to $99.97....
then to $99.98....
(i.e. want to monitor trader behavior...)
We know when order is created...
and when last updated to current state...
but miss interim modification to $99.97...
and original limit price.
_id, user, side, qty, limit, ..., createdAt, updatedAt
1, trader1, buy, 500000, 99.98, ..., "2014-11-14 11:34:11", "2014-11-14 12:11:05"
2, trader2, sell, 200000, 99.95, ..., "2014-11-14 11:39:11", "2014-11-14 11:39:11"
Orders
Trader1 modifies buy order limit to $99.96
then to $99.97....
then to $99.98....
Have full history of changes...
But what changed? Have to infer from prev record...
Also o(n) updates + o(n) writes on EVERY transaction.
_id, version, user, side, qty, limit, ..., createdAt, updatedAt
1, 1, trader1, buy, 500000, 99.95, ..., "2014-11-14 11:34:11", "2014-11-14 11:34:11"
1, 2, trader1, buy, 500000, 99.96, ..., "2014-11-14 11:34:11", "2014-11-14 11:37:08"
1, 3, trader1, buy, 500000, 99.97, ..., "2014-11-14 11:34:11", "2014-11-14 11:54:32"
1, 4, trader1, buy, 500000, 99.98, ..., "2014-11-14 11:34:11", "2014-11-14 12:11:05"
OrderHistory
Maintain an immutable Event Log
that represents a single-source of truth...
PERFECT audit history,
O(1) db APPEND operation on EVERY transaction,
Current state of system held in memory only.
_id, event, createdAt, payload,
1, "createOrder", "2014-11-14 11:34:11", { _id: 1, cusip: "92343VBC7", user: trader1, side: "buy", qty: 500000, limit: 99.95, ... }
2, "replaceOrder", "2014-11-14 11:37:08", { _id: 1, limit: 99.96 }
3, "replaceOrder", "2014-11-14 11:54:32", { _id: 1, limit: 99.97, updatedBy: algo1 }
4, "replaceOrder", "2014-11-14 12:11:05", { _id: 1, limit: 99.98 }
MarketEvents
marketId, version, name, createdAt, payload,
"92343VBC7", 1, "createOrder", "2014-11-14 11:34:11", { _id: 1, user: trader1, side: "buy", qty: 500000, limit: 99.95, ... }
"92343VBC7", 2, "replaceOrder", "2014-11-14 11:37:08", { _id: 1, limit: 99.96 }
"92343VBC7", 3, "replaceOrder", "2014-11-14 11:54:32", { _id: 1, limit: 99.97, updatedBy: algo1 }
"92343VBC7", 4, "replaceOrder", "2014-11-14 12:11:05", { _id: 1, limit: 99.98 }
MarketEvents
A: Replay events on startup to rebuild current system state.
Market.prototype.init = function(marketId) {
var self = this;
Events.find({ marketId: marketId }, {orderBy: {version: 1}}).forEach(function(err, event) {
self[event.name].apply(self, event.payload);
});
}
Market.prototype.createOrder = function(payload, cb) {
...
if(cb) cb();
};
Market.prototype.replaceOrder = function(payload, cb) {
...
if(cb) cb();
};
Market.js
marketId, version, name, createdAt, payload,
"92343VBC7", 1, "createOrder", "2014-11-14 11:34:11", { _id: 1, user: trader1, side: "buy", qty: 500000, limit: 99.95, ... }
"92343VBC7", 2, "replaceOrder", "2014-11-14 11:37:08", { _id: 1, limit: 99.96 }
"92343VBC7", 3, "replaceOrder", "2014-11-14 11:54:32", { _id: 1, limit: 99.97, updatedBy: algo1 }
"92343VBC7", 4, "replaceOrder", "2014-11-14 12:11:05", { _id: 1, limit: 99.98 }
MarketEvents
A: Snapshot the system state every n-events to create a checkpoint.
{
marketId: "92343VBC7",
version: 4,
data: {
orders: [
{ _id: 1, side: "buy", limit: 99.98, ...}
]
}
}
MarketSnapshots
marketId, version, name, createdAt, payload,
"92343VBC7", 1, "createOrder", "2014-11-14 11:34:11", { _id: 1, user: trader1, side: "buy", qty: 500000, limit: 99.95, ... }
"92343VBC7", 2, "replaceOrder", "2014-11-14 11:37:08", { _id: 1, limit: 99.96 }
"92343VBC7", 3, "replaceOrder", "2014-11-14 11:54:32", { _id: 1, limit: 99.97, updatedBy: algo1 }
"92343VBC7", 4, "replaceOrder", "2014-11-14 12:11:05", { _id: 1, limit: 99.98 }
"92343VBC7", 5, "replaceOrder", "2014-11-14 12:11:18", { _id: 1, limit: 99.97 }
Market Events
A: You don't, instead perform a "reverse" operation.
Market.prototype.replaceOrder = function(payload, cb) {
// find the order from list of orders
var order = _.find(this.orders, function(o) { o._id === payload._id });
// existential check on replacable fields
if(payload.limit) order.limit = payload.limit;
if(payload.quantity) order.quantity = payload.quantity;
// notify that order has been changed
this.notify('orderReplaced', order);
if(cb) cb();
};
Market.js
A: Emit a message representing changes
Thanks @mateodelnorte!
var Order = require('./order');
var Entity = require('sourced').Entity;
var util = require('util');
function Market () {
this.id = null;
this.orders = [];
this.trades = [];
Entity.call(this);
}
util.inherits(Market, Entity);
Market.prototype.initialize = function(id, cb) {
this.id = id;
if(cb) cb();
};
Market.prototype.createOrder = function(o, cb) {
// tell sourced to automatically digest the event and params
this.digest('createOrder', o);
this.orders.push(new Order(o));
if(cb) cb();
};
module.exports = Market;
var Promise = require('bluebird');
var sourcedRepoMongo = require('sourced-repo-mongo');
var MongoRepository = sourcedRepoMongo.Repository;
var Market = require('./market');
var util = require('util');
function MarketRepository () {
this.cache = {};
MongoRepository.call(this, Market);
}
util.inherits(MarketRepository, MongoRepository);
MarketRepository.prototype.get = function (id, cb) {
var self = this;
var promise = new Promise(function (resolve, reject) {
var market = self.cache[id];
if(!market) {
// rebuild from event snapshots and store
MarketRepository.super_.prototype.get.call(self, id, function (err, market) {
self.cache[id] = market;
resolve(market);
});
} else {
resolve(market);
}
});
promise.done(function (market) {
cb(null, market);
});
};
module.exports = MarketRepository;
// from sourced-repo-mongo.js
Repository.prototype.get = function get (id, cb) {
var self = this;
log('getting %s for id %s', this.entityType.name, id);
this.initialized.done(function () {
self.snapshots
.find({ id: id })
.sort({ version: -1 })
.limit(-1)
.toArray(function (err, docs) {
if (err) return cb(err);
var snapshot = docs[0];
var criteria = (snapshot) ? { id: id, version: { $gt: snapshot.version } } : { id: id };
self.events.find(criteria)
.sort({ version: 1 })
.toArray(function (err, events) {
if (err) return cb(err);
if (snapshot) delete snapshot._id;
return self.deserialize(id, snapshot, events, cb);
});
});
});
};
var bus = require('servicebus');
var MarketRepository = require('./marketRepository.js');
var repo = new MarketRepository( );
bus.listen('market.*.command', function(msg) {
switch(msg.command) {
case "createOrder":
// fetch from repo - pull from cache or rebuild from db
repo.get(msg.data.id, function(err, market) {
if(err) return msg.reject(err);
// apply event
market.createOrder(msg.data, function(err) {
if(err) return msg.reject(err);
// write the event to the database
repo.commit(function(err) {
if(err) return msg.reject(err);
msg.ack();
});
});
});
break;
default:
msg.reject("Command not recognized");
}
});
// In MarketRepository.js
var bus = require('servicebus');
market.on('order.created', function (order) {
bus.publish('market.order.created', order);
});
// In Market.js
Market.prototype.createOrder = function(o, cb) {
// ...
// !!! Don't do this!
// We haven't committed the event so we can't emit yet
this.emit('order.created', order);
// Instead, use enqueue provided by sourced, which waits to emit until AFTER commit
// also suppress events during replay
this.enqueue('order.created', order);
if(cb) cb();
};
{
userId: "trader1",
bondId: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95
}
HTTP POST /orders
Express
MongoDB
order.save()
{
_id: 1,
userId: "trader1",
bondId: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95,
status: "open",
filled: 0,
remaining: 500000
}
1 Database Save
How are state changes handled when not initiated by a user request?
SockJS
MarketSvc
{
_id: 1,
userId: "trader1",
cusip: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.98,
status: "open",
filled: 200000,
remaining: 300000
}
order.updated
(via WebSockets)
order.updated
(via AMQP)
Command-Query-Responsiblity-Segregation
(in node.js)
Data persisted in services isn't necessarily
complete for end-users
{
_id: 1,
userId: "trader1",
cusip: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95,
status: "open",
filled: 0,
remaining: 500000
}
{
_id: 1,
user: {
userId: "trader1",
username: "John Smith",
firm: "Acme Capital"
},
bond: {
cusip: "92343VBC7",
name: "VZ 3.45 03/15/21",
},
trades: [
{ ... }
],
side: "buy",
quantity: 500000,
limit: 99.95,
status: "open",
filled: 0,
remaining: 500000
}
Order - Service View
Order - UI View
{
_id: 1,
userId: "trader1",
cusip: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95,
status: "open",
filled: 0,
remaining: 500000
}
End User
{
_id: 1,
userId: "trader1",
cusip: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95,
status: "open",
filled: 0,
remaining: 500000,
creditCapacityUsed: 750000,
marketMakerId: "A93"
}
Internal Admin
{
_id: 1,
userId: "trader1",
cusip: "92343VBC7",
side: "buy",
quantity: 500000,
limit: 99.95,
status: "open",
filled: 0,
remaining: 500000,
marketSnapshot: {
bids: [ ... ]
offers: [ ... ]
}
}
Reporting
Webapp
MongoDB
event.save()
Market
createOrder
event.publish('order.created')
Denormalizer
MongoDB
Read-only
orderUpdated
cmd.send('createOrder')
bus.subscribe('order.updated', function (event, cb) {
var order = event.data.order;
var query = {
$set: {
'bond': order.bond,
'clientOrderId': order.clientOrderId,
'createdAt': order.createdAt || Date.now(),
'limitType': order.limitType,
'qty': order.qty,
'qtyFilled': order.qtyFilled,
'qtyRemaining': order.qtyRemaining,
'type': order.type,
'updatedAt': Date.now()
}
}
Order.update({ _id: order._id }, query, { upsert: true }, function (err, result) {
if( err ) log.error(err);
return cb(err);
});
});
It's really helpful when you need to build aggregated or consolidated views from different domains
Market
Denormalizer
MongoDB
Treasury
Accounts
Client ViewModel
Events
Denormalizer
MongoDB
Client ViewModel
Events
Meteor App
Data auto-pushed via Websockets
UI
Oplog Tailing
Refactor frequent and refactor often!