https://slides.com/elpete/itb2023-cbq
What this talk is
- An overview of cbq
- Examples of problems that queues and jobs can solve
- Overview of how to create new cbq protocols
What this talk isn't
- How to use the ColdBox Async Manager
- Overview of ColdBox Scheduled Tasks
What is cbq?
A protocol-based queueing system for ColdBox
A protocol-based queueing system for ColdBox
Can interact with different providers, like the ColdBox Async Engine, a database, Redis, or Rabbit MQ
A protocol-based queueing system for ColdBox
A protocol-based queueing system for ColdBox
Sends a message to be consumed later.
It can be consumed by the same application or a completely different application, language, or service.
A protocol-based queueing system for ColdBox
- Defines a Job as the unit of work on a queue
- Push a Job onto a queue connection to be worked later
- Define multiple queues or named piles of work
- Register worker pools to work the jobs passed to queues
- Ability to switch between Queue Providers easily
What does cbq give me?
Why not just use...?
- cfthread, runAsync, AsyncManager
- Redis, Rabbit MQ, etc.
- Homegrown queue framework
Why would I use cbq?
Sending Email
Preparing Large Spreadsheets
Video Processing
Background Uploads
Sequenced Jobs
Scheduled SMS Messages
Email Verification
Processing Payments
Cancelling Abandoned Orders
Send Monthly Invoices
Other reasons
- Easier testing making sure the right jobs were queued
- Sync in development, Rabbit in production
- Adjust the worker pool scale dynamically
- Retry, timeout, and backoff built in.
Definitions
Job
- Does its work in the `handle` method
- Serializes and deserializes itself to the queue protocol
- Set instance data in the `properties`
-
Exist in the context of your application
Job
Queue
- A named stack of jobs or messages to be delivered
- A queue connection must have at least one queue which is usually `default`
- A queue connection can have as many queues as desired
Queue
Queue Provider
- How a queue connection connects to a backend like Redis, RabbitMQ, or a database
- Can be used multiple times in a single application to define multiple queue connections with different configuration options
Queue Provider
Queue Connection
- A named combination of Queue Provider and properties
- Allows you to connect with multiple Database providers or multiple Redis providers
Queue Connection
Worker Pool
- A group of workers for a Queue Connection
- Can optionally work a subset of queues
- Can optionally work queues in a specific order
- Can be scaled up or down as needed
Worker Pool
Installation
box install cbq
component {
this.javaSettings = {
loadPaths : [ expandPath( "./modules/cbq/lib" ) ],
loadColdFusionClassPath : true,
reloadOnChange : false
};
}Batch-specific Setup
Configuration
Module Settings
moduleSettings = {
"cbq" : {
// The path the custom config file to
// register connections and worker pools
"configPath" : "config.cbq",
// Flag if workers should be registered.
// If your application only pushes to the queues,
// you can set this to false.
"registerWorkers" : getSystemSetting( "CBQ_REGISTER_WORKERS", true ),
// The interval to poll for changes to the worker pool scaling.
// Defaults to 0 which turns off the scheduled scaling feature.
"scaleInterval" : 0,
// continued...
}
};moduleSettings = {
"cbq" : {
// The default amount of time, in seconds, to delay a job.
// Used if the connection and job doesn't define their own.
"defaultWorkerBackoff" : 0,
// The default amount of time, in seconds, to wait before timing out a job.
// Used if the connection and job doesn't define their own.
"defaultWorkerTimeout" : 60,
// The default amount of attempts to try before failing a job.
// Used if the connection and job doesn't define their own.
"defaultWorkerMaxAttempts" : 1,
// continued...
}
};moduleSettings = {
"cbq" : {
// Datasource information for tracking batches.
"batchRepositoryProperties" : {
"tableName" : "cbq_batches",
"datasource" : "", // `datasource` can also be a struct
"queryOptions" : {} // The sibling `datasource` property overrides
// any defined datasource in queryOptions.
},
// Flag to turn on logging failed jobs to a database table.
"logFailedJobs" : false,
// Datasource information for loggin failed jobs.
"logFailedJobsProperties" : {
"tableName" : "cbq_failed_jobs",
"datasource" : "", // `datasource` can also be a struct.
"queryOptions" : {} // The sibling `datasource` property overrides
// any defined datasource in queryOptions.
}
}
};cbq Config File
component {
function configure() {
newConnection( "default" )
.setProvider( "ColdBoxAsyncProvider@cbq" );
newConnection( "database" )
.setProvider( "DBProvider@cbq" )
.setProperties( { "tableName": "custom_jobs_table" } );
newConnection( "external" )
.setProvider( "DBProvider@cbq" )
.setProperties( { "datasource": "external" } );
newWorkerPool( "default" )
.forConnection( "default" )
.setTimeout( 5 )
.setMaxAttempts( 5 );
newWorkerPool( "priority" )
.forConnection( "database" )
.onQueues( [ "priority", "*" ] )
.setQuantity( 3 );
}
}Usage
Create a Job
// SendWelcomeEmailJob.cfc
component extends="cbq.models.Jobs.AbstractJob" {
property name="mailService" inject="provider:MailService@cbmailservices";
property name="userId";
function handle() {
var user = getInstance( "User" ).findOrFail( getUserId() );
variables.mailService
.newMail(
from = "no-reply@example.com",
to = user.getEmail(),
subject = "Welcome!",
type = "html"
)
.setView( "/_emails/users/welcome" )
.setBodyTokens( {
"firstName" : user.getFirstName(),
"lastName" : user.getLastName()
} )
.send();
}
}Per-Job Configuration
// SendWelcomeEmailJob.cfc
component extends="cbq.models.Jobs.AbstractJob" {
variables.connection = "emails";
variables.maxAttempts = 3;
variables.timeout = 10;
function handle() {
// ...
}
}Dispatching Jobs
// handlers/Main.cfc
component {
function create() {
var user = createUser( rc );
getInstance( "SendWelcomeEmailJob" )
.setProperties( { "userId": user.getId() } )
.dispatch();
}
}Setting up Worker Pools
// config/cbq.cfc
component {
function configure() {
newConnection( "default" ).setProvider( "DBProvider@cbq" );
newWorkerPool( "default" )
.forConnection( "default" )
.setTimeout( 30 )
.setMaxAttempts( 5 )
.setQuantity( 3 );
newWorkerPool( "default" )
.forConnection( "default" )
.onQueues( "emails" )
.setTimeout( 60 )
.setMaxAttempts( 10 )
.setQuantity( 10 );
}
}
moduleSettings = {
"cbq" : {
"registerWorkers" : true // this is also the default
}
};cbq Model Helper
// handlers/Main.cfc
component {
property name="cbq" inject="cbq@cbq";
function create() {
var user = createUser( rc );
cbq.job( "SendWelcomeEmailJob", {
"userId": user.getId()
} ).dispatch();
}
}Job Chains
// FulfillOrderJob.cfc
component extends="cbq.models.Jobs.AbstractJob" {
function handle() {
var productId = getProperties().productId;
processPayment( productId );
getInstance( "SendProductLinkEmail" )
.onConnection( "fulfillment" )
.setProperties( {
"productId" : productId,
"userId" : getProperties().userId
} )
.dispatch();
}
}// handlers/Main.cfc
component {
function create() {
getInstance( "FulfillOrderJob" )
.setProperties( { "productId": rc.productId } )
.chain( [
getInstance( "SendProductLinkEmail" )
.onConnection( "fulfillment" )
.setProperties( {
"productId" : rc.productId,
"userId" : auth().getUserId()
} )
] )
}
}// handlers/Main.cfc
component {
property name="cbq" inject="cbq@cbq";
function create() {
cbq.chain( [
cbq.job( "FulfillOrderJob", { "productId": rc.productId } ),
cbq.job( job = "SendProductLinkEmail", properties = {
"productId": rc.productId,
"userId": auth().getUserId()
}, connection = "fulfillment" ),
] ).dispatch();
}
}Job Batches
// handlers/Main.cfc
component {
property name="cbq" inject="cbq@cbq";
function create() {
cbq.batch( [
cbq.job( "ImportCsvJob", { "start": 1, "end": 100 } ),
cbq.job( "ImportCsvJob", { "start": 101, "end": 200 } ),
cbq.job( "ImportCsvJob", { "start": 201, "end": 300 } ),
cbq.job( "ImportCsvJob", { "start": 301, "end": 400 } ),
cbq.job( "ImportCsvJob", { "start": 401, "end": 500 } )
] ).then( cbq.job( "ImportCsvSuccessfulJob" ) )
.catch( cbq.job( "ImportCsvFailedJob" ) )
.finally( cbq.job( "ImportCsvCompletedJob" ) )
.dispatch();
}
}Demo
Bonus
Delay the Current Job
// MonitorPendingCartJob.cfc
component extends="cbq.models.Jobs.AbstractJob" {
function handle() {
var cartId = getProperties().cartId;
var cart = getInstance( "Cart" ).findOrFail( cartId );
if ( cart.isConfirmed() || cart.isCancelled() ) {
return;
}
if ( abs( dateDiff( "n", cart.getModifiedDate(), now() ) ) > 60 ) {
cart.cancel();
return;
}
getInstance( "MonitorPendingCartJob" )
.setProperties( { "cartId" : cartId } )
.delay( 15 * 1000 )
.dispatch();
}
}// MonitorPendingCartJob.cfc
component extends="cbq.models.Jobs.AbstractJob" {
function handle() {
var cartId = getProperties().cartId;
var cart = getInstance( "Cart" ).findOrFail( cartId );
if ( cart.isConfirmed() || cart.isCancelled() ) {
return;
}
if ( abs( dateDiff( "n", cart.getModifiedDate(), now() ) ) > 60 ) {
cart.cancel();
return;
}
this.release( 15 * 1000 );
}
}Using `release`
// MonitorPendingCartJob.cfc
component extends="cbq.models.Jobs.AbstractJob" {
function handle() {
var cartId = getProperties().cartId;
var cart = getInstance( "Cart" ).findOrFail( cartId );
if ( cart.isConfirmed() || cart.isCancelled() ) {
return;
}
if ( abs( dateDiff( "n", cart.getModifiedDate(), now() ) ) > 60 ) {
cart.cancel();
return;
}
var delay = 2 ^ this.getCurrentAttempt() * 1000;
this.release( delay );
}
}Exponential Backoff
ITB 2023 — cbq
By Eric Peterson
