pywren with AWS Lambda sqs worker
@jpizarrom
@jpizarrom
- Lead Software Engineer @ Admetricks.com
- pywren AWS Lambda sqs worker
- pywren AWS Lambda sqs invoker
- Demo with notebook in jupyter
- QA
Pywren
lets you run your existing python code at massive scale via AWS Lambda
http://pywren.io/
pywren AWS Lambda sqs worker
diff --git a/pywren/wrenhandler.py b/pywren/wrenhandler.py
index 89924ce..d49420a 100644
--- a/pywren/wrenhandler.py
+++ b/pywren/wrenhandler.py
...
def generic_handler(event, context_dict):
"""
context_dict is generic infromation about the context
that we are running in, provided by the scheduler
"""
+ logger.info(json.dumps(event))
try:
+ message_id = None
+ receipt_handle = None
+ queue_url = None
+ if 'Body' in event:
+ logger.info("invocation started from sqs")
+ old_event = event
+ event = json.loads(old_event['Body'])
+ logger.info(json.dumps(event))
+ queue_url = old_event['QueueUrl'] if 'QueueUrl' in old_event else None
+ receipt_handle = old_event['ReceiptHandle'] if 'ReceiptHandle' in old_event else None
+ message_id = old_event['MessageId'] if 'MessageId' in old_event else None
+
response_status = {'exception' : None}
s3 = boto3.resource('s3')
...
@@ ... @@ def generic_handler(event, context_dict):
response_status['exception_traceback'] = traceback.format_exc()
finally:
+ delete_sqs_message(queue_url, receipt_handle)
if __name__ == "__main__":
...
diff --git a/pywren/wrenhandler.py b/pywren/wrenhandler.py
index 89924ce..d49420a 100644
--- a/pywren/wrenhandler.py
+++ b/pywren/wrenhandler.py
...
+def delete_sqs_message(queue_url, receipt_handle):
+ if queue_url is not None and receipt_handle is not None:
+ sqs_client = boto3.client('sqs', region_name='us-east-1')
+
+ try:
+ response = sqs_client.delete_message(
+ QueueUrl=queue_url,
+ ReceiptHandle=receipt_handle
+ )
+ except botocore.exceptions.ClientError as e:
+ if e.response['Error']['Code'] == 'ReceiptHandleIsInvalid':
+ pass
+ else:
+ raise e
+
...
pywren AWS Lambda sqs invoker
var AWS = require("aws-sdk");
var async = require("async");
var QUEUE_URL = "https://sqs.us-east-1.amazonaws.com/XXXXXXX/pywren-jobs-1";
var WORKER_LAMBDA_FUNCTION_NAME = "pywren_1";
var REGION = "us-east-1";
var sqs = new AWS.SQS({region: REGION});
var lambda = new AWS.Lambda({region: REGION});
function receiveMessages(callback) {
var params = {
QueueUrl: QUEUE_URL,
MaxNumberOfMessages: 1
};
sqs.receiveMessage(params, function(err, data) {
if (err) {
console.error(err, err.stack);
callback(err);
} else {
callback(null, data.Messages);
}
});
}
...
...
function invokeWorkerLambda(task, callback) {
task['QueueUrl'] = QUEUE_URL;
var params = {
FunctionName: WORKER_LAMBDA_FUNCTION_NAME,
InvocationType: 'Event',
Payload: JSON.stringify(task)
};
console.log(params);
lambda.invoke(params, function(err, data) {
if (err) {
console.error(err, err.stack);
callback(err);
} else {
callback(null, data)
}
});
}
...
...
function handleSQSMessages(context, callback) {
receiveMessages(function(err, messages) {
if (messages && messages.length > 0) {
var invocations = [];
messages.forEach(function(message) {
invocations.push(function(callback) {
invokeWorkerLambda(message, callback)
});
});
async.parallel(invocations, function(err) {
if (err) {
console.error(err, err.stack);
callback(err);
} else {
if (context.getRemainingTimeInMillis() > 20000) {
handleSQSMessages(context, callback);
} else {
callback(null, "PAUSE");
}
}
});
} else {
callback(null, "DONE");
}
});
}
...
...
exports.handler = function(event, context, callback) {
handleSQSMessages(context, callback);
};
Demo with notebook in jupyter
QA
Title Text
- http://pywren.io/
- http://pywren.io/pages/gettingstarted.html
- https://github.com/pywren/examples
- https://github.com/pywren/pywren
- Bullet Three
pywren with AWS Lambda sqs worker
By Juan Pizarro
pywren with AWS Lambda sqs worker
- 1,270