pywren with AWS Lambda sqs worker

@jpizarrom

@jpizarrom

  • Lead Software Engineer @ Admetricks.com

https://www.youtube.com/watch?v=2G366R5T0Dc

  • pywren AWS Lambda sqs worker
  • pywren AWS Lambda sqs invoker
  • Demo with notebook in jupyter
  • QA

Pywren

lets you run your existing python code at massive scale via AWS Lambda

http://pywren.io/

pywren AWS Lambda sqs worker

diff --git a/pywren/wrenhandler.py b/pywren/wrenhandler.py
index 89924ce..d49420a 100644
--- a/pywren/wrenhandler.py
+++ b/pywren/wrenhandler.py
...
 
 def generic_handler(event, context_dict):
     """
     context_dict is generic infromation about the context
     that we are running in, provided by the scheduler
     """
+    logger.info(json.dumps(event))
     try:
+        message_id = None
+        receipt_handle = None
+        queue_url = None
+        if 'Body' in event:
+            logger.info("invocation started from sqs")
+            old_event = event
+            event = json.loads(old_event['Body'])
+            logger.info(json.dumps(event))
+            queue_url = old_event['QueueUrl'] if 'QueueUrl' in old_event else None
+            receipt_handle = old_event['ReceiptHandle'] if 'ReceiptHandle' in old_event else None
+            message_id = old_event['MessageId'] if 'MessageId' in old_event else None
+
         response_status = {'exception' : None}
         s3 = boto3.resource('s3')
...

@@ ... @@ def generic_handler(event, context_dict):
         response_status['exception_traceback'] = traceback.format_exc()
     finally:
+        delete_sqs_message(queue_url, receipt_handle)

 if __name__ == "__main__":
...
diff --git a/pywren/wrenhandler.py b/pywren/wrenhandler.py
index 89924ce..d49420a 100644
--- a/pywren/wrenhandler.py
+++ b/pywren/wrenhandler.py

...
 
+def delete_sqs_message(queue_url, receipt_handle):
+    if queue_url is not None and receipt_handle is not None:
+        sqs_client = boto3.client('sqs', region_name='us-east-1')
+
+        try:
+            response = sqs_client.delete_message(
+                QueueUrl=queue_url,
+                ReceiptHandle=receipt_handle
+            )
+        except botocore.exceptions.ClientError as e:
+            if e.response['Error']['Code'] == 'ReceiptHandleIsInvalid':
+                pass
+            else:
+                raise e
+

...

pywren AWS Lambda sqs invoker

var AWS = require("aws-sdk");
var async = require("async");

var QUEUE_URL = "https://sqs.us-east-1.amazonaws.com/XXXXXXX/pywren-jobs-1";
var WORKER_LAMBDA_FUNCTION_NAME = "pywren_1";
var REGION = "us-east-1";

var sqs = new AWS.SQS({region: REGION});
var lambda = new AWS.Lambda({region: REGION});

function receiveMessages(callback) {
    var params = {
        QueueUrl: QUEUE_URL,
        MaxNumberOfMessages: 1
    };
    sqs.receiveMessage(params, function(err, data) {
        if (err) {
            console.error(err, err.stack);
            callback(err);
        } else {
            callback(null, data.Messages);
        }
    });
}


...
...

function invokeWorkerLambda(task, callback) {
    task['QueueUrl'] = QUEUE_URL;
    
    var params = {
        FunctionName: WORKER_LAMBDA_FUNCTION_NAME,
        InvocationType: 'Event',
        Payload: JSON.stringify(task)
    };
    console.log(params);
    lambda.invoke(params, function(err, data) {
        if (err) {
            console.error(err, err.stack);
            callback(err);
        } else {
            callback(null, data)
        }
    });
}

...
...

function handleSQSMessages(context, callback) {
    receiveMessages(function(err, messages) {
        if (messages && messages.length > 0) {
            var invocations = [];
            messages.forEach(function(message) {
                invocations.push(function(callback) {
                    invokeWorkerLambda(message, callback)
                });
            });
            async.parallel(invocations, function(err) {
                if (err) {
                    console.error(err, err.stack);
                    callback(err);
                } else {
                    if (context.getRemainingTimeInMillis() > 20000) {
                        handleSQSMessages(context, callback);
                    } else {
                        callback(null, "PAUSE");
                    }
                }
            });
        } else {
            callback(null, "DONE");
        }
    });
}
...
...

exports.handler = function(event, context, callback) {
    handleSQSMessages(context, callback);
};

Demo with notebook in jupyter

QA

Title Text

  • http://pywren.io/
  • http://pywren.io/pages/gettingstarted.html
  • https://github.com/pywren/examples
  • https://github.com/pywren/pywren
  • Bullet Three

pywren with AWS Lambda sqs worker

By Juan Pizarro

pywren with AWS Lambda sqs worker

  • 1,240