Celery Best Practices

Tian Chu @ Offerpop

June 30, 2015

Distributed Task Queue

from celery import task
from pymongo import MongoClient


@task()
def log_event(event):
    mongo_client = MongoClient()
    mongo_db = mongo_client.test_db
    mongo_db.event.save(event)


def run():
    log_event.delay({
        "_id": 274738,
        "type": "visited page",
        "ip_address": "65.47.3.23",
    })

Behind the Scenes

vagrant@heritage:~$ redis-cli

redis 127.0.0.1:6379> keys *
1) "celery"
2) "_kombu.binding.celery.pidbox"
3) "_kombu.binding.celery"

redis 127.0.0.1:6379> type celery
list

redis 127.0.0.1:6379> lrange celery 0 1
1) "{\"body\": \"gAJ9cQEoVQdleHBpcmVzcQJOVQN1dGNxA4hVBGFyZ3NxBH1xBShVCmlwX2Fk
ZHJlc3NxBlUKNjUuNDcuMy4yM3EHVQNfaWRxCEoyMQQAVQR0eXBlcQlVDHZpc2l0ZWQgcGFnZXEKd
YVxC1UFY2hvcmRxDE5VCWNhbGxiYWNrc3ENTlUIZXJyYmFja3NxDk5VB3Rhc2tzZXRxD05VAmlkcR
BVJDM2ZDY4ZDAxLWM0ZGYtNDdhYS1hYWE1LTEwMjNiNDhlZThmYXERVQdyZXRyaWVzcRJLAFUEdGF
za3ETVRFkZWZhdWx0LmxvZ19ldmVudHEUVQNldGFxFU5VBmt3YXJnc3EWfXEXdS4=\", \"header
s\": {}, \"content-type\": \"application/x-python-serialize\", \"properties\"
: {\"body_encoding\": \"base64\", \"delivery_info\": {\"priority\": 0, \"rout
ing_key\": \"celery\", \"exchange\": \"celery\"}, \"delivery_mode\": 2, \"del
ivery_tag\": \"90837bd0-c760-42b7-9665-6aa9a814309d\"}, \"content-encoding\":
 \"binary\"}"
import pickle
import base64
import pprint

msg = pickle.loads(base64.b64decode(
    "gAJ9cQEoVQdleHBpcmVzcQJOVQN1dGNxA4hVBGFyZ3NxBH1xBShVCmlwX2"
    "FkZHJlc3NxBlUKNjUuNDcuMy4yM3EHVQNfaWRxCEoyMQQAVQR0eXBlcQlV"
    "DHZpc2l0ZWQgcGFnZXEKdYVxC1UFY2hvcmRxDE5VCWNhbGxiYWNrc3ENTl"
    "UIZXJyYmFja3NxDk5VB3Rhc2tzZXRxD05VAmlkcRBVJDM2ZDY4ZDAxLWM0"
    "ZGYtNDdhYS1hYWE1LTEwMjNiNDhlZThmYXERVQdyZXRyaWVzcRJLAFUEdG"
    "Fza3ETVRFkZWZhdWx0LmxvZ19ldmVudHEUVQNldGFxFU5VBmt3YXJnc3EW"
    "fXEXdS4="
))

pprint.pprint(msg)

{'args': ({'_id': 274738, 'ip_address': '65.47.3.23', 'type': 'visited page'},),
 'callbacks': None,
 'chord': None,
 'errbacks': None,
 'eta': None,
 'expires': None,
 'id': '36d68d01-c4df-47aa-aaa5-1023b48ee8fa',
 'kwargs': {},
 'retries': 0,
 'task': 'default.log_event',
 'taskset': None,
 'utc': True}
  • Queues are implemented as (linked) lists in Redis
  • Tasks messages are pickled and b64-encoded
  • Enqueue and dequeue operations are efficient
  • Workers are polling the Redis every 0.5 sec (default) 

Brief Summary

  • There is pull request to replace polling with Pub/Sub 
  • JSON rather than pickle as the default serializer
  • Django projects should use Celery APIs directly rather than relying on the bridge library Django-celery

In the near future

#1 Passing Big Objects to Tasks => Memory Leak

from celery import task
from pymongo import MongoClient


@task()
def process_file(file_content):
    print file_content


def run():
    for i in range(1000):
        file = open('big_file_%s.text' % i, 'r')
        file_content = file.read()
        process_file.delay(file_content)
from celery import task
from pymongo import MongoClient


@task()
def process_file(file_name):
    file = open('big_file_%s.text' % i, 'r')
    file_content = file.read()
    print file_content


def run():
    for i in range(1000):
        process_file.delay('big_file_%s.text' % i)

#2 Passing Database/ORM Objects => Race Condition

from celery import task

@task()
def update_user_picture(user_object, picture):
    """Update user profile picture in background, since the
       uploading process takes a while to complete."""

    new_profile_picture_url = upload_picture(picture)
    user_object.profile_picture = new_profile_picture_url
    user_object.save()

def update_username(request, user_id, username):
    user_object = db.user.get(user_id=user_id)
    user_object.username = username
    user_object.save()

def update_user_picture(request, user_id, picture):
    user_object = db.user.get(user_id=user_id)
    update_user_picture.delay(user_object, picture)
@task()
def update_user_picture(user_id, picture):
    user_object = db.user.get(user_id=user_id)
    new_profile_picture_url = upload_picture(picture)
    user_object.profile_picture = new_profile_picture_url
    user_object.save()

def update_username(user_id, username):
    user_object = db.user.get(user_id=user_id)
    user_object.username = username
    user_object.save()

def update_user_picture(request, user_id, picture):
    update_user_picture.delay(user_id, picture)
@task()
def update_user_picture(user_id, picture):

    # Uploading takes time, make sure to get a fresh
    # user_object before updating/saving it.
    new_profile_picture_url = upload_picture(picture)

    user_object = db.user.get(user_id=user_id)
    user_object.profile_picture = new_profile_picture_url
    user_object.save()

def update_username(user_id, username):
    user_object = db.user.get(user_id=user_id)
    user_object.username = username
    user_object.save()

def update_user_picture(request, user_id, picture):
    update_user_picture.delay(user_id, picture)

#3 Route Tasks to Their Own Queues

CELERY_ROUTES = {
    'default.log_event': {
        'queue': 'log_event',
    },
    'default.update_username': {
        'queue': 'update_user_profile',
    },
    'default.update_password': {
        'queue': 'update_user_profile',
    },
    'default.update_user_picture': {
        'queue': 'update_user_profile',
    },
}
@task()
def log_event():
    pass

@task()
def update_username():
    pass

@task()
def update_password():
    pass

@task()
def update_user_picture():
    pass

def run():
    log_event.delay()
    update_username.delay()
    update_password.delay()
    update_user_picture.delay()

Behind the Scenes

redis 127.0.0.1:6379> keys *
1) "log_event"
2) "_kombu.binding.celery.pidbox"
3) "_kombu.binding.update_user_profile"
4) "_kombu.binding.log_event"
5) "_kombu.binding.celery"
6) "update_user_profile"

redis 127.0.0.1:6379> type update_user_profile
list

redis 127.0.0.1:6379> llen update_user_profile
(integer) 3

redis 127.0.0.1:6379> lpop update_user_profile
"{\"body\": \"gAJ9cQEoVQdleHBpcmVzcQJOVQN1dGNxA4hVBGFyZ3NxBF1xBVUFY2hvcm
RxBk5VCWNhbGxiYWNrc3EHTlUIZXJyYmFja3NxCE5VB3Rhc2tzZXRxCU5VAmlkcQpVJGE5Zm
Y1YzhiLTVhZDEtNDhlNy04MzY1LWI1YTQyNGQ1N2Q2M3ELVQdyZXRyaWVzcQxLAFUEdGFza3
ENVRtkZWZhdWx0LnVwZGF0ZV91c2VyX3BpY3R1cmVxDlUDZXRhcQ9OVQZrd2FyZ3NxEH1xEX
Uu\", \"headers\": {}, \"content-type\": \"application/x-python-serializ
e\", \"properties\": {\"body_encoding\": \"base64\", \"delivery_info\": 
{\"priority\": 0, \"routing_key\": \"update_user_profile\", \"exchange\"
: \"update_user_profile\"}, \"delivery_mode\": 2, \"delivery_tag\": \"f6
605c64-31bc-48e6-b8d5-2fbf16c0ec17\"}, \"content-encoding\": \"binary\"}"

Long-Running Tasks Won't Block Short-Lived Ones

CELERY_ROUTES = {
    'default.log_event': {
        'queue': 'log_event',
    },
    'default.update_username': {
        'queue': 'update_username', # a separate queue for easier monitoring
    },
    'default.update_password': {
        'queue': 'update_password', # a separate queue for easier monitoring
    },
    'default.update_user_picture': {
        'queue': 'update_user_picture', # long-running, use a separate queue
    },
}

Monitoring & Management

redis 127.0.0.1:6379> llen update_user_profile
(integer) 1273

redis 127.0.0.1:6379> DEL update_user_profile
(integer) 1

#4 Retry & Idempotent

from celery import task

@task()
def log_event(event):
    """
        Simply save the event object to database.
        Retrying will cause duplicate objects saved.
    """
    db.save(event)
  • Celery Tasks may fail or be interrupted.
  • Never assume the current state of the system when a task begins.
  • Change as little external state as possible.

Idempotent Design Allows Retrying Failed Tasks

from celery import task

@task(default_retry_delay=10, max_retries=3)
def log_event(event):
    """
        Save the event object to database, only if it's 
        not been created yet.
    """
    if not db.event.find_one(ip=event.ip, user_agent=event.user_agent):
        db.save(event)
  • Idempotent function f(x): f(f(x)) = f(x)
  • Function can be repeated many times without unintented effects

#5 Property Caching

from celery import Task

class DatabaseTask(Task):
    abstract = True
    _db = None

    @property
    def db(self):
        """Cache the Database connection for reuse."""
        if self._db is None:
            self._db = Database.connect()
        return self._db

    def run(self, user_id, username):
        user = self.db.user.find_one(user_id=user_id)
        user["username"] = username
        self.db.user.save(user)

#6 Keeping States

from celery import Task

class NaiveAuthenticateServer(Task):

    def __init__(self):
        self.users = {'george': 'password'}

    def run(self, username, password):
        try:
            return self.users[username] == password
        except KeyError:
            return False

#7 Eventlet V.S. Prefork

# Eventlet

$ python manage.py celery worker -P eventlet --concurrency=1000

$ ps -ef | grep celery
vagrant   7483   3090   16   23:08   pts/5 00:00:04 
python manage.py celery worker -P eventlet --concurrency=1000


# Prefork

$ python manage.py celery worker --concurrency=5

$ ps -ef | grep celery
vagrant 7536 3090 00:00:04 python manage.py celery worker --concurrency=5
vagrant 7548 7536 00:00:00 python manage.py celery worker --concurrency=5
vagrant 7549 7536 00:00:00 python manage.py celery worker --concurrency=5
vagrant 7550 7536 00:00:00 python manage.py celery worker --concurrency=5
vagrant 7551 7536 00:00:00 python manage.py celery worker --concurrency=5
vagrant 7552 7536 00:00:00 python manage.py celery worker --concurrency=5

http://celery.readthedocs.org/en/latest/userguide/concurrency/eventlet.html

Which to Use?

CPU-bounded tasks => Prefork

  • Fully utilize computational resources by creating more processes.
  • Concurrency default to number of CPU cores.
  • Use cases:
    • Image processing
    • Data processing

Which to Use? (cont'd)

I/O bounded tasks => Eventlet

  • Easily achieve a 1,000 concurrency by non-blocking I/O.
  • Use cases:
    • Making Facebook/Twitter API calls
    • Querying Databases (e.g., reports generation)

#8 Class Method as Task

from celery import task


class UserProfileUpdater(object):

    @staticmethod
    @celery.task()
    def update_user_picture(user_id, picture):
        new_profile_picture_url = upload_picture(picture)
        user = db.user.find_one(user_id=user_id)
        user.profile_picture = new_profile_picture_url
        user.save()

Build-in support celery.contrib.methods has been removed. Too many bugs to be usable.

Use staticmethod as a workaround.

 

Thank You!

Questions?

Celery Best Practices

By Tian Chu

Made with Slides.com

Celery Best Practices

Celery Best Practices

  • 1,312