A Bit of Python Hacking

Maarten van Schaik

Python BBQ Meetup

16 July 2015

Who Am I?

  • Software Engineer @ Byte
  • Makes tools for Magento devs
  • Likes Python, FP, BBQ

Most used type in Python?

d = {"one": 1,
     "two": 2,
     "three": 3}

Dicts are used everywhere!

def set_car_temp(vehicle_id, driver, passenger):
    data = {
        "driver_temp": driver,
        "passenger_temp": passenger,
    }
    url = "/vehicles/%s/command/set_temps" % vehicle_id
    resp = requests.post(TESLA_API_URL + url, 
                         data=data,
                         headers=get_auth_headers())
    verify_response(resp)

actual dict

keywords args are passed as dict

vars are stored in locals()-dict

def set_car_temp(vehicle_id, driver, passenger):
    data = {
        "driver_temp": driver,
        "passenger_temp": passenger,
    }
    url = "/vehicles/%s/command/set_temps" % vehicle_id
    resp = requests.post(TESLA_API_URL + url, 
                         data=data,
                         headers=get_auth_headers())
    verify_response(resp)
def set_car_temp(vehicle_id, driver, passenger):
    data = {
        "driver_temp": driver,
        "passenger_temp": passenger,
    }
    url = "/vehicles/%s/command/set_temps" % vehicle_id
    resp = requests.post(TESLA_API_URL + url, 
                         data=data,
                         headers=get_auth_headers())
    verify_response(resp)
def set_car_temp(vehicle_id, driver, passenger):
    data = {
        "driver_temp": driver,
        "passenger_temp": passenger,
    }
    url = "/vehicles/%s/command/set_temps" % vehicle_id
    resp = requests.post(TESLA_API_URL + url, 
                         data=data,
                         headers=get_auth_headers())
    verify_response(resp)

globals are stored in globals() dict

def set_car_temp(vehicle_id, driver, passenger):
    data = {
        "driver_temp": driver,
        "passenger_temp": passenger,
    }
    url = "/vehicles/%s/command/set_temps" % vehicle_id
    resp = requests.post(TESLA_API_URL + url, 
                         data=data,
                         headers=get_auth_headers())
    verify_response(resp)

almost

Objects are not dicts

>>> resp["status_code"]
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: 'Response' object has no attribute '__getitem__'

Objects vs dicts

Field access

# Dict
>>> data["driver_temp"]
21
>>> data["driver_temp"] = 23

# Object
>>> resp.status_code
200
>>> resp.status_code = 1000

Dynamic field access

# Dict
>>> field = "driver_temp"
>>> data[field]
21
>>> data[field] = 23

# Object
>>> field = "status_code"
>>> getattr(resp, field)
200
>>> setattr(resp, field, 1000)

Dynamic field access

# Dict
>>> field = "driver_temp"
>>> data.__getitem__(field)
21
>>> data.__setitem__(field, 23)

# Object
>>> field = "status_code"
>>> resp.__getattribute__(field)
200
>>> resp.__setattr__(field, 1000)

Getters

>>> operator import *

# Dict
>>> map(itemgetter("driver_temp"), [d1, d2, d3])
[21, 24, 21]

# Object
>>> map(attrgetter("status_code"), [r1, r2, r3])
[200, 404, 201]

What's the difference?

>>> resp.status_code
200
>>> resp.__dict__["status_code"]
200

actually a dict!

Why this difference?

  • No object literal syntax :-(
  • Inconvenient to create stub objects
  • I want both attr and item access!

In the meanwhile...

var myObj = {'status_code': 200,
             'driver_temp': 21}
myObj.status_code
200
myObj["status_code"]
200
myObj.driver_temp
21

Let's fix this!

Objects have a dict

>>> my_obj = object()
>>> my_obj.__dict__

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: 'object' object has no attribute '__dict__'

Ehhh, not all objects have a __dict__?

>>> class CustomObject(object):
...     pass
... 
>>> my_obj = CustomObject()
>>> my_obj.__dict__
{}

Where does __dict__ come from?

  • My own object has a __dict__
  • It's superclass does't
  • Must be added by class!

What is a class?

class CustomClass(object):
    pass


# Actually:
CustomClass = type("CustomClass", 
                   (object,), 
                   {})

name

class CustomClass(object):
    pass


# Actually:
CustomClass = type("CustomClass", 
                   (object,), 
                   {})

base classes

class CustomClass(object):
    pass


# Actually:
CustomClass = type("CustomClass", 
                   (object,), 
                   {})

dict!

What does type() do?

What is a type?

typedef struct _typeobject {
    PyObject_VAR_HEAD
    const char *tp_name; /* For printing */
    Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */

    struct PyMethodDef *tp_methods;
    struct PyMemberDef *tp_members;

    /* Methods to implement standard operations */
    hashfunc tp_hash;  // __hash__
    reprfunc tp_repr;  // __repr__
    newfunc tp_new;    // __new__

    Py_ssize_t tp_dictoffset;

    PyObject *tp_bases;
} PyTypeObject;

type() creates a

new PyTypeObject

static PyObject *
type_new(PyTypeObject *metatype, 
         PyObject *args, PyObject *kwds)
{
    PyArg_ParseTupleAndKeywords(args, kwds, 
                                "SO!O!:type", kwlist,
                                &name, &bases, &dict);

    type = (PyTypeObject *)metatype->tp_alloc();

    type->tp_name = PyString_AS_STRING(name);
    type->tp_bases = bases;
    type->tp_dict = dict = PyDict_Copy(dict);

    type->tp_dictoffset = base->tp_basicsize;
    type->tp_basicsize += sizeof(PyObject *);

    return (PyObject *)type;
}

I want a dict

with a __dict__!

(or an object with item-access to its attrs, but then I still would not have literal syntax)

Making a dict with a __dict__

>>> d = {}
>>> d.__dict__
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: 'dict' object has no attribute '__dict__'

# By subclassing, we obtain a __dict__
>>> mydict = type('dict', (dict,), {})
>>> d = mydict()
>>> d
{}
>>> d.__dict__
{}

But now we have 2 dicts...

>>> d.status_code = 200
>>> d['driver_temp'] = 23
>>> d
{'driver_temp': 23}
>>> d.__dict__
{'status_code': 200}

We can assign __dict__ to itself

>>> d.__dict__ = d
>>> d.driver_temp = 21
>>> d
{'driver_temp': 21}
>>> d.driver_temp
21
>>> d['driver_temp']
21

But still no object literals...

yet!

This is the dict

typedef struct _dictobject PyDictObject;
struct _dictobject {
    PyObject_HEAD
    Py_ssize_t ma_fill;  /* # Active + # Dummy */
    Py_ssize_t ma_used;  /* # Active */

    PyDictEntry *ma_table;
    PyDictEntry *(*ma_lookup)(PyDictObject *mp, 
                              PyObject *key, 
                              long hash);
};
typedef struct _dictobject PyDictObject;
struct _dictobject {
    PyObject_HEAD
    Py_ssize_t ma_fill;  /* # Active + # Dummy */
    Py_ssize_t ma_used;  /* # Active */

    PyDictEntry *ma_table;
    PyDictEntry *(*ma_lookup)(PyDictObject *mp, 
                              PyObject *key, 
                              long hash);

    PyObject* dict;
};

This is the dict type

PyTypeObject PyDict_Type = {
    PyVarObject_HEAD_INIT(&PyType_Type, 0)
    "dict",
    sizeof(PyDictObject),
    0,
    // ...
    mapp_methods,                         // tp_methods
    0,                                    // tp_members
    (hashfunc)PyObject_HashNotImplemented,// tp_hash 
    (reprfunc)dict_repr,                  // tp_repr
    0,                                    // tp_dictoffset
    dict_new,                             // tp_new
};
PyTypeObject PyDict_Type = {
    PyVarObject_HEAD_INIT(&PyType_Type, 0)
    "dict",
    sizeof(PyDictObject),
    0,
    // ...
    mapp_methods,                         // tp_methods
    0,                                    // tp_members
    (hashfunc)PyObject_HashNotImplemented,// tp_hash 
    (reprfunc)dict_repr,                  // tp_repr
    offsetof(PyDictObject, dict),         // tp_dictoffset
    dict_new,                             // tp_new
};

Dict constructor

static PyObject *
dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
    PyObject *self;

    self = type->tp_alloc(type, 0);
    PyDictObject *d = (PyDictObject *)self;
    
    INIT_NONZERO_DICT_SLOTS(d);
    d->ma_lookup = lookdict_string;

    return self;
}
static PyObject *
dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
    PyObject *self;

    self = type->tp_alloc(type, 0);
    PyDictObject *d = (PyDictObject *)self;
    
    INIT_NONZERO_DICT_SLOTS(d);
    d->ma_lookup = lookdict_string;
    
    PyObject **dictptr = _PyObject_GetDictPtr(self);
    *dictptr = self;

    return self;
}

Does it work?

Yes!

>>> resp = {'status_code': 200}
>>> resp.status_code
200
>>> resp["content"] = "hi there!"
>>> resp
{'content': 'hi there!', 'status_code': 200}

My very own improved  fork of Python!

I'm calling it "mython"

Next time

  • Add obj["item"] access to all objects
  • Add implicit type conversions
  • Add "this" sometimes pointing to "self"
  • Replace classes by prototypes

https://github.com/mvschaik/mython

Thanks!

A Bit of Python Hacking

By maarten

Made with Slides.com

A Bit of Python Hacking

  • 189

More from maarten