is +=1 thread safe

[Carl Banks]

On May 4, 12:03 pm, Gary Herron <gher... at islandtraining.com> wrote:
> Alexander Schmolck wrote:
> > Gary Herron <gher... at islandtraining.com> writes:
>
> >> But... It's not!
>
> >> A simple test shows that.   I've attached a tiny test program that shows this
> >> extremely clearly.  Please run it and watch it fail.
>
> > In [7]: run ~/tmp/t.py
> > final count: 2000000
> >   should be: 2000000
>
> > (I took the liberty to correct your test to actually do what I said, namely
> > use a numpy.array; just replace ``count = 0`` with ``import numpy; count =
> > numpy.array(0)``).
>
> The test was meant to simulate the OP's problem, but even with your
> suggestion of using numpy, it *still* fails!


Ok, so numpy scalars don't support += atomically.  Thank you for your
skepticism in discovering this.

However, what I said was not wholly untrue: code in C extensions is
protected by the GIL and thus not interruptable, unless it either
releases the GIL, or calls back into Python code (which is apparently
what numpy scalars do).

To illustrate, a small extension module is included below.  Just to
make things interesting, I added a long busy loop in between reading
and setting the counter, just to give the other thread maximum
opportunity to cut in.

If you were to compile it and run the test, you would find that it
works perfectly.


===========================
/* atomic.c */

#include <Python.h>
#include <structmember.h>

typedef struct {
    PyObject_HEAD
    long value;
} AtomicCounterObject;

static int init(AtomicCounterObject* self, PyObject* args,
                      PyObject* kwargs) {
    self->value = 0;
    return 0;
}

static PyObject* iadd(AtomicCounterObject* self, PyObject* inc) {
    long incval = PyInt_AsLong(inc);
    long store, i;
    static int bigarray[100000];
    if (incval == -1 && PyErr_Occurred()) return 0;

    store = self->value;

    /* Give the thread plenty of time to interrupt */
    for (i = 0; i < 100000; i++) bigarray[i]++;

    self->value = store + incval;

    return (PyObject*)self;
}

static PyObject* toint(AtomicCounterObject* self) {
    return PyInt_FromLong(self->value);
}

static PyNumberMethods ac_as_number = {
    0,                             /*nb_add*/
    0,                             /*nb_subtract*/
    0,                             /*nb_multiply*/
    0,                             /*nb_divide*/
    0,                             /*nb_remainder*/
    0,                             /*nb_divmod*/
    0,                             /*nb_power*/
    0,                             /*nb_negative*/
    0,                             /*nb_positive*/
    0,                             /*nb_absolute*/
    0,                             /*nb_nonzero*/
    0,                             /*nb_invert*/
    0,                             /*nb_lshift*/
    0,                             /*nb_rshift*/
    0,                             /*nb_and*/
    0,                             /*nb_xor*/
    0,                             /*nb_or*/
    0,                             /*nb_coerce*/
    (unaryfunc)toint,              /*nb_int*/
    0,                             /*nb_long*/
    0,                             /*nb_float*/
    0,                             /*nb_oct*/
    0,                             /*nb_hex*/
    (binaryfunc)iadd,              /*nb_inplace_add*/
    0,                             /*nb_inplace_subtract*/
    0,                             /*nb_inplace_multiply*/
    0,                             /*nb_inplace_divide*/
    0,                             /*nb_inplace_remainder*/
    0,                             /*nb_inplace_power*/
    0,                             /*nb_inplace_lshift*/
    0,                             /*nb_inplace_rshift*/
    0,                             /*nb_inplace_and*/
    0,                             /*nb_inplace_xor*/
    0,                             /*nb_inplace_or*/
    0,                             /* nb_floor_divide */
    0,                             /* nb_true_divide */
    0,                             /* nb_inplace_floor_divide */
    0,                             /* nb_inplace_true_divide */
    0,                             /* nb_index */
};

static PyTypeObject AtomicCounterType = {
    PyObject_HEAD_INIT(NULL)
    0,                             /*ob_size*/
    "AtomicCounter",               /*tp_name*/
    sizeof(AtomicCounterObject),   /*tp_basicsize*/
    0,                             /*tp_itemsize*/
    0,                             /*tp_dealloc*/
    0,                             /*tp_print*/
    0,                             /*tp_getattr*/
    0,                             /*tp_setattr*/
    0,                             /*tp_compare*/
    0,                             /*tp_repr*/
    &ac_as_number,                 /*tp_as_number*/
    0,                             /*tp_as_sequence*/
    0,                             /*tp_as_mapping*/
    0,                             /*tp_hash */
    0,                             /*tp_call*/
    0,                             /*tp_str*/
    0,                             /*tp_getattro*/
    0,                             /*tp_setattro*/
    0,                             /*tp_as_buffer*/
    Py_TPFLAGS_DEFAULT,            /*tp_flags*/
    0,                             /*tp_doc */
    0,                             /*tp_traverse*/
    0,                             /*tp_clear*/
    0,                             /*tp_richcompare*/
    0,                             /*tp_weaklistoffset*/
    0,                             /*tp_iter*/
    0,                             /*tp_iternext*/
    0,                             /*tp_methods*/
    0,                             /*tp_members*/
    0,                             /*tp_getset*/
    0,                             /*tp_base*/
    0,                             /*tp_dict*/
    0,                             /*tp_descr_get*/
    0,                             /*tp_descr_set*/
    0,                             /*tp_dictoffset*/
    (initproc)init,                /*tp_init*/
    0,
    PyType_GenericNew,
};


static PyMethodDef methods[] = {
    {0} /* sentinel */
};


PyMODINIT_FUNC initatomic(void) {
    PyObject* m;
    if (PyType_Ready(&AtomicCounterType) < 0)
        return;
    m = Py_InitModule("atomic", methods);
    if (m == NULL)
        return;
    Py_INCREF(&AtomicCounterType);
    PyModule_AddObject(m, "AtomicCounter", (PyObject
*)&AtomicCounterType);
}

========================
# actest.py

import threading

N = 200000
import atomic; count = atomic.AtomicCounter()

class timer(threading.Thread):
    def run(self):
        global count
        for i in xrange(N):
            count += 1

def test():
    thread1=timer()
    thread2=timer()
    thread1.start()
    thread2.start()
    thread1.join()
    thread2.join()
    print 'final count:', count
    print '  should be:', 2*N

if __name__=='__main__':
    test()
========================


Carl Banks