[Python-checkins] r68461 - in python/trunk: Doc/c-api/init.rst Doc/whatsnew/2.7.rst Lib/test/test_capi.py Modules/_testcapimodule.c
kristjan.jonsson
python-checkins at python.org
Fri Jan 9 22:35:16 CET 2009
Author: kristjan.jonsson
Date: Fri Jan 9 22:35:16 2009
New Revision: 68461
Log:
Issue 4293: Make Py_AddPendingCall() thread safe
Add test cases and documentation
Modified:
python/trunk/Doc/c-api/init.rst
python/trunk/Doc/whatsnew/2.7.rst
python/trunk/Lib/test/test_capi.py
python/trunk/Modules/_testcapimodule.c
Modified: python/trunk/Doc/c-api/init.rst
==============================================================================
--- python/trunk/Doc/c-api/init.rst (original)
+++ python/trunk/Doc/c-api/init.rst Fri Jan 9 22:35:16 2009
@@ -780,6 +780,50 @@
.. versionadded:: 2.3
+
+Asynchronous Notifications
+==========================
+
+A mechanism is provided to make asynchronous notifications to the the main
+interpreter thread. These notifications take the form of a function
+pointer and a void argument.
+
+.. index:: single: setcheckinterval() (in module sys)
+
+Every check interval, when the interpreter lock is released and reacquired,
+python will also call any such provided functions. This can be used for
+example by asynchronous IO handlers. The notification can be scheduled
+from a worker thread and the actual call than made at the earliest
+convenience by the main thread where it has possession of the global
+interpreter lock and can perform any Python API calls.
+
+.. cfunction:: void Py_AddPendingCall( int (*func)(void *), void *arg) )
+
+ .. index:: single: Py_AddPendingCall()
+
+ Post a notification to the Python main thread. If successful,
+ \*:attr`func` will be called with the argument :attr:`arg` at the earliest
+ convenience. \*:attr:`func` will be called having the global interpreter
+ lock held and can thus use the full Python API and can take any
+ action such as setting object attributes to signal IO completion.
+ It must return 0 on success, or -1 signalling an exception.
+ The notification function won't be interrupted to perform another
+ asynchronous notification recursively,
+ but it can still be interrupted to switch threads if the interpreter
+ lock is released, for example, if it calls back into python code.
+
+ This function returns 0 on success in which case the notification has been
+ scheduled. Otherwise, for example if the notification buffer is full,
+ it returns -1 without setting any exception.
+
+ This function can be called on any thread, be it a Python thread or
+ some other system thread. If it is a Python thread, it doesen't matter if
+ it holds the global interpreter lock or not.
+
+ .. versionadded:: 2.7
+
+
+
.. _profiling:
Profiling and Tracing
Modified: python/trunk/Doc/whatsnew/2.7.rst
==============================================================================
--- python/trunk/Doc/whatsnew/2.7.rst (original)
+++ python/trunk/Doc/whatsnew/2.7.rst Fri Jan 9 22:35:16 2009
@@ -60,6 +60,11 @@
.. ========================================================================
+Kristján Valur Jónsson, issue 4293
+Py_AddPendingCall is now thread safe. This allows any worker thread
+to submit notifications to the python main thread. This is particularly
+useful for asynchronous IO operations.
+
Other Language Changes
======================
Modified: python/trunk/Lib/test/test_capi.py
==============================================================================
--- python/trunk/Lib/test/test_capi.py (original)
+++ python/trunk/Lib/test/test_capi.py Fri Jan 9 22:35:16 2009
@@ -2,9 +2,74 @@
# these are all functions _testcapi exports whose name begins with 'test_'.
import sys
+import time
+import random
+import unittest
+import threading
from test import test_support
import _testcapi
+class TestPendingCalls(unittest.TestCase):
+
+ def pendingcalls_submit(self, l, n):
+ def callback():
+ #this function can be interrupted by thread switching so let's
+ #use an atomic operation
+ l.append(None)
+
+ for i in range(n):
+ time.sleep(random.random()*0.02) #0.01 secs on average
+ #try submitting callback until successful.
+ #rely on regular interrupt to flush queue if we are
+ #unsuccessful.
+ while True:
+ if _testcapi._pending_threadfunc(callback):
+ break;
+
+ def pendingcalls_wait(self, l, n):
+ #now, stick around until l[0] has grown to 10
+ count = 0;
+ while len(l) != n:
+ #this busy loop is where we expect to be interrupted to
+ #run our callbacks. Note that callbacks are only run on the
+ #main thread
+ if False and test_support.verbose:
+ print "(%i)"%(len(l),),
+ for i in xrange(1000):
+ a = i*i
+ count += 1
+ self.failUnless(count < 10000,
+ "timeout waiting for %i callbacks, got %i"%(n, len(l)))
+ if False and test_support.verbose:
+ print "(%i)"%(len(l),)
+
+ def test_pendingcalls_threaded(self):
+ l = []
+
+ #do every callback on a separate thread
+ n = 32
+ threads = []
+ for i in range(n):
+ t = threading.Thread(target=self.pendingcalls_submit, args = (l, 1))
+ t.start()
+ threads.append(t)
+
+ self.pendingcalls_wait(l, n)
+
+ for t in threads:
+ t.join()
+
+ def test_pendingcalls_non_threaded(self):
+ #again, just using the main thread, likely they will all be dispathced at
+ #once. It is ok to ask for too many, because we loop until we find a slot.
+ #the loop can be interrupted to dispatch.
+ #there are only 32 dispatch slots, so we go for twice that!
+ l = []
+ n = 64
+ self.pendingcalls_submit(l, n)
+ self.pendingcalls_wait(l, n)
+
+
def test_main():
for name in dir(_testcapi):
@@ -50,5 +115,7 @@
t.start()
t.join()
+ test_support.run_unittest(TestPendingCalls)
+
if __name__ == "__main__":
test_main()
Modified: python/trunk/Modules/_testcapimodule.c
==============================================================================
--- python/trunk/Modules/_testcapimodule.c (original)
+++ python/trunk/Modules/_testcapimodule.c Fri Jan 9 22:35:16 2009
@@ -837,6 +837,43 @@
return NULL;
Py_RETURN_NONE;
}
+
+/* test Py_AddPendingCalls using threads */
+static int _pending_callback(void *arg)
+{
+ /* we assume the argument is callable object to which we own a reference */
+ PyObject *callable = (PyObject *)arg;
+ PyObject *r = PyObject_CallObject(callable, NULL);
+ Py_DECREF(callable);
+ Py_XDECREF(r);
+ return r != NULL ? 0 : -1;
+}
+
+/* The following requests n callbacks to _pending_callback. It can be
+ * run from any python thread.
+ */
+PyObject *pending_threadfunc(PyObject *self, PyObject *arg)
+{
+ PyObject *callable;
+ int r;
+ if (PyArg_ParseTuple(arg, "O", &callable) == 0)
+ return NULL;
+
+ /* create the reference for the callbackwhile we hold the lock */
+ Py_INCREF(callable);
+
+ Py_BEGIN_ALLOW_THREADS
+ r = Py_AddPendingCall(&_pending_callback, callable);
+ Py_END_ALLOW_THREADS
+
+ if (r<0) {
+ Py_DECREF(callable); /* unsuccessful add, destroy the extra reference */
+ Py_INCREF(Py_False);
+ return Py_False;
+ }
+ Py_INCREF(Py_True);
+ return Py_True;
+}
#endif
/* Some tests of PyString_FromFormat(). This needs more tests. */
@@ -941,6 +978,7 @@
#endif
#ifdef WITH_THREAD
{"_test_thread_state", test_thread_state, METH_VARARGS},
+ {"_pending_threadfunc", pending_threadfunc, METH_VARARGS},
#endif
{"traceback_print", traceback_print, METH_VARARGS},
{NULL, NULL} /* sentinel */
More information about the Python-checkins
mailing list