multiprocessing module - isn't it a bug?

dmitrey dmitrey.kroshko at scipy.org
Sat Mar 14 13:31:09 EDT 2009


# THIS WORKS OK
from multiprocessing import Pool
N = 400
K = 800
processes = 2

def costlyFunction2(z):
    r = 0
    for k in xrange(1, K+2):
        r += z ** (1 / k**1.5)
    return r

class ABC:
    def __init__(self): pass
    def testParallel(self):
        po = Pool(processes=processes)
        r = po.map(costlyFunction2, xrange(N), chunksize = N/
(10*processes))
A=ABC()
A.testParallel()
print 'done'

# But when I define costlyFunction2 inside of class, it doesn't work:
from multiprocessing import Pool
N = 400
K = 800
processes = 2
class ABC:
    def __init__(self): pass
    def testParallel(self):
        po = Pool(processes=processes)
        def costlyFunction2(z):
            r = 0
            for k in xrange(1, K+2):
                r += z ** (1 / k**1.5)
            return r
        r = po.map(costlyFunction2, xrange(N), chunksize = N/
(10*processes))
A=ABC()
A.testParallel()
print 'done'

Exception in thread Thread-1:
Traceback (most recent call last):
  File "/usr/lib/python2.6/threading.py", line 522, in
__bootstrap_inner
    self.run()
  File "/usr/lib/python2.6/threading.py", line 477, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/lib/python2.6/multiprocessing/pool.py", line 225, in
_handle_tasks
    put(task)
PicklingError: Can't pickle <type 'function'>: attribute lookup
__builtin__.function failed

This doesn't work for
costlyFunction2 = lambda x: 11
as well; and it doesn't work for imap, apply_async as well (same
error).
So, isn't it a bug, or it can be somehow fixed?
Thank you in advance, D.



More information about the Python-list mailing list