[pypy-issue] [issue1484] numpy pickling

[mikefc]

mikefc <coolbutuseless at gmail.com> added the comment:

Theres a weird bug with pickling and numpy operations. New code and output attached.

#========================================================
# Code
#========================================================
import sys
try:
        import numpypy as np
except:
        import numpy as np
import multiprocessing

def f1(x): pass
def f2(x): return 1
def f3(x): return x
def f4(x): return x + 1
def f5(x): return np.sum(x)

a = [np.zeros(3) for _ in xrange(4)]

# All five functions work with the builtin map
# but a multiprocessing map (which has to pickle objects) fails on f5
for map in (map, multiprocessing.Pool(2).map):
    print "f1:", map(f1, a); sys.stdout.flush()
    print "f2:", map(f2, a); sys.stdout.flush()
    print "f3:", map(f3, a); sys.stdout.flush()
    print "f4:", map(f4, a); sys.stdout.flush()
    print "f5:", map(f5, a); sys.stdout.flush() # FAILS with multiprocessing.map



#========================================================
# Output (trimmed. full output attached)
#========================================================
f1: [None, None, None, None]
f2: [1, 1, 1, 1]
f3: [array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.])]
f4: [array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.])]
f5: [0.0, 0.0, 0.0, 0.0]


f1: [None, None, None, None]
f2: [1, 1, 1, 1]
f3: [array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.])]
f4: [array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.])]
...

  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 600, in save_list
    self._batch_appends(iter(obj))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 636, in _batch_appends
    save(tmp[0])
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 306, in save
    rv = reduce(self.proto)
TypeError: unsupported operand type for __getnewargs__: 'float64'

________________________________________
PyPy bug tracker <tracker at bugs.pypy.org>
<https://bugs.pypy.org/issue1484>
________________________________________
-------------- next part --------------
f1: [None, None, None, None]
f2: [1, 1, 1, 1]
f3: [array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.])]
f4: [array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.])]
f5: [0.0, 0.0, 0.0, 0.0]
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
f1: [None, None, None, None]
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
f2: [1, 1, 1, 1]
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
f3: [array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.]), array([ 0.,  0.,  0.])]
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
f4: [array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.]), array([ 1.,  1.,  1.])]
<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
Process PoolWorker-1:
Traceback (most recent call last):
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/process.py", line 258, in _bootstrap
Process PoolWorker-2:
Traceback (most recent call last):
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/pool.py", line 99, in worker
    self.run()
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/pool.py", line 99, in worker
    put((job, i, result))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/queues.py", line 390, in put
    put((job, i, result))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/queues.py", line 390, in put
    return send(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 1417, in dumps
    return send(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 1417, in dumps
    Pickler(file, protocol).dump(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 224, in dump
    Pickler(file, protocol).dump(obj)
    self.save(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 224, in dump
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
    self.save(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 548, in save_tuple
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 548, in save_tuple
    save(element)
    save(element)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 548, in save_tuple
    save(element)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 600, in save_list
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 548, in save_tuple
    self._batch_appends(iter(obj))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 636, in _batch_appends
    save(element)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    save(tmp[0])
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 306, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 600, in save_list
    rv = reduce(self.proto)
    self._batch_appends(iter(obj))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 636, in _batch_appends
TypeError: unsupported operand type for __getnewargs__: 'float64'
    save(tmp[0])
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 306, in save
    rv = reduce(self.proto)
TypeError: unsupported operand type for __getnewargs__: 'float64'
f5:<Float64 object at 0x101755110>
<Float64 object at 0x101755110>
Process PoolWorker-3:
Traceback (most recent call last):
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/process.py", line 258, in _bootstrap
Process PoolWorker-4:
    self.run()
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/process.py", line 114, in run
Traceback (most recent call last):
    self._target(*self._args, **self._kwargs)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/pool.py", line 99, in worker
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/process.py", line 258, in _bootstrap
    put((job, i, result))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/queues.py", line 390, in put
    self.run()
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/pool.py", line 99, in worker
    return send(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 1417, in dumps
    put((job, i, result))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/multiprocessing/queues.py", line 390, in put
    return send(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 1417, in dumps
    Pickler(file, protocol).dump(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 224, in dump
    self.save(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 548, in save_tuple
    save(element)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 548, in save_tuple
    save(element)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 600, in save_list
    self._batch_appends(iter(obj))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 636, in _batch_appends
    Pickler(file, protocol).dump(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 224, in dump
    save(tmp[0])
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 306, in save
    self.save(obj)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    rv = reduce(self.proto)
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 548, in save_tuple
TypeError: unsupported operand type for __getnewargs__: 'float64'
    save(element)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 548, in save_tuple
    save(element)
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 286, in save
    f(self, obj) # Call unbound method with explicit self
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 600, in save_list
    self._batch_appends(iter(obj))
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 636, in _batch_appends
    save(tmp[0])
  File "/Users/mike/python/pypy-latest/lib-python/2.7/pickle.py", line 306, in save
    rv = reduce(self.proto)
TypeError: unsupported operand type for __getnewargs__: 'float64'