[pypy-commit] pypy numpy-multidim-shards: Implement shards correctly. Allow fortran and C order. Temporarily replace repr
fijal
noreply at buildbot.pypy.org
Thu Nov 17 08:26:06 CET 2011
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim-shards
Changeset: r49486:09ed665d55c6
Date: 2011-11-17 09:25 +0200
http://bitbucket.org/pypy/pypy/changeset/09ed665d55c6/
Log: Implement shards correctly. Allow fortran and C order. Temporarily
replace repr and str with something that works.
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,6 +1,6 @@
from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.error import OperationError, operationerrfmt
+from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
from pypy.interpreter.typedef import TypeDef, GetSetProperty
from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
from pypy.rlib import jit
@@ -39,7 +39,8 @@
shape.append(size)
batch = new_batch
-def descr_new_array(space, w_subtype, w_item_or_iterable, w_dtype=None):
+def descr_new_array(space, w_subtype, w_item_or_iterable, w_dtype=None,
+ w_order=NoneNotWrapped):
# find scalar
if not space.issequence_w(w_item_or_iterable):
w_dtype = interp_ufuncs.find_dtype_for_scalar(space,
@@ -48,7 +49,15 @@
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype))
return scalar_w(space, dtype, w_item_or_iterable)
+ if w_order is None:
+ order = 'C'
+ else:
+ order = space.str_w(w_order)
+ if order != 'C': # or order != 'F':
+ raise operationerrfmt(space.w_ValueError, "Unknown order: %s",
+ order)
shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable)
+ # they come back in C order
size = len(elems_w)
if space.is_w(w_dtype, space.w_None):
w_dtype = None
@@ -62,11 +71,12 @@
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
- arr = NDimArray(size, shape[:], dtype=dtype)
- i = 0
+ arr = NDimArray(size, shape[:], dtype=dtype, order=order)
+ arr_iter = arr.start_iter()
for i in range(len(elems_w)):
w_elem = elems_w[i]
- dtype.setitem_w(space, arr.storage, i, w_elem)
+ dtype.setitem_w(space, arr.storage, arr_iter.offset, w_elem)
+ arr_iter = arr_iter.next()
return arr
class BaseIterator(object):
@@ -109,7 +119,7 @@
indices = self.indices[:]
done = False
offset = self.offset
- for i in range(len(self.indices)):
+ for i in range(len(self.indices) -1, -1, -1):
if indices[i] < self.arr.shape[i] - 1:
indices[i] += 1
offset += self.arr.shards[i]
@@ -168,28 +178,31 @@
class BaseArray(Wrappable):
_attrs_ = ["invalidates", "signature", "shape", "shards", "backshards",
- "start"]
+ "start", 'order']
#_immutable_fields_ = ['shape[*]', "shards[*]", "backshards[*]", 'start']
shards = None
start = 0
- def __init__(self, shape):
+ def __init__(self, shape, order):
self.invalidates = []
self.shape = shape
+ self.order = order
if self.shards is None:
self.shards = []
self.backshards = []
s = 1
shape_rev = shape[:]
- shape_rev.reverse()
+ if order == 'C':
+ shape_rev.reverse()
for sh in shape_rev:
self.shards.append(s)
self.backshards.append(s * (sh - 1))
s *= sh
- self.shards.reverse()
- self.backshards.reverse()
+ if order == 'C':
+ self.shards.reverse()
+ self.backshards.reverse()
def invalidated(self):
if self.invalidates:
@@ -340,9 +353,23 @@
def descr_repr(self, space):
res = StringBuilder()
+ concrete = self.get_concrete()
+ i = concrete.start_iter()
+ start = True
+ dtype = self.find_dtype()
+ while not i.done():
+ if start:
+ start = False
+ else:
+ res.append(", ")
+ res.append(dtype.str_format(concrete.getitem(i.offset)))
+ i = i.next()
+ return space.wrap(res.build())
+
+ res = StringBuilder()
res.append("array([")
concrete = self.get_concrete()
- i = concrete.start_iter(offset=0, indices=[0])
+ i = concrete.start_iter()#offset=0, indices=[0])
start = True
dtype = concrete.find_dtype()
if not concrete.find_size():
@@ -386,7 +413,7 @@
if ndims > 1:
builder.append('[')
builder.append("xxx")
- i = self.start_iter(offest=0, indices=[0])
+ i = self.start_iter()
while not i.done():
i.to_str(comma, builder, indent=indent + ' ')
builder.append('\n')
@@ -416,6 +443,7 @@
return builder.build()
def descr_str(self, space):
+ return self.descr_repr(space)
# Simple implementation so that we can see the array.
# Since what we want is to print a plethora of 2d views, let
# a slice do the work for us.
@@ -425,7 +453,6 @@
return space.wrap(r.to_str(False, s))
def _index_of_single_item(self, space, w_idx):
- # we assume C ordering for now
if space.isinstance_w(w_idx, space.w_int):
idx = space.int_w(w_idx)
if not self.shape:
@@ -605,7 +632,7 @@
_attrs_ = ["dtype", "value", "shape"]
def __init__(self, dtype, value):
- BaseArray.__init__(self, [])
+ BaseArray.__init__(self, [], 'C')
self.dtype = dtype
self.value = value
@@ -631,8 +658,8 @@
"""
Class for representing virtual arrays, such as binary ops or ufuncs
"""
- def __init__(self, signature, shape, res_dtype):
- BaseArray.__init__(self, shape)
+ def __init__(self, signature, shape, res_dtype, order):
+ BaseArray.__init__(self, shape, order)
self.forced_result = None
self.signature = signature
self.res_dtype = res_dtype
@@ -688,8 +715,9 @@
class Call1(VirtualArray):
- def __init__(self, signature, shape, res_dtype, values):
- VirtualArray.__init__(self, signature, shape, res_dtype)
+ def __init__(self, signature, shape, res_dtype, values, order):
+ VirtualArray.__init__(self, signature, shape, res_dtype,
+ values.order)
self.values = values
def _del_sources(self):
@@ -720,7 +748,8 @@
Intermediate class for performing binary operations.
"""
def __init__(self, signature, shape, calc_dtype, res_dtype, left, right):
- VirtualArray.__init__(self, signature, shape, res_dtype)
+ # XXX do something if left.order != right.order
+ VirtualArray.__init__(self, signature, shape, res_dtype, left.order)
self.left = left
self.right = right
self.calc_dtype = calc_dtype
@@ -759,7 +788,7 @@
def __init__(self, parent, signature, shards, backshards, shape):
self.shards = shards
self.backshards = backshards
- BaseArray.__init__(self, shape)
+ BaseArray.__init__(self, shape, parent.order)
self.signature = signature
self.parent = parent
self.invalidates = parent.invalidates
@@ -851,8 +880,8 @@
""" A class representing contiguous array. We know that each iteration
by say ufunc will increase the data index by one
"""
- def __init__(self, size, shape, dtype):
- BaseArray.__init__(self, shape)
+ def __init__(self, size, shape, dtype, order='C'):
+ BaseArray.__init__(self, shape, order)
self.size = size
self.dtype = dtype
self.storage = dtype.malloc(size)
@@ -892,7 +921,9 @@
self.dtype.setitem(self.storage, item, value)
def start_iter(self, offset=0, indices=None):
- return ArrayIterator(self.size, offset=offset)
+ if self.order == 'C':
+ return ArrayIterator(self.size, offset=offset)
+ raise NotImplementedError # use ViewIterator simply, test it
def __del__(self):
lltype.free(self.storage, flavor='raw', track_allocation=False)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -104,7 +104,7 @@
return self.func(res_dtype, w_obj.value.convert_to(res_dtype)).wrap(space)
new_sig = signature.Signature.find_sig([self.signature, w_obj.signature])
- w_res = Call1(new_sig, w_obj.shape, res_dtype, w_obj)
+ w_res = Call1(new_sig, w_obj.shape, res_dtype, w_obj, w_obj.order)
w_obj.add_invalidates(w_res)
return w_res
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -21,14 +21,41 @@
args_w.append(arg)
return self.space.newtuple(args_w)
- def test_shards(self):
- a = NDimArray(100, [10, 5, 3], MockDtype())
+ def test_shards_f(self):
+ a = NDimArray(100, [10, 5, 3], MockDtype(), 'F')
+ assert a.shards == [1, 10, 50]
+ assert a.backshards == [9, 40, 100]
+
+ def test_shards_c(self):
+ a = NDimArray(100, [10, 5, 3], MockDtype(), 'C')
assert a.shards == [15, 3, 1]
assert a.backshards == [135, 12, 2]
- def test_create_slice(self):
+ def test_create_slice_f(self):
space = self.space
- a = NDimArray(10*5*3, [10, 5, 3], MockDtype())
+ a = NDimArray(10*5*3, [10, 5, 3], MockDtype(), 'F')
+ s = a.create_slice(space, space.wrap(3))
+ assert s.start == 3
+ assert s.shards == [10, 50]
+ assert s.backshards == [40, 100]
+ s = a.create_slice(space, self.newslice(1, 9, 2))
+ assert s.start == 1
+ assert s.shards == [2, 10, 50]
+ assert s.backshards == [6, 40, 100]
+ s = a.create_slice(space, space.newtuple([
+ self.newslice(1, 5, 3), self.newslice(1, 2, 1), space.wrap(1)]))
+ assert s.start == 61
+ assert s.shape == [2, 1]
+ assert s.shards == [3, 10]
+ assert s.backshards == [3, 0]
+ s = a.create_slice(space, self.newtuple(
+ self.newslice(None, None, None), space.wrap(2)))
+ assert s.start == 20
+ assert s.shape == [10, 3]
+
+ def test_create_slice_c(self):
+ space = self.space
+ a = NDimArray(10*5*3, [10, 5, 3], MockDtype(), 'C')
s = a.create_slice(space, space.wrap(3))
assert s.start == 45
assert s.shards == [3, 1]
@@ -48,9 +75,28 @@
assert s.start == 6
assert s.shape == [10, 3]
- def test_slice_of_slice(self):
+ def test_slice_of_slice_f(self):
space = self.space
- a = NDimArray(10*5*3, [10, 5, 3], MockDtype())
+ a = NDimArray(10*5*3, [10, 5, 3], MockDtype(), 'F')
+ s = a.create_slice(space, space.wrap(5))
+ assert s.start == 5
+ s2 = s.create_slice(space, space.wrap(3))
+ assert s2.shape == [3]
+ assert s2.shards == [50]
+ assert s2.parent is a
+ assert s2.backshards == [100]
+ assert s2.start == 35
+ s = a.create_slice(space, self.newslice(1, 5, 3))
+ s2 = s.create_slice(space, space.newtuple([
+ self.newslice(None, None, None), space.wrap(2)]))
+ assert s2.shape == [2, 3]
+ assert s2.shards == [3, 50]
+ assert s2.backshards == [3, 100]
+ assert s2.start == 1*15 + 2*3
+
+ def test_slice_of_slice_c(self):
+ space = self.space
+ a = NDimArray(10*5*3, [10, 5, 3], MockDtype(), order='C')
s = a.create_slice(space, space.wrap(5))
assert s.start == 15*5
s2 = s.create_slice(space, space.wrap(3))
@@ -67,16 +113,35 @@
assert s2.backshards == [45, 2]
assert s2.start == 1*15 + 2*3
- def test_negative_step(self):
+ def test_negative_step_f(self):
space = self.space
- a = NDimArray(10*5*3, [10, 5, 3], MockDtype())
+ a = NDimArray(10*5*3, [10, 5, 3], MockDtype(), 'F')
+ s = a.create_slice(space, self.newslice(None, None, -2))
+ assert s.start == 9
+ assert s.shards == [-2, 10, 50]
+ assert s.backshards == [-8, 40, 100]
+
+ def test_negative_step_c(self):
+ space = self.space
+ a = NDimArray(10*5*3, [10, 5, 3], MockDtype(), order='C')
s = a.create_slice(space, self.newslice(None, None, -2))
assert s.start == 135
assert s.shards == [-30, 3, 1]
assert s.backshards == [-120, 12, 2]
- def test_index_of_single_item(self):
- a = NDimArray(10*5*3, [10, 5, 3], MockDtype())
+ def test_index_of_single_item_f(self):
+ a = NDimArray(10*5*3, [10, 5, 3], MockDtype(), 'F')
+ r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
+ assert r == 1 + 2 * 10 + 2 * 50
+ s = a.create_slice(self.space, self.newtuple(
+ self.newslice(None, None, None), 2))
+ r = s._index_of_single_item(self.space, self.newtuple(1, 0))
+ assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
+ r = s._index_of_single_item(self.space, self.newtuple(1, 1))
+ assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 1))
+
+ def test_index_of_single_item_c(self):
+ a = NDimArray(10*5*3, [10, 5, 3], MockDtype(), 'C')
r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
assert r == 1 * 3 * 5 + 2 * 3 + 2
s = a.create_slice(self.space, self.newtuple(
@@ -675,6 +740,7 @@
assert numpy.zeros(1).shape == (1,)
assert numpy.zeros((2, 2)).shape == (2,2)
assert numpy.zeros((3, 1, 2)).shape == (3, 1, 2)
+ assert numpy.array([[1], [2], [3]]).shape == (3, 1)
assert len(numpy.zeros((3, 1, 2))) == 3
raises(TypeError, len, numpy.zeros(()))
@@ -770,6 +836,8 @@
from numpy import array
a = array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14]])
b = a[::2]
+ print a
+ print b
assert (b == [[1, 2], [5, 6], [9, 10], [13, 14]]).all()
c = b + b
assert c[1][1] == 12
More information about the pypy-commit
mailing list