[pypy-commit] pypy default: Merge numpy-subarrays
rguillebert
noreply at buildbot.pypy.org
Wed May 15 16:10:36 CEST 2013
Author: Romain Guillebert <romain.py at gmail.com>
Branch:
Changeset: r64166:1168b982d455
Date: 2013-05-15 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/1168b982d455/
Log: Merge numpy-subarrays
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,6 @@
.. branch: numpy-pickle
Pickling of numpy arrays and dtypes (including record dtypes)
+
+.. branch: numpy-subarrays
+It is now possible to create arrays and dtypes that use subarrays
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -11,7 +11,6 @@
from rpython.rtyper.lltypesystem import rffi, lltype
from rpython.rlib.rawstorage import free_raw_storage, raw_storage_getitem,\
raw_storage_setitem, RAW_STORAGE
-from pypy.module.micronumpy.arrayimpl.sort import argsort_array
from rpython.rlib.debug import make_sure_not_resized
@@ -324,6 +323,7 @@
orig_array)
def argsort(self, space, w_axis):
+ from pypy.module.micronumpy.arrayimpl.sort import argsort_array
return argsort_array(self, space, w_axis)
def base(self):
diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py
--- a/pypy/module/micronumpy/arrayimpl/sort.py
+++ b/pypy/module/micronumpy/arrayimpl/sort.py
@@ -12,7 +12,7 @@
from rpython.rlib.objectmodel import specialize
from pypy.interpreter.error import OperationError
from pypy.module.micronumpy.base import W_NDimArray
-from pypy.module.micronumpy import interp_dtype, types
+from pypy.module.micronumpy import types
from pypy.module.micronumpy.iter import AxisIterator
INT_SIZE = rffi.sizeof(lltype.Signed)
@@ -20,7 +20,7 @@
def make_sort_function(space, itemtype, comp_type, count=1):
TP = itemtype.T
step = rffi.sizeof(TP)
-
+
class Repr(object):
def __init__(self, index_stride_size, stride_size, size, values,
indexes, index_start, start):
@@ -69,12 +69,13 @@
class ArgArrayRepWithStorage(Repr):
def __init__(self, index_stride_size, stride_size, size):
+ from pypy.module.micronumpy import interp_dtype
start = 0
dtype = interp_dtype.get_dtype_cache(space).w_longdtype
self.indexes = dtype.itemtype.malloc(size*dtype.get_size())
- self.values = alloc_raw_storage(size * stride_size,
+ self.values = alloc_raw_storage(size * stride_size,
track_allocation=False)
- Repr.__init__(self, index_stride_size, stride_size,
+ Repr.__init__(self, index_stride_size, stride_size,
size, self.values, self.indexes, start, start)
def __del__(self):
@@ -96,7 +97,7 @@
for i in range(stop-start):
retval.setitem(i, lst.getitem(i+start))
return retval
-
+
if count < 2:
def arg_lt(a, b):
# Does numpy do <= ?
@@ -108,13 +109,14 @@
return True
elif a[0][i] > b[0][i]:
return False
- # Does numpy do True?
+ # Does numpy do True?
return False
ArgSort = make_timsort_class(arg_getitem, arg_setitem, arg_length,
arg_getitem_slice, arg_lt)
def argsort(arr, space, w_axis, itemsize):
+ from pypy.module.micronumpy import interp_dtype
if w_axis is space.w_None:
# note that it's fine ot pass None here as we're not going
# to pass the result around (None is the link to base in slices)
@@ -180,7 +182,7 @@
class SortCache(object):
built = False
-
+
def __init__(self, space):
if self.built:
return
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -46,11 +46,11 @@
class W_Dtype(W_Root):
- _immutable_fields_ = ["itemtype", "num", "kind"]
+ _immutable_fields_ = ["itemtype", "num", "kind", "shape"]
def __init__(self, itemtype, num, kind, name, char, w_box_type,
alternate_constructors=[], aliases=[],
- fields=None, fieldnames=None, native=True):
+ fields=None, fieldnames=None, native=True, shape=[], subdtype=None):
self.itemtype = itemtype
self.num = num
self.kind = kind
@@ -63,6 +63,8 @@
self.fieldnames = fieldnames
self.native = native
self.float_type = None
+ self.shape = list(shape)
+ self.subdtype = subdtype
@specialize.argtype(1)
def box(self, value):
@@ -111,8 +113,12 @@
def descr_get_alignment(self, space):
return space.wrap(self.itemtype.alignment)
+ def descr_get_subdtype(self, space):
+ return space.newtuple([space.wrap(self.subdtype), self.descr_get_shape(space)])
+
def descr_get_shape(self, space):
- return space.newtuple([])
+ w_shape = [space.wrap(dim) for dim in self.shape]
+ return space.newtuple(w_shape)
def eq(self, space, w_other):
w_other = space.call_function(space.gettypefor(W_Dtype), w_other)
@@ -279,15 +285,22 @@
ofs_and_items = []
fieldnames = []
for w_elem in lst_w:
- w_fldname, w_flddesc = space.fixedview(w_elem, 2)
- subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc)
+ size = 1
+ w_shape = space.newtuple([])
+ if space.len_w(w_elem) == 3:
+ w_fldname, w_flddesc, w_shape = space.fixedview(w_elem)
+ if not base.issequence_w(space, w_shape):
+ w_shape = space.newtuple([w_shape,])
+ else:
+ w_fldname, w_flddesc = space.fixedview(w_elem)
+ subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc, w_shape=w_shape)
fldname = space.str_w(w_fldname)
if fldname in fields:
raise OperationError(space.w_ValueError, space.wrap("two fields with the same name"))
assert isinstance(subdtype, W_Dtype)
fields[fldname] = (offset, subdtype)
ofs_and_items.append((offset, subdtype.itemtype))
- offset += subdtype.itemtype.get_element_size()
+ offset += subdtype.itemtype.get_element_size() * size
fieldnames.append(fldname)
itemtype = types.RecordType(ofs_and_items, offset)
return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
@@ -333,10 +346,24 @@
raise OperationError(space.w_NotImplementedError, space.wrap(
"dtype from spec"))
-def descr__new__(space, w_subtype, w_dtype, w_align=None, w_copy=None):
+def descr__new__(space, w_subtype, w_dtype, w_align=None, w_copy=None, w_shape=None):
# w_align and w_copy are necessary for pickling
cache = get_dtype_cache(space)
+ if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or space.len_w(w_shape) > 0):
+ subdtype = descr__new__(space, w_subtype, w_dtype, w_align, w_copy)
+ assert isinstance(subdtype, W_Dtype)
+ size = 1
+ if space.isinstance_w(w_shape, space.w_int):
+ w_shape = space.newtuple([w_shape])
+ shape = []
+ for w_dim in space.fixedview(w_shape):
+ dim = space.int_w(w_dim)
+ shape.append(dim)
+ size *= dim
+ return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), 20, VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size),
+ "V", space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype)
+
if space.is_none(w_dtype):
return cache.w_float64dtype
elif space.isinstance_w(w_dtype, w_subtype):
@@ -355,6 +382,8 @@
"data type %s not understood" % name))
elif space.isinstance_w(w_dtype, space.w_list):
return dtype_from_list(space, w_dtype)
+ elif space.isinstance_w(w_dtype, space.w_tuple):
+ return descr__new__(space, w_subtype, space.getitem(w_dtype, space.wrap(0)), w_align, w_copy, w_shape=space.getitem(w_dtype, space.wrap(1)))
elif space.isinstance_w(w_dtype, space.w_dict):
return dtype_from_dict(space, w_dtype)
for dtype in cache.builtin_dtypes:
@@ -391,6 +420,7 @@
name = interp_attrproperty('name', cls=W_Dtype),
fields = GetSetProperty(W_Dtype.descr_get_fields),
names = GetSetProperty(W_Dtype.descr_get_names),
+ subdtype = GetSetProperty(W_Dtype.descr_get_subdtype),
)
W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -723,7 +723,7 @@
x = int8(42).ravel()
assert x.dtype == int8
assert (x == array(42)).all()
-
+
class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
@@ -790,6 +790,26 @@
d = dtype({'names': ['a', 'b', 'c'],
})
+ def test_create_subarrays(self):
+ from numpypy import dtype
+ d = dtype([("x", "float", (2,)), ("y", "int", (2,))])
+ assert d.itemsize == 32
+ assert d.name == "void256"
+ keys = d.fields.keys()
+ assert "x" in keys
+ assert "y" in keys
+ assert d["x"].shape == (2,)
+ assert d["x"].itemsize == 16
+ e = dtype([("x", "float", 2), ("y", "int", 2)])
+ assert e.fields.keys() == keys
+ assert e['x'].shape == (2,)
+
+ dt = dtype((float, 10))
+ assert dt.shape == (10,)
+ assert dt.kind == 'V'
+ assert dt.fields == None
+ assert dt.subdtype == (dtype("float64"), (10,))
+
class AppTestNotDirect(BaseNumpyAppTest):
def setup_class(cls):
BaseNumpyAppTest.setup_class.im_func(cls)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2699,6 +2699,20 @@
assert a[0]['y'] == 2
assert a[1]['y'] == 1
+ def test_subarrays(self):
+ from numpypy import dtype, array
+
+ d = dtype([("x", "int", 3), ("y", "float", 5)])
+ a = array([([1, 2, 3], [0.5, 1.5, 2.5, 3.5, 4.5]), ([4, 5, 6], [5.5, 6.5, 7.5, 8.5, 9.5])], dtype=d)
+
+ assert (a[0]["x"] == [1, 2, 3]).all()
+ assert (a[0]["y"] == [0.5, 1.5, 2.5, 3.5, 4.5]).all()
+ assert (a[1]["x"] == [4, 5, 6]).all()
+ assert (a[1]["y"] == [5.5, 6.5, 7.5, 8.5, 9.5]).all()
+
+ a[0]["x"][0] = 200
+ assert a[0]["x"][0] == 200
+
class AppTestPyPy(BaseNumpyAppTest):
def setup_class(cls):
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -3,7 +3,9 @@
from pypy.interpreter.error import OperationError
from pypy.module.micronumpy import interp_boxes
+from pypy.module.micronumpy import support
from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage
+from pypy.module.micronumpy.arrayimpl.concrete import SliceArray
from pypy.objspace.std.floatobject import float2string
from pypy.objspace.std.complexobject import str_format
from rpython.rlib import rfloat, clibffi, rcomplex
@@ -1076,7 +1078,7 @@
def to_builtin_type(self, space, box):
real,imag = self.for_computation(self.unbox(box))
- return space.newcomplex(real, imag)
+ return space.newcomplex(real, imag)
def read_bool(self, arr, i, offset):
v = self.for_computation(self._read(arr.storage, i, offset))
@@ -1217,7 +1219,7 @@
@raw_binary_op
def le(self, v1, v2):
- return self._lt(v1, v2) or self._eq(v1, v2)
+ return self._lt(v1, v2) or self._eq(v1, v2)
@raw_binary_op
def gt(self, v1, v2):
@@ -1225,7 +1227,7 @@
@raw_binary_op
def ge(self, v1, v2):
- return self._lt(v2, v1) or self._eq(v2, v1)
+ return self._lt(v2, v1) or self._eq(v2, v1)
def _bool(self, v):
return bool(v[0]) or bool(v[1])
@@ -1341,7 +1343,7 @@
return rcomplex.c_div((v[0], -v[1]), (a2, 0.))
except ZeroDivisionError:
return rfloat.NAN, rfloat.NAN
-
+
# No floor, ceil, trunc in numpy for complex
#@simple_unary_op
#def floor(self, v):
@@ -1696,10 +1698,36 @@
for j in range(i + 1, self.size):
arr.storage[j] = '\x00'
return interp_boxes.W_StringBox(arr, 0, arr.dtype)
-
+
class VoidType(BaseType, BaseStringType):
T = lltype.Char
+ def coerce(self, space, dtype, w_items):
+ items_w = space.fixedview(w_items)
+ arr = VoidBoxStorage(self.size, dtype)
+ ofs = 0
+ for i in range(len(items_w)):
+ subdtype = dtype.subdtype
+ itemtype = subdtype.itemtype
+ w_box = itemtype.coerce(space, dtype.subdtype, items_w[i])
+ itemtype.store(arr, 0, ofs, w_box)
+ ofs += itemtype.get_element_size()
+ return interp_boxes.W_VoidBox(arr, 0, dtype)
+
+ @jit.unroll_safe
+ def store(self, arr, i, ofs, box):
+ assert isinstance(box, interp_boxes.W_VoidBox)
+ for k in range(self.get_element_size()):
+ arr.storage[k + ofs] = box.arr.storage[k + box.ofs]
+
+ def read(self, arr, i, offset, dtype=None):
+ from pypy.module.micronumpy.base import W_NDimArray
+ if dtype is None:
+ dtype = arr.dtype
+ strides, backstrides = support.calc_strides(dtype.shape, dtype.subdtype, arr.order)
+ implementation = SliceArray(i + offset, strides, backstrides, dtype.shape, arr, arr, dtype.subdtype)
+ return W_NDimArray(implementation)
+
NonNativeVoidType = VoidType
NonNativeStringType = StringType
@@ -1733,7 +1761,7 @@
if not space.issequence_w(w_item):
raise OperationError(space.w_TypeError, space.wrap(
"expected sequence"))
- if len(self.offsets_and_fields) != space.int_w(space.len(w_item)):
+ if len(self.offsets_and_fields) != space.len_w(w_item):
raise OperationError(space.w_ValueError, space.wrap(
"wrong length"))
items_w = space.fixedview(w_item)
More information about the pypy-commit
mailing list