[pypy-commit] pypy default: (fijal, agaynor) Merge numpy-record-dtypes branch.
fijal
noreply at buildbot.pypy.org
Sun Mar 18 10:06:36 CET 2012
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch:
Changeset: r53783:c3a08f3e51fe
Date: 2012-03-18 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/c3a08f3e51fe/
Log: (fijal, agaynor) Merge numpy-record-dtypes branch.
This branch implements most of APIs for record dtypes as well as few
for string and unicode dtypes. This is not complete, but anyway most
of the stuff works nicely.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1336,7 +1336,7 @@
if not self.is_true(self.isinstance(w_obj, self.w_str)):
raise OperationError(self.w_TypeError,
self.wrap('argument must be a string'))
- return self.str_w(w_obj)
+ return self.str_w(w_obj)
def unicode_w(self, w_obj):
return w_obj.unicode_w(self)
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -37,26 +37,44 @@
'True_': 'types.Bool.True',
'False_': 'types.Bool.False',
+ 'typeinfo': 'interp_dtype.get_dtype_cache(space).w_typeinfo',
+
'generic': 'interp_boxes.W_GenericBox',
'number': 'interp_boxes.W_NumberBox',
'integer': 'interp_boxes.W_IntegerBox',
'signedinteger': 'interp_boxes.W_SignedIntegerBox',
'unsignedinteger': 'interp_boxes.W_UnsignedIntegerBox',
'bool_': 'interp_boxes.W_BoolBox',
+ 'bool8': 'interp_boxes.W_BoolBox',
'int8': 'interp_boxes.W_Int8Box',
+ 'byte': 'interp_boxes.W_Int8Box',
'uint8': 'interp_boxes.W_UInt8Box',
+ 'ubyte': 'interp_boxes.W_UInt8Box',
'int16': 'interp_boxes.W_Int16Box',
+ 'short': 'interp_boxes.W_Int16Box',
'uint16': 'interp_boxes.W_UInt16Box',
+ 'ushort': 'interp_boxes.W_UInt16Box',
'int32': 'interp_boxes.W_Int32Box',
+ 'intc': 'interp_boxes.W_Int32Box',
'uint32': 'interp_boxes.W_UInt32Box',
+ 'uintc': 'interp_boxes.W_UInt32Box',
'int64': 'interp_boxes.W_Int64Box',
'uint64': 'interp_boxes.W_UInt64Box',
+ 'longlong': 'interp_boxes.W_LongLongBox',
+ 'ulonglong': 'interp_boxes.W_ULongLongBox',
'int_': 'interp_boxes.W_LongBox',
'inexact': 'interp_boxes.W_InexactBox',
'floating': 'interp_boxes.W_FloatingBox',
'float_': 'interp_boxes.W_Float64Box',
'float32': 'interp_boxes.W_Float32Box',
'float64': 'interp_boxes.W_Float64Box',
+ 'intp': 'types.IntP.BoxType',
+ 'uintp': 'types.UIntP.BoxType',
+ 'flexible': 'interp_boxes.W_FlexibleBox',
+ 'character': 'interp_boxes.W_CharacterBox',
+ 'str_': 'interp_boxes.W_StringBox',
+ 'unicode_': 'interp_boxes.W_UnicodeBox',
+ 'void': 'interp_boxes.W_VoidBox',
}
# ufuncs
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -33,7 +33,7 @@
pass
SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
- "unegative", "flat"]
+ "unegative", "flat", "tostring"]
TWO_ARG_FUNCTIONS = ["dot", 'take']
class FakeSpace(object):
@@ -51,6 +51,8 @@
w_long = "long"
w_tuple = 'tuple'
w_slice = "slice"
+ w_str = "str"
+ w_unicode = "unicode"
def __init__(self):
"""NOT_RPYTHON"""
@@ -91,8 +93,12 @@
return BoolObject(obj)
elif isinstance(obj, int):
return IntObject(obj)
+ elif isinstance(obj, long):
+ return LongObject(obj)
elif isinstance(obj, W_Root):
return obj
+ elif isinstance(obj, str):
+ return StringObject(obj)
raise NotImplementedError
def newlist(self, items):
@@ -120,6 +126,11 @@
return int(w_obj.floatval)
raise NotImplementedError
+ def str_w(self, w_obj):
+ if isinstance(w_obj, StringObject):
+ return w_obj.v
+ raise NotImplementedError
+
def int(self, w_obj):
if isinstance(w_obj, IntObject):
return w_obj
@@ -151,7 +162,13 @@
return instantiate(klass)
def newtuple(self, list_w):
- raise ValueError
+ return ListObject(list_w)
+
+ def newdict(self):
+ return {}
+
+ def setitem(self, dict, item, value):
+ dict[item] = value
def len_w(self, w_obj):
if isinstance(w_obj, ListObject):
@@ -178,6 +195,11 @@
def __init__(self, intval):
self.intval = intval
+class LongObject(W_Root):
+ tp = FakeSpace.w_long
+ def __init__(self, intval):
+ self.intval = intval
+
class ListObject(W_Root):
tp = FakeSpace.w_list
def __init__(self, items):
@@ -190,6 +212,11 @@
self.stop = stop
self.step = step
+class StringObject(W_Root):
+ tp = FakeSpace.w_str
+ def __init__(self, v):
+ self.v = v
+
class InterpreterState(object):
def __init__(self, code):
self.code = code
@@ -407,6 +434,9 @@
w_res = neg.call(interp.space, [arr])
elif self.name == "flat":
w_res = arr.descr_get_flatiter(interp.space)
+ elif self.name == "tostring":
+ arr.descr_tostring(interp.space)
+ w_res = None
else:
assert False # unreachable code
elif self.name in TWO_ARG_FUNCTIONS:
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -1,24 +1,25 @@
from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.error import operationerrfmt
+from pypy.interpreter.error import operationerrfmt, OperationError
from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.typedef import TypeDef
from pypy.objspace.std.floattype import float_typedef
+from pypy.objspace.std.stringtype import str_typedef
+from pypy.objspace.std.unicodetype import unicode_typedef, unicode_from_object
from pypy.objspace.std.inttype import int_typedef
from pypy.rlib.rarithmetic import LONG_BIT
from pypy.tool.sourcetools import func_with_new_name
-
MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
def new_dtype_getter(name):
- def get_dtype(space):
+ def _get_dtype(space):
from pypy.module.micronumpy.interp_dtype import get_dtype_cache
return getattr(get_dtype_cache(space), "w_%sdtype" % name)
def new(space, w_subtype, w_value):
- dtype = get_dtype(space)
+ dtype = _get_dtype(space)
return dtype.itemtype.coerce_subtype(space, w_subtype, w_value)
- return func_with_new_name(new, name + "_box_new"), staticmethod(get_dtype)
+ return func_with_new_name(new, name + "_box_new"), staticmethod(_get_dtype)
class PrimitiveBox(object):
_mixin_ = True
@@ -37,6 +38,9 @@
w_subtype.getname(space, '?')
)
+ def get_dtype(self, space):
+ return self._get_dtype(space)
+
def descr_str(self, space):
return space.wrap(self.get_dtype(space).itemtype.str_format(self))
@@ -44,12 +48,12 @@
return space.format(self.item(space), w_spec)
def descr_int(self, space):
- box = self.convert_to(W_LongBox.get_dtype(space))
+ box = self.convert_to(W_LongBox._get_dtype(space))
assert isinstance(box, W_LongBox)
return space.wrap(box.value)
def descr_float(self, space):
- box = self.convert_to(W_Float64Box.get_dtype(space))
+ box = self.convert_to(W_Float64Box._get_dtype(space))
assert isinstance(box, W_Float64Box)
return space.wrap(box.value)
@@ -130,7 +134,7 @@
class W_BoolBox(W_GenericBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("bool")
+ descr__new__, _get_dtype = new_dtype_getter("bool")
class W_NumberBox(W_GenericBox):
_attrs_ = ()
@@ -146,34 +150,40 @@
pass
class W_Int8Box(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("int8")
+ descr__new__, _get_dtype = new_dtype_getter("int8")
class W_UInt8Box(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("uint8")
+ descr__new__, _get_dtype = new_dtype_getter("uint8")
class W_Int16Box(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("int16")
+ descr__new__, _get_dtype = new_dtype_getter("int16")
class W_UInt16Box(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("uint16")
+ descr__new__, _get_dtype = new_dtype_getter("uint16")
class W_Int32Box(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("int32")
+ descr__new__, _get_dtype = new_dtype_getter("int32")
class W_UInt32Box(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("uint32")
+ descr__new__, _get_dtype = new_dtype_getter("uint32")
class W_LongBox(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("long")
+ descr__new__, _get_dtype = new_dtype_getter("long")
class W_ULongBox(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("ulong")
+ descr__new__, _get_dtype = new_dtype_getter("ulong")
class W_Int64Box(W_SignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("int64")
+ descr__new__, _get_dtype = new_dtype_getter("int64")
+
+class W_LongLongBox(W_SignedIntegerBox, PrimitiveBox):
+ descr__new__, _get_dtype = new_dtype_getter('longlong')
class W_UInt64Box(W_UnsignedIntegerBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("uint64")
+ descr__new__, _get_dtype = new_dtype_getter("uint64")
+
+class W_ULongLongBox(W_SignedIntegerBox, PrimitiveBox):
+ descr__new__, _get_dtype = new_dtype_getter('ulonglong')
class W_InexactBox(W_NumberBox):
_attrs_ = ()
@@ -182,16 +192,71 @@
_attrs_ = ()
class W_Float32Box(W_FloatingBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("float32")
+ descr__new__, _get_dtype = new_dtype_getter("float32")
class W_Float64Box(W_FloatingBox, PrimitiveBox):
- descr__new__, get_dtype = new_dtype_getter("float64")
+ descr__new__, _get_dtype = new_dtype_getter("float64")
+class W_FlexibleBox(W_GenericBox):
+ def __init__(self, arr, ofs, dtype):
+ self.arr = arr # we have to keep array alive
+ self.ofs = ofs
+ self.dtype = dtype
+
+ def get_dtype(self, space):
+ return self.arr.dtype
+
@unwrap_spec(self=W_GenericBox)
def descr_index(space, self):
return space.index(self.item(space))
+class W_VoidBox(W_FlexibleBox):
+ @unwrap_spec(item=str)
+ def descr_getitem(self, space, item):
+ try:
+ ofs, dtype = self.dtype.fields[item]
+ except KeyError:
+ raise OperationError(space.w_IndexError,
+ space.wrap("Field %s does not exist" % item))
+ return dtype.itemtype.read(self.arr, 1, self.ofs, ofs, dtype)
+
+ @unwrap_spec(item=str)
+ def descr_setitem(self, space, item, w_value):
+ try:
+ ofs, dtype = self.dtype.fields[item]
+ except KeyError:
+ raise OperationError(space.w_IndexError,
+ space.wrap("Field %s does not exist" % item))
+ dtype.itemtype.store(self.arr, 1, self.ofs, ofs,
+ dtype.coerce(space, w_value))
+
+class W_CharacterBox(W_FlexibleBox):
+ pass
+
+class W_StringBox(W_CharacterBox):
+ def descr__new__string_box(space, w_subtype, w_arg):
+ from pypy.module.micronumpy.interp_numarray import W_NDimArray
+ from pypy.module.micronumpy.interp_dtype import new_string_dtype
+
+ arg = space.str_w(space.str(w_arg))
+ arr = W_NDimArray([1], new_string_dtype(space, len(arg)))
+ for i in range(len(arg)):
+ arr.storage[i] = arg[i]
+ return W_StringBox(arr, 0, arr.dtype)
+
+
+class W_UnicodeBox(W_CharacterBox):
+ def descr__new__unicode_box(space, w_subtype, w_arg):
+ from pypy.module.micronumpy.interp_numarray import W_NDimArray
+ from pypy.module.micronumpy.interp_dtype import new_unicode_dtype
+
+ arg = space.unicode_w(unicode_from_object(space, w_arg))
+ arr = W_NDimArray([1], new_unicode_dtype(space, len(arg)))
+ # XXX not this way, we need store
+ #for i in range(len(arg)):
+ # arr.storage[i] = arg[i]
+ return W_UnicodeBox(arr, 0, arr.dtype)
W_GenericBox.typedef = TypeDef("generic",
__module__ = "numpypy",
@@ -348,3 +413,28 @@
__new__ = interp2app(W_Float64Box.descr__new__.im_func),
)
+
+W_FlexibleBox.typedef = TypeDef("flexible", W_GenericBox.typedef,
+ __module__ = "numpypy",
+)
+
+W_VoidBox.typedef = TypeDef("void", W_FlexibleBox.typedef,
+ __module__ = "numpypy",
+ __getitem__ = interp2app(W_VoidBox.descr_getitem),
+ __setitem__ = interp2app(W_VoidBox.descr_setitem),
+)
+
+W_CharacterBox.typedef = TypeDef("character", W_FlexibleBox.typedef,
+ __module__ = "numpypy",
+)
+
+W_StringBox.typedef = TypeDef("string_", (str_typedef, W_CharacterBox.typedef),
+ __module__ = "numpypy",
+ __new__ = interp2app(W_StringBox.descr__new__string_box.im_func),
+)
+
+W_UnicodeBox.typedef = TypeDef("unicode_", (unicode_typedef, W_CharacterBox.typedef),
+ __module__ = "numpypy",
+ __new__ = interp2app(W_UnicodeBox.descr__new__unicode_box.im_func),
+)
+
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -1,26 +1,29 @@
+
+import sys
from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
interp_attrproperty, interp_attrproperty_w)
-from pypy.module.micronumpy import types, signature, interp_boxes
+from pypy.module.micronumpy import types, interp_boxes
from pypy.rlib.objectmodel import specialize
-from pypy.rlib.rarithmetic import LONG_BIT
-from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong
UNSIGNEDLTR = "u"
SIGNEDLTR = "i"
BOOLLTR = "b"
FLOATINGLTR = "f"
-
-
-VOID_STORAGE = lltype.Array(lltype.Char, hints={'nolength': True, 'render_as_void': True})
+VOIDLTR = 'V'
+STRINGLTR = 'S'
+UNICODELTR = 'U'
class W_Dtype(Wrappable):
_immutable_fields_ = ["itemtype", "num", "kind"]
- def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]):
+ def __init__(self, itemtype, num, kind, name, char, w_box_type,
+ alternate_constructors=[], aliases=[],
+ fields=None, fieldnames=None, native=True):
self.itemtype = itemtype
self.num = num
self.kind = kind
@@ -29,53 +32,28 @@
self.w_box_type = w_box_type
self.alternate_constructors = alternate_constructors
self.aliases = aliases
-
- def malloc(self, length):
- # XXX find out why test_zjit explodes with tracking of allocations
- return lltype.malloc(VOID_STORAGE, self.itemtype.get_element_size() * length,
- zero=True, flavor="raw",
- track_allocation=False, add_memory_pressure=True
- )
+ self.fields = fields
+ self.fieldnames = fieldnames
+ self.native = native
@specialize.argtype(1)
def box(self, value):
return self.itemtype.box(value)
def coerce(self, space, w_item):
- return self.itemtype.coerce(space, w_item)
+ return self.itemtype.coerce(space, self, w_item)
- def getitem(self, storage, i):
- return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0)
+ def getitem(self, arr, i):
+ return self.itemtype.read(arr, 1, i, 0)
- def getitem_bool(self, storage, i):
- isize = self.itemtype.get_element_size()
- return self.itemtype.read_bool(storage, isize, i, 0)
+ def getitem_bool(self, arr, i):
+ return self.itemtype.read_bool(arr, 1, i, 0)
- def setitem(self, storage, i, box):
- self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box)
+ def setitem(self, arr, i, box):
+ self.itemtype.store(arr, 1, i, 0, box)
def fill(self, storage, box, start, stop):
- self.itemtype.fill(storage, self.itemtype.get_element_size(), box, start, stop, 0)
-
- def descr__new__(space, w_subtype, w_dtype):
- cache = get_dtype_cache(space)
-
- if space.is_w(w_dtype, space.w_None):
- return cache.w_float64dtype
- elif space.isinstance_w(w_dtype, w_subtype):
- return w_dtype
- elif space.isinstance_w(w_dtype, space.w_str):
- name = space.str_w(w_dtype)
- for dtype in cache.builtin_dtypes:
- if dtype.name == name or dtype.char == name or name in dtype.aliases:
- return dtype
- else:
- for dtype in cache.builtin_dtypes:
- if w_dtype in dtype.alternate_constructors:
- return dtype
- if w_dtype is dtype.w_box_type:
- return dtype
- raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+ self.itemtype.fill(storage, self.get_size(), box, start, stop, 0)
def descr_str(self, space):
return space.wrap(self.name)
@@ -86,6 +64,14 @@
def descr_get_itemsize(self, space):
return space.wrap(self.itemtype.get_element_size())
+ def descr_get_byteorder(self, space):
+ if self.native:
+ return space.wrap('=')
+ return space.wrap(nonnative_byteorder_prefix)
+
+ def descr_get_alignment(self, space):
+ return space.wrap(self.itemtype.alignment)
+
def descr_get_shape(self, space):
return space.newtuple([])
@@ -99,31 +85,193 @@
def descr_ne(self, space, w_other):
return space.wrap(not self.eq(space, w_other))
+ def descr_get_fields(self, space):
+ if self.fields is None:
+ return space.w_None
+ w_d = space.newdict()
+ for name, (offset, subdtype) in self.fields.iteritems():
+ space.setitem(w_d, space.wrap(name), space.newtuple([subdtype,
+ space.wrap(offset)]))
+ return w_d
+
+ def descr_get_names(self, space):
+ if self.fieldnames is None:
+ return space.w_None
+ return space.newtuple([space.wrap(name) for name in self.fieldnames])
+
+ @unwrap_spec(item=str)
+ def descr_getitem(self, space, item):
+ if self.fields is None:
+ raise OperationError(space.w_KeyError, space.wrap("There are no keys in dtypes %s" % self.name))
+ try:
+ return self.fields[item][1]
+ except KeyError:
+ raise OperationError(space.w_KeyError, space.wrap("Field named %s not found" % item))
+
def is_int_type(self):
return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
self.kind == BOOLLTR)
+ def is_signed(self):
+ return self.kind == SIGNEDLTR
+
def is_bool_type(self):
return self.kind == BOOLLTR
+ def is_record_type(self):
+ return self.fields is not None
+
+ def __repr__(self):
+ if self.fields is not None:
+ return '<DType %r>' % self.fields
+ return '<DType %r>' % self.itemtype
+
+ def get_size(self):
+ return self.itemtype.get_element_size()
+
+def dtype_from_list(space, w_lst):
+ lst_w = space.listview(w_lst)
+ fields = {}
+ offset = 0
+ ofs_and_items = []
+ fieldnames = []
+ for w_elem in lst_w:
+ w_fldname, w_flddesc = space.fixedview(w_elem, 2)
+ subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc)
+ fldname = space.str_w(w_fldname)
+ if fldname in fields:
+ raise OperationError(space.w_ValueError, space.wrap("two fields with the same name"))
+ assert isinstance(subdtype, W_Dtype)
+ fields[fldname] = (offset, subdtype)
+ ofs_and_items.append((offset, subdtype.itemtype))
+ offset += subdtype.itemtype.get_element_size()
+ fieldnames.append(fldname)
+ itemtype = types.RecordType(ofs_and_items, offset)
+ return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
+ "V", space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
+ fieldnames=fieldnames)
+
+def dtype_from_dict(space, w_dict):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "dtype from dict"))
+
+def variable_dtype(space, name):
+ if name[0] in '<>=':
+ name = name[1:]
+ char = name[0]
+ if len(name) == 1:
+ size = 0
+ else:
+ try:
+ size = int(name[1:])
+ except ValueError:
+ raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+ if char == 'S':
+ itemtype = types.StringType(size)
+ basename = 'string'
+ num = 18
+ w_box_type = space.gettypefor(interp_boxes.W_StringBox)
+ elif char == 'V':
+ num = 20
+ basename = 'void'
+ w_box_type = space.gettypefor(interp_boxes.W_VoidBox)
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "pure void dtype"))
+ else:
+ assert char == 'U'
+ basename = 'unicode'
+ itemtype = types.UnicodeType(size)
+ num = 19
+ w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
+ return W_Dtype(itemtype, num, char,
+ basename + str(8 * itemtype.get_element_size()),
+ char, w_box_type)
+
+def dtype_from_spec(space, name):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "dtype from spec"))
+
+def descr__new__(space, w_subtype, w_dtype):
+ cache = get_dtype_cache(space)
+
+ if space.is_w(w_dtype, space.w_None):
+ return cache.w_float64dtype
+ elif space.isinstance_w(w_dtype, w_subtype):
+ return w_dtype
+ elif space.isinstance_w(w_dtype, space.w_str):
+ name = space.str_w(w_dtype)
+ if ',' in name:
+ return dtype_from_spec(space, name)
+ try:
+ return cache.dtypes_by_name[name]
+ except KeyError:
+ pass
+ if name[0] in 'VSU' or name[0] in '<>=' and name[1] in 'VSU':
+ return variable_dtype(space, name)
+ elif space.isinstance_w(w_dtype, space.w_list):
+ return dtype_from_list(space, w_dtype)
+ elif space.isinstance_w(w_dtype, space.w_dict):
+ return dtype_from_dict(space, w_dtype)
+ else:
+ for dtype in cache.builtin_dtypes:
+ if w_dtype in dtype.alternate_constructors:
+ return dtype
+ if w_dtype is dtype.w_box_type:
+ return dtype
+ raise OperationError(space.w_TypeError, space.wrap("data type not understood"))
+
W_Dtype.typedef = TypeDef("dtype",
__module__ = "numpypy",
- __new__ = interp2app(W_Dtype.descr__new__.im_func),
+ __new__ = interp2app(descr__new__),
__str__= interp2app(W_Dtype.descr_str),
__repr__ = interp2app(W_Dtype.descr_repr),
__eq__ = interp2app(W_Dtype.descr_eq),
__ne__ = interp2app(W_Dtype.descr_ne),
+ __getitem__ = interp2app(W_Dtype.descr_getitem),
num = interp_attrproperty("num", cls=W_Dtype),
kind = interp_attrproperty("kind", cls=W_Dtype),
+ char = interp_attrproperty("char", cls=W_Dtype),
type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
+ byteorder = GetSetProperty(W_Dtype.descr_get_byteorder),
itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
+ alignment = GetSetProperty(W_Dtype.descr_get_alignment),
shape = GetSetProperty(W_Dtype.descr_get_shape),
name = interp_attrproperty('name', cls=W_Dtype),
+ fields = GetSetProperty(W_Dtype.descr_get_fields),
+ names = GetSetProperty(W_Dtype.descr_get_names),
)
W_Dtype.typedef.acceptable_as_base_class = False
+if sys.byteorder == 'little':
+ byteorder_prefix = '<'
+ nonnative_byteorder_prefix = '>'
+else:
+ byteorder_prefix = '>'
+ nonnative_byteorder_prefix = '<'
+
+def new_string_dtype(space, size):
+ return W_Dtype(
+ types.StringType(size),
+ num=18,
+ kind=STRINGLTR,
+ name='string',
+ char='S' + str(size),
+ w_box_type = space.gettypefor(interp_boxes.W_StringBox),
+ )
+
+def new_unicode_dtype(space, size):
+ return W_Dtype(
+ types.UnicodeType(size),
+ num=19,
+ kind=UNICODELTR,
+ name='unicode',
+ char='U' + str(size),
+ w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
+ )
+
+
class DtypeCache(object):
def __init__(self, space):
self.w_booldtype = W_Dtype(
@@ -211,7 +359,6 @@
name="int64",
char="q",
w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
- alternate_constructors=[space.w_long],
)
self.w_uint64dtype = W_Dtype(
types.UInt64(),
@@ -239,18 +386,149 @@
alternate_constructors=[space.w_float],
aliases=["float"],
)
-
+ self.w_longlongdtype = W_Dtype(
+ types.Int64(),
+ num=9,
+ kind=SIGNEDLTR,
+ name='int64',
+ char='q',
+ w_box_type = space.gettypefor(interp_boxes.W_LongLongBox),
+ alternate_constructors=[space.w_long],
+ )
+ self.w_ulonglongdtype = W_Dtype(
+ types.UInt64(),
+ num=10,
+ kind=UNSIGNEDLTR,
+ name='uint64',
+ char='Q',
+ w_box_type = space.gettypefor(interp_boxes.W_ULongLongBox),
+ )
+ self.w_stringdtype = W_Dtype(
+ types.StringType(1),
+ num=18,
+ kind=STRINGLTR,
+ name='string',
+ char='S',
+ w_box_type = space.gettypefor(interp_boxes.W_StringBox),
+ alternate_constructors=[space.w_str],
+ )
+ self.w_unicodedtype = W_Dtype(
+ types.UnicodeType(1),
+ num=19,
+ kind=UNICODELTR,
+ name='unicode',
+ char='U',
+ w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
+ alternate_constructors=[space.w_unicode],
+ )
+ self.w_voiddtype = W_Dtype(
+ types.VoidType(0),
+ num=20,
+ kind=VOIDLTR,
+ name='void',
+ char='V',
+ w_box_type = space.gettypefor(interp_boxes.W_VoidBox),
+ #alternate_constructors=[space.w_buffer],
+ # XXX no buffer in space
+ )
self.builtin_dtypes = [
self.w_booldtype, self.w_int8dtype, self.w_uint8dtype,
self.w_int16dtype, self.w_uint16dtype, self.w_int32dtype,
self.w_uint32dtype, self.w_longdtype, self.w_ulongdtype,
- self.w_int64dtype, self.w_uint64dtype, self.w_float32dtype,
- self.w_float64dtype
+ self.w_longlongdtype, self.w_ulonglongdtype,
+ self.w_float32dtype,
+ self.w_float64dtype, self.w_stringdtype, self.w_unicodedtype,
+ self.w_voiddtype,
]
self.dtypes_by_num_bytes = sorted(
(dtype.itemtype.get_element_size(), dtype)
for dtype in self.builtin_dtypes
)
+ self.dtypes_by_name = {}
+ for dtype in self.builtin_dtypes:
+ self.dtypes_by_name[dtype.name] = dtype
+ can_name = dtype.kind + str(dtype.itemtype.get_element_size())
+ self.dtypes_by_name[can_name] = dtype
+ self.dtypes_by_name[byteorder_prefix + can_name] = dtype
+ self.dtypes_by_name['=' + can_name] = dtype
+ new_name = nonnative_byteorder_prefix + can_name
+ itemtypename = dtype.itemtype.__class__.__name__
+ itemtype = getattr(types, 'NonNative' + itemtypename)()
+ self.dtypes_by_name[new_name] = W_Dtype(
+ itemtype,
+ dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
+ native=False)
+ for alias in dtype.aliases:
+ self.dtypes_by_name[alias] = dtype
+ self.dtypes_by_name[dtype.char] = dtype
+
+ typeinfo_full = {
+ 'LONGLONG': self.w_int64dtype,
+ 'SHORT': self.w_int16dtype,
+ 'VOID': self.w_voiddtype,
+ #'LONGDOUBLE':,
+ 'UBYTE': self.w_uint8dtype,
+ 'UINTP': self.w_ulongdtype,
+ 'ULONG': self.w_ulongdtype,
+ 'LONG': self.w_longdtype,
+ 'UNICODE': self.w_unicodedtype,
+ #'OBJECT',
+ 'ULONGLONG': self.w_ulonglongdtype,
+ 'STRING': self.w_stringdtype,
+ #'CDOUBLE',
+ #'DATETIME',
+ 'UINT': self.w_uint32dtype,
+ 'INTP': self.w_longdtype,
+ #'HALF',
+ 'BYTE': self.w_int8dtype,
+ #'CFLOAT': ,
+ #'TIMEDELTA',
+ 'INT': self.w_int32dtype,
+ 'DOUBLE': self.w_float64dtype,
+ 'USHORT': self.w_uint16dtype,
+ 'FLOAT': self.w_float32dtype,
+ 'BOOL': self.w_booldtype,
+ #, 'CLONGDOUBLE']
+ }
+ typeinfo_partial = {
+ 'Generic': interp_boxes.W_GenericBox,
+ 'Character': interp_boxes.W_CharacterBox,
+ 'Flexible': interp_boxes.W_FlexibleBox,
+ 'Inexact': interp_boxes.W_InexactBox,
+ 'Integer': interp_boxes.W_IntegerBox,
+ 'SignedInteger': interp_boxes.W_SignedIntegerBox,
+ 'UnsignedInteger': interp_boxes.W_UnsignedIntegerBox,
+ #'ComplexFloating',
+ 'Number': interp_boxes.W_NumberBox,
+ 'Floating': interp_boxes.W_FloatingBox
+ }
+ w_typeinfo = space.newdict()
+ for k, v in typeinfo_partial.iteritems():
+ space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v))
+ for k, dtype in typeinfo_full.iteritems():
+ itemsize = dtype.itemtype.get_element_size()
+ items_w = [space.wrap(dtype.char),
+ space.wrap(dtype.num),
+ space.wrap(itemsize * 8), # in case of changing
+ # number of bits per byte in the future
+ space.wrap(itemsize or 1)]
+ if dtype.is_int_type():
+ if dtype.kind == BOOLLTR:
+ w_maxobj = space.wrap(1)
+ w_minobj = space.wrap(0)
+ elif dtype.is_signed():
+ w_maxobj = space.wrap(r_longlong((1 << (itemsize*8 - 1))
+ - 1))
+ w_minobj = space.wrap(r_longlong(-1) << (itemsize*8 - 1))
+ else:
+ w_maxobj = space.wrap(r_ulonglong(1 << (itemsize*8)) - 1)
+ w_minobj = space.wrap(0)
+ items_w = items_w + [w_maxobj, w_minobj]
+ items_w = items_w + [dtype.w_box_type]
+
+ w_tuple = space.newtuple(items_w)
+ space.setitem(w_typeinfo, space.wrap(k), w_tuple)
+ self.w_typeinfo = w_typeinfo
def get_dtype_cache(space):
return space.fromcache(DtypeCache)
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -2,7 +2,7 @@
from pypy.rlib import jit
from pypy.rlib.objectmodel import instantiate
from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
- calculate_slice_strides, calculate_dot_strides
+ calculate_slice_strides, calculate_dot_strides, enumerate_chunks
""" This is a mini-tutorial on iterators, strides, and
memory layout. It assumes you are familiar with the terms, see
@@ -42,25 +42,67 @@
we can go faster.
All the calculations happen in next()
-next_step_x() tries to do the iteration for a number of steps at once,
+next_skip_x() tries to do the iteration for a number of steps at once,
but then we cannot gaurentee that we only overflow one single shape
dimension, perhaps we could overflow times in one big step.
"""
# structures to describe slicing
-class Chunk(object):
+class BaseChunk(object):
+ pass
+
+class RecordChunk(BaseChunk):
+ def __init__(self, name):
+ self.name = name
+
+ def apply(self, arr):
+ from pypy.module.micronumpy.interp_numarray import W_NDimSlice
+
+ arr = arr.get_concrete()
+ ofs, subdtype = arr.dtype.fields[self.name]
+ # strides backstrides are identical, ofs only changes start
+ return W_NDimSlice(arr.start + ofs, arr.strides[:], arr.backstrides[:],
+ arr.shape[:], arr, subdtype)
+
+class Chunks(BaseChunk):
+ def __init__(self, l):
+ self.l = l
+
+ @jit.unroll_safe
+ def extend_shape(self, old_shape):
+ shape = []
+ i = -1
+ for i, c in enumerate_chunks(self.l):
+ if c.step != 0:
+ shape.append(c.lgt)
+ s = i + 1
+ assert s >= 0
+ return shape[:] + old_shape[s:]
+
+ def apply(self, arr):
+ from pypy.module.micronumpy.interp_numarray import W_NDimSlice,\
+ VirtualSlice, ConcreteArray
+
+ shape = self.extend_shape(arr.shape)
+ if not isinstance(arr, ConcreteArray):
+ return VirtualSlice(arr, self, shape)
+ r = calculate_slice_strides(arr.shape, arr.start, arr.strides,
+ arr.backstrides, self.l)
+ _, start, strides, backstrides = r
+ return W_NDimSlice(start, strides[:], backstrides[:],
+ shape[:], arr)
+
+
+class Chunk(BaseChunk):
axis_step = 1
+
def __init__(self, start, stop, step, lgt):
self.start = start
self.stop = stop
self.step = step
self.lgt = lgt
- def extend_shape(self, shape):
- if self.step != 0:
- shape.append(self.lgt)
-
def __repr__(self):
return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
self.lgt)
@@ -106,17 +148,19 @@
raise NotImplementedError
class ArrayIterator(BaseIterator):
- def __init__(self, size):
+ def __init__(self, size, element_size):
self.offset = 0
self.size = size
+ self.element_size = element_size
def next(self, shapelen):
return self.next_skip_x(1)
- def next_skip_x(self, ofs):
+ def next_skip_x(self, x):
arr = instantiate(ArrayIterator)
arr.size = self.size
- arr.offset = self.offset + ofs
+ arr.offset = self.offset + x * self.element_size
+ arr.element_size = self.element_size
return arr
def next_no_increase(self, shapelen):
@@ -163,7 +207,7 @@
elif isinstance(t, ViewTransform):
r = calculate_slice_strides(self.res_shape, self.offset,
self.strides,
- self.backstrides, t.chunks)
+ self.backstrides, t.chunks.l)
return ViewIterator(r[1], r[2], r[3], r[0])
@jit.unroll_safe
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -7,10 +7,10 @@
from pypy.module.micronumpy.appbridge import get_appbridge_cache
from pypy.module.micronumpy.dot import multidim_dot, match_dot_shapes
from pypy.module.micronumpy.interp_iter import (ArrayIterator,
- SkipLastAxisIterator, Chunk, NewAxisChunk, ViewIterator)
-from pypy.module.micronumpy.strides import (calculate_slice_strides,
- shape_agreement, find_shape_and_elems, get_shape_from_iterable,
- calc_new_strides, to_coords, enumerate_chunks)
+ SkipLastAxisIterator, Chunk, ViewIterator, Chunks, RecordChunk,
+ NewAxisChunk)
+from pypy.module.micronumpy.strides import (shape_agreement,
+ find_shape_and_elems, get_shape_from_iterable, calc_new_strides, to_coords)
from pypy.rlib import jit
from pypy.rlib.rstring import StringBuilder
from pypy.rpython.lltypesystem import lltype, rffi
@@ -47,7 +47,7 @@
)
flat_set_driver = jit.JitDriver(
greens=['shapelen', 'base'],
- reds=['step', 'ai', 'lngth', 'arr', 'basei'],
+ reds=['step', 'lngth', 'ri', 'arr', 'basei'],
name='numpy_flatset',
)
@@ -79,8 +79,8 @@
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
- size, shape = _find_size_and_shape(space, w_size)
- return space.wrap(W_NDimArray(size, shape[:], dtype=dtype))
+ shape = _find_shape(space, w_size)
+ return space.wrap(W_NDimArray(shape[:], dtype=dtype))
def _unaryop_impl(ufunc_name):
def impl(self, space):
@@ -225,8 +225,7 @@
return scalar_w(space, dtype, space.wrap(0))
# Do the dims match?
out_shape, other_critical_dim = match_dot_shapes(space, self, other)
- out_size = support.product(out_shape)
- result = W_NDimArray(out_size, out_shape, dtype)
+ result = W_NDimArray(out_shape, dtype)
# This is the place to add fpypy and blas
return multidim_dot(space, self.get_concrete(),
other.get_concrete(), result, dtype,
@@ -245,7 +244,7 @@
return space.wrap(self.find_dtype().itemtype.get_element_size())
def descr_get_nbytes(self, space):
- return space.wrap(self.size * self.find_dtype().itemtype.get_element_size())
+ return space.wrap(self.size)
@jit.unroll_safe
def descr_get_shape(self, space):
@@ -253,13 +252,16 @@
def descr_set_shape(self, space, w_iterable):
new_shape = get_shape_from_iterable(space,
- self.size, w_iterable)
+ support.product(self.shape), w_iterable)
if isinstance(self, Scalar):
return
self.get_concrete().setshape(space, new_shape)
def descr_get_size(self, space):
- return space.wrap(self.size)
+ return space.wrap(self.get_size())
+
+ def get_size(self):
+ return self.size // self.find_dtype().get_size()
def descr_copy(self, space):
return self.copy(space)
@@ -279,7 +281,7 @@
def empty_copy(self, space, dtype):
shape = self.shape
- return W_NDimArray(support.product(shape), shape[:], dtype, 'C')
+ return W_NDimArray(shape[:], dtype, 'C')
def descr_len(self, space):
if len(self.shape):
@@ -320,6 +322,8 @@
""" The result of getitem/setitem is a single item if w_idx
is a list of scalars that match the size of shape
"""
+ if space.isinstance_w(w_idx, space.w_str):
+ return False
shape_len = len(self.shape)
if space.isinstance_w(w_idx, space.w_tuple):
for w_item in space.fixedview(w_idx):
@@ -347,11 +351,18 @@
@jit.unroll_safe
def _prepare_slice_args(self, space, w_idx):
+ if space.isinstance_w(w_idx, space.w_str):
+ idx = space.str_w(w_idx)
+ dtype = self.find_dtype()
+ if not dtype.is_record_type() or idx not in dtype.fields:
+ raise OperationError(space.w_ValueError, space.wrap(
+ "field named %s not defined" % idx))
+ return RecordChunk(idx)
if (space.isinstance_w(w_idx, space.w_int) or
space.isinstance_w(w_idx, space.w_slice)):
- return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
+ return Chunks([Chunk(*space.decode_index4(w_idx, self.shape[0]))])
elif space.is_w(w_idx, space.w_None):
- return [NewAxisChunk()]
+ return Chunks([NewAxisChunk()])
result = []
i = 0
for w_item in space.fixedview(w_idx):
@@ -361,30 +372,30 @@
result.append(Chunk(*space.decode_index4(w_item,
self.shape[i])))
i += 1
- return result
+ return Chunks(result)
- def count_all_true(self, arr):
- sig = arr.find_sig()
- frame = sig.create_frame(arr)
- shapelen = len(arr.shape)
+ def count_all_true(self):
+ sig = self.find_sig()
+ frame = sig.create_frame(self)
+ shapelen = len(self.shape)
s = 0
iter = None
while not frame.done():
- count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+ count_driver.jit_merge_point(arr=self, frame=frame, iter=iter, s=s,
shapelen=shapelen)
iter = frame.get_final_iter()
- s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+ s += self.dtype.getitem_bool(self, iter.offset)
frame.next(shapelen)
return s
def getitem_filter(self, space, arr):
concr = arr.get_concrete()
- if concr.size > self.size:
+ if concr.get_size() > self.get_size():
raise OperationError(space.w_IndexError,
space.wrap("index out of range for array"))
- size = self.count_all_true(concr)
- res = W_NDimArray(size, [size], self.find_dtype())
- ri = ArrayIterator(size)
+ size = concr.count_all_true()
+ res = W_NDimArray([size], self.find_dtype())
+ ri = res.create_iter()
shapelen = len(self.shape)
argi = concr.create_iter()
sig = self.find_sig()
@@ -394,7 +405,7 @@
filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
frame=frame, v=v, res=res, sig=sig,
shapelen=shapelen, self=self)
- if concr.dtype.getitem_bool(concr.storage, argi.offset):
+ if concr.dtype.getitem_bool(concr, argi.offset):
v = sig.eval(frame, self)
res.setitem(ri.offset, v)
ri = ri.next(1)
@@ -404,23 +415,6 @@
frame.next(shapelen)
return res
- def setitem_filter(self, space, idx, val):
- size = self.count_all_true(idx)
- arr = SliceArray([size], self.dtype, self, val)
- sig = arr.find_sig()
- shapelen = len(self.shape)
- frame = sig.create_frame(arr)
- idxi = idx.create_iter()
- while not frame.done():
- filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
- frame=frame, arr=arr,
- shapelen=shapelen)
- if idx.dtype.getitem_bool(idx.storage, idxi.offset):
- sig.eval(frame, arr)
- frame.next_from_second(1)
- frame.next_first(shapelen)
- idxi = idxi.next(shapelen)
-
def descr_getitem(self, space, w_idx):
if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
w_idx.find_dtype().is_bool_type()):
@@ -430,7 +424,24 @@
item = concrete._index_of_single_item(space, w_idx)
return concrete.getitem(item)
chunks = self._prepare_slice_args(space, w_idx)
- return self.create_slice(chunks)
+ return chunks.apply(self)
+
+ def setitem_filter(self, space, idx, val):
+ size = idx.count_all_true()
+ arr = SliceArray([size], self.dtype, self, val)
+ sig = arr.find_sig()
+ shapelen = len(self.shape)
+ frame = sig.create_frame(arr)
+ idxi = idx.create_iter()
+ while not frame.done():
+ filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+ frame=frame, arr=arr,
+ shapelen=shapelen)
+ if idx.dtype.getitem_bool(idx, idxi.offset):
+ sig.eval(frame, arr)
+ frame.next_from_second(1)
+ frame.next_first(shapelen)
+ idxi = idxi.next(shapelen)
def descr_setitem(self, space, w_idx, w_value):
self.invalidated()
@@ -448,26 +459,9 @@
if not isinstance(w_value, BaseArray):
w_value = convert_to_array(space, w_value)
chunks = self._prepare_slice_args(space, w_idx)
- view = self.create_slice(chunks).get_concrete()
+ view = chunks.apply(self).get_concrete()
view.setslice(space, w_value)
- @jit.unroll_safe
- def create_slice(self, chunks):
- shape = []
- i = -1
- for i, chunk in enumerate_chunks(chunks):
- chunk.extend_shape(shape)
- s = i + 1
- assert s >= 0
- shape += self.shape[s:]
- if not isinstance(self, ConcreteArray):
- return VirtualSlice(self, chunks, shape)
- r = calculate_slice_strides(self.shape, self.start, self.strides,
- self.backstrides, chunks)
- _, start, strides, backstrides = r
- return W_NDimSlice(start, strides[:], backstrides[:],
- shape[:], self)
-
def descr_reshape(self, space, args_w):
"""reshape(...)
a.reshape(shape)
@@ -484,7 +478,8 @@
w_shape = args_w[0]
else:
w_shape = space.newtuple(args_w)
- new_shape = get_shape_from_iterable(space, self.size, w_shape)
+ new_shape = get_shape_from_iterable(space, support.product(self.shape),
+ w_shape)
return self.reshape(space, new_shape)
def reshape(self, space, new_shape):
@@ -522,7 +517,7 @@
def descr_mean(self, space, w_axis=None):
if space.is_w(w_axis, space.w_None):
w_axis = space.wrap(-1)
- w_denom = space.wrap(self.size)
+ w_denom = space.wrap(support.product(self.shape))
else:
dim = space.int_w(w_axis)
w_denom = space.wrap(self.shape[dim])
@@ -541,7 +536,7 @@
concr.fill(space, w_value)
def descr_nonzero(self, space):
- if self.size > 1:
+ if self.get_size() > 1:
raise OperationError(space.w_ValueError, space.wrap(
"The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()"))
concr = self.get_concrete_or_scalar()
@@ -620,8 +615,7 @@
space.wrap("axis unsupported for take"))
index_i = index.create_iter()
res_shape = index.shape
- size = support.product(res_shape)
- res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order)
+ res = W_NDimArray(res_shape[:], concr.dtype, concr.order)
res_i = res.create_iter()
shapelen = len(index.shape)
sig = concr.find_sig()
@@ -660,6 +654,11 @@
raise OperationError(space.w_NotImplementedError, space.wrap(
"non-int arg not supported"))
+ def descr_tostring(self, space):
+ ra = ToStringArray(self)
+ loop.compute(ra)
+ return space.wrap(ra.s.build())
+
def compute_first_step(self, sig, frame):
pass
@@ -681,8 +680,7 @@
"""
Intermediate class representing a literal.
"""
- size = 1
- _attrs_ = ["dtype", "value", "shape"]
+ _attrs_ = ["dtype", "value", "shape", "size"]
def __init__(self, dtype, value):
self.shape = []
@@ -690,6 +688,7 @@
self.dtype = dtype
assert isinstance(value, interp_boxes.W_GenericBox)
self.value = value
+ self.size = dtype.get_size()
def find_dtype(self):
return self.dtype
@@ -707,8 +706,7 @@
return self
def reshape(self, space, new_shape):
- size = support.product(new_shape)
- res = W_NDimArray(size, new_shape, self.dtype, 'C')
+ res = W_NDimArray(new_shape, self.dtype, 'C')
res.setitem(0, self.value)
return res
@@ -721,6 +719,7 @@
self.forced_result = None
self.res_dtype = res_dtype
self.name = name
+ self.size = support.product(self.shape) * res_dtype.get_size()
def _del_sources(self):
# Function for deleting references to source arrays,
@@ -728,7 +727,7 @@
raise NotImplementedError
def compute(self):
- ra = ResultArray(self, self.size, self.shape, self.res_dtype)
+ ra = ResultArray(self, self.shape, self.res_dtype)
loop.compute(ra)
return ra.left
@@ -756,7 +755,6 @@
def __init__(self, child, chunks, shape):
self.child = child
self.chunks = chunks
- self.size = support.product(shape)
VirtualArray.__init__(self, 'slice', shape, child.find_dtype())
def create_sig(self):
@@ -768,7 +766,7 @@
def force_if_needed(self):
if self.forced_result is None:
concr = self.child.get_concrete()
- self.forced_result = concr.create_slice(self.chunks)
+ self.forced_result = self.chunks.apply(concr)
def _del_sources(self):
self.child = None
@@ -801,7 +799,6 @@
self.left = left
self.right = right
self.calc_dtype = calc_dtype
- self.size = support.product(self.shape)
def _del_sources(self):
self.left = None
@@ -829,15 +826,30 @@
self.left.create_sig(), self.right.create_sig())
class ResultArray(Call2):
- def __init__(self, child, size, shape, dtype, res=None, order='C'):
+ def __init__(self, child, shape, dtype, res=None, order='C'):
if res is None:
- res = W_NDimArray(size, shape, dtype, order)
+ res = W_NDimArray(shape, dtype, order)
Call2.__init__(self, None, 'assign', shape, dtype, dtype, res, child)
def create_sig(self):
return signature.ResultSignature(self.res_dtype, self.left.create_sig(),
self.right.create_sig())
+class ToStringArray(Call1):
+ def __init__(self, child):
+ dtype = child.find_dtype()
+ self.item_size = dtype.itemtype.get_element_size()
+ self.s = StringBuilder(child.size * self.item_size)
+ Call1.__init__(self, None, 'tostring', child.shape, dtype, dtype,
+ child)
+ self.res = W_NDimArray([1], dtype, 'C')
+ self.res_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
+ self.res.storage)
+
+ def create_sig(self):
+ return signature.ToStringSignature(self.calc_dtype,
+ self.values.create_sig())
+
def done_if_true(dtype, val):
return dtype.itemtype.bool(val)
@@ -909,13 +921,13 @@
"""
_immutable_fields_ = ['storage']
- def __init__(self, size, shape, dtype, order='C', parent=None):
- self.size = size
+ def __init__(self, shape, dtype, order='C', parent=None):
self.parent = parent
+ self.size = support.product(shape) * dtype.get_size()
if parent is not None:
self.storage = parent.storage
else:
- self.storage = dtype.malloc(size)
+ self.storage = dtype.itemtype.malloc(self.size)
self.order = order
self.dtype = dtype
if self.strides is None:
@@ -934,13 +946,14 @@
return self.dtype
def getitem(self, item):
- return self.dtype.getitem(self.storage, item)
+ return self.dtype.getitem(self, item)
def setitem(self, item, value):
self.invalidated()
- self.dtype.setitem(self.storage, item, value)
+ self.dtype.setitem(self, item, value)
def calc_strides(self, shape):
+ dtype = self.find_dtype()
strides = []
backstrides = []
s = 1
@@ -948,8 +961,8 @@
if self.order == 'C':
shape_rev.reverse()
for sh in shape_rev:
- strides.append(s)
- backstrides.append(s * (sh - 1))
+ strides.append(s * dtype.get_size())
+ backstrides.append(s * (sh - 1) * dtype.get_size())
s *= sh
if self.order == 'C':
strides.reverse()
@@ -997,9 +1010,9 @@
shapelen = len(self.shape)
if shapelen == 1:
rffi.c_memcpy(
- rffi.ptradd(self.storage, self.start * itemsize),
- rffi.ptradd(w_value.storage, w_value.start * itemsize),
- self.size * itemsize
+ rffi.ptradd(self.storage, self.start),
+ rffi.ptradd(w_value.storage, w_value.start),
+ self.size
)
else:
dest = SkipLastAxisIterator(self)
@@ -1014,7 +1027,7 @@
dest.next()
def copy(self, space):
- array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
+ array = W_NDimArray(self.shape[:], self.dtype, self.order)
array.setslice(space, self)
return array
@@ -1028,14 +1041,15 @@
class W_NDimSlice(ViewArray):
- def __init__(self, start, strides, backstrides, shape, parent):
+ def __init__(self, start, strides, backstrides, shape, parent, dtype=None):
assert isinstance(parent, ConcreteArray)
if isinstance(parent, W_NDimSlice):
parent = parent.parent
self.strides = strides
self.backstrides = backstrides
- ViewArray.__init__(self, support.product(shape), shape, parent.dtype,
- parent.order, parent)
+ if dtype is None:
+ dtype = parent.dtype
+ ViewArray.__init__(self, shape, dtype, parent.order, parent)
self.start = start
def create_iter(self, transforms=None):
@@ -1050,12 +1064,13 @@
# but then calc_strides would have to accept a stepping factor
strides = []
backstrides = []
- s = self.strides[0]
+ dtype = self.find_dtype()
+ s = self.strides[0] // dtype.get_size()
if self.order == 'C':
new_shape.reverse()
for sh in new_shape:
- strides.append(s)
- backstrides.append(s * (sh - 1))
+ strides.append(s * dtype.get_size())
+ backstrides.append(s * (sh - 1) * dtype.get_size())
s *= max(1, sh)
if self.order == 'C':
strides.reverse()
@@ -1083,14 +1098,16 @@
"""
def setitem(self, item, value):
self.invalidated()
- self.dtype.setitem(self.storage, item, value)
+ self.dtype.setitem(self, item, value)
def setshape(self, space, new_shape):
self.shape = new_shape
self.calc_strides(new_shape)
def create_iter(self, transforms=None):
- return ArrayIterator(self.size).apply_transformations(self, transforms)
+ esize = self.find_dtype().get_size()
+ return ArrayIterator(self.size, esize).apply_transformations(self,
+ transforms)
def create_sig(self):
return signature.ArraySignature(self.dtype)
@@ -1098,18 +1115,13 @@
def __del__(self):
lltype.free(self.storage, flavor='raw', track_allocation=False)
-def _find_size_and_shape(space, w_size):
+def _find_shape(space, w_size):
if space.isinstance_w(w_size, space.w_int):
- size = space.int_w(w_size)
- shape = [size]
- else:
- size = 1
- shape = []
- for w_item in space.fixedview(w_size):
- item = space.int_w(w_item)
- size *= item
- shape.append(item)
- return size, shape
+ return [space.int_w(w_size)]
+ shape = []
+ for w_item in space.fixedview(w_size):
+ shape.append(space.int_w(w_item))
+ return shape
@unwrap_spec(subok=bool, copy=bool, ownmaskna=bool)
def array(space, w_item_or_iterable, w_dtype=None, w_order=None,
@@ -1143,28 +1155,28 @@
if copy:
return w_item_or_iterable.copy(space)
return w_item_or_iterable
- shape, elems_w = find_shape_and_elems(space, w_item_or_iterable)
+ if w_dtype is None or space.is_w(w_dtype, space.w_None):
+ dtype = None
+ else:
+ dtype = space.interp_w(interp_dtype.W_Dtype,
+ space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype))
+ shape, elems_w = find_shape_and_elems(space, w_item_or_iterable, dtype)
# they come back in C order
- size = len(elems_w)
- if w_dtype is None or space.is_w(w_dtype, space.w_None):
- w_dtype = None
+ if dtype is None:
for w_elem in elems_w:
- w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem,
- w_dtype)
- if w_dtype is interp_dtype.get_dtype_cache(space).w_float64dtype:
+ dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem,
+ dtype)
+ if dtype is interp_dtype.get_dtype_cache(space).w_float64dtype:
break
- if w_dtype is None:
- w_dtype = space.w_None
- dtype = space.interp_w(interp_dtype.W_Dtype,
- space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
- )
- arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
+ if dtype is None:
+ dtype = interp_dtype.get_dtype_cache(space).w_float64dtype
+ arr = W_NDimArray(shape[:], dtype=dtype, order=order)
shapelen = len(shape)
- arr_iter = ArrayIterator(arr.size)
+ arr_iter = arr.create_iter()
# XXX we might want to have a jitdriver here
for i in range(len(elems_w)):
w_elem = elems_w[i]
- dtype.setitem(arr.storage, arr_iter.offset,
+ dtype.setitem(arr, arr_iter.offset,
dtype.coerce(space, w_elem))
arr_iter = arr_iter.next(shapelen)
return arr
@@ -1173,22 +1185,22 @@
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
- size, shape = _find_size_and_shape(space, w_size)
+ shape = _find_shape(space, w_size)
if not shape:
return scalar_w(space, dtype, space.wrap(0))
- return space.wrap(W_NDimArray(size, shape[:], dtype=dtype))
+ return space.wrap(W_NDimArray(shape[:], dtype=dtype))
def ones(space, w_size, w_dtype=None):
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
- size, shape = _find_size_and_shape(space, w_size)
+ shape = _find_shape(space, w_size)
if not shape:
return scalar_w(space, dtype, space.wrap(1))
- arr = W_NDimArray(size, shape[:], dtype=dtype)
+ arr = W_NDimArray(shape[:], dtype=dtype)
one = dtype.box(1)
- arr.dtype.fill(arr.storage, one, 0, size)
+ arr.dtype.fill(arr.storage, one, 0, arr.size)
return space.wrap(arr)
@unwrap_spec(arr=BaseArray, skipna=bool, keepdims=bool)
@@ -1236,13 +1248,13 @@
"array dimensions must agree except for axis being concatenated"))
elif i == axis:
shape[i] += axis_size
- res = W_NDimArray(support.product(shape), shape, dtype, 'C')
+ res = W_NDimArray(shape, dtype, 'C')
chunks = [Chunk(0, i, 1, i) for i in shape]
axis_start = 0
for arr in args_w:
chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1,
arr.shape[axis])
- res.create_slice(chunks).setslice(space, arr)
+ Chunks(chunks).apply(res).setslice(space, arr)
axis_start += arr.shape[axis]
return res
@@ -1330,6 +1342,7 @@
std = interp2app(BaseArray.descr_std),
fill = interp2app(BaseArray.descr_fill),
+ tostring = interp2app(BaseArray.descr_tostring),
copy = interp2app(BaseArray.descr_copy),
flatten = interp2app(BaseArray.descr_flatten),
@@ -1352,7 +1365,7 @@
self.iter = sig.create_frame(arr).get_final_iter()
self.base = arr
self.index = 0
- ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order,
+ ViewArray.__init__(self, [arr.get_size()], arr.dtype, arr.order,
arr)
def descr_next(self, space):
@@ -1367,7 +1380,7 @@
return self
def descr_len(self, space):
- return space.wrap(self.size)
+ return space.wrap(self.get_size())
def descr_index(self, space):
return space.wrap(self.index)
@@ -1385,28 +1398,26 @@
raise OperationError(space.w_IndexError,
space.wrap('unsupported iterator index'))
base = self.base
- start, stop, step, lngth = space.decode_index4(w_idx, base.size)
+ start, stop, step, lngth = space.decode_index4(w_idx, base.get_size())
# setslice would have been better, but flat[u:v] for arbitrary
# shapes of array a cannot be represented as a[x1:x2, y1:y2]
basei = ViewIterator(base.start, base.strides,
- base.backstrides,base.shape)
+ base.backstrides, base.shape)
shapelen = len(base.shape)
basei = basei.next_skip_x(shapelen, start)
if lngth <2:
return base.getitem(basei.offset)
- ri = ArrayIterator(lngth)
- res = W_NDimArray(lngth, [lngth], base.dtype,
- base.order)
+ res = W_NDimArray([lngth], base.dtype, base.order)
+ ri = res.create_iter()
while not ri.done():
flat_get_driver.jit_merge_point(shapelen=shapelen,
base=base,
basei=basei,
step=step,
res=res,
- ri=ri,
- )
+ ri=ri)
w_val = base.getitem(basei.offset)
- res.setitem(ri.offset,w_val)
+ res.setitem(ri.offset, w_val)
basei = basei.next_skip_x(shapelen, step)
ri = ri.next(shapelen)
return res
@@ -1417,27 +1428,28 @@
raise OperationError(space.w_IndexError,
space.wrap('unsupported iterator index'))
base = self.base
- start, stop, step, lngth = space.decode_index4(w_idx, base.size)
+ start, stop, step, lngth = space.decode_index4(w_idx, base.get_size())
arr = convert_to_array(space, w_value)
- ai = 0
+ ri = arr.create_iter()
basei = ViewIterator(base.start, base.strides,
- base.backstrides,base.shape)
+ base.backstrides, base.shape)
shapelen = len(base.shape)
basei = basei.next_skip_x(shapelen, start)
while lngth > 0:
flat_set_driver.jit_merge_point(shapelen=shapelen,
- basei=basei,
- base=base,
- step=step,
- arr=arr,
- ai=ai,
- lngth=lngth,
- )
- v = arr.getitem(ai).convert_to(base.dtype)
+ basei=basei,
+ base=base,
+ step=step,
+ arr=arr,
+ lngth=lngth,
+ ri=ri)
+ v = arr.getitem(ri.offset).convert_to(base.dtype)
base.setitem(basei.offset, v)
# need to repeat input values until all assignments are done
- ai = (ai + 1) % arr.size
basei = basei.next_skip_x(shapelen, step)
+ ri = ri.next(shapelen)
+ # WTF is numpy thinking?
+ ri.offset %= arr.size
lngth -= 1
def create_sig(self):
@@ -1445,9 +1457,9 @@
def create_iter(self, transforms=None):
return ViewIterator(self.base.start, self.base.strides,
- self.base.backstrides,
- self.base.shape).apply_transformations(self.base,
- transforms)
+ self.base.backstrides,
+ self.base.shape).apply_transformations(self.base,
+ transforms)
def descr_base(self, space):
return space.wrap(self.base)
diff --git a/pypy/module/micronumpy/interp_support.py b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -51,9 +51,11 @@
raise OperationError(space.w_ValueError, space.wrap(
"string is smaller than requested size"))
- a = W_NDimArray(num_items, [num_items], dtype=dtype)
- for i, val in enumerate(items):
- a.dtype.setitem(a.storage, i, val)
+ a = W_NDimArray([num_items], dtype=dtype)
+ ai = a.create_iter()
+ for val in items:
+ a.dtype.setitem(a, ai.offset, val)
+ ai = ai.next(1)
return space.wrap(a)
@@ -61,6 +63,7 @@
from pypy.module.micronumpy.interp_numarray import W_NDimArray
itemsize = dtype.itemtype.get_element_size()
+ assert itemsize >= 0
if count == -1:
count = length / itemsize
if length % itemsize != 0:
@@ -71,20 +74,23 @@
raise OperationError(space.w_ValueError, space.wrap(
"string is smaller than requested size"))
- a = W_NDimArray(count, [count], dtype=dtype)
- fromstring_loop(a, count, dtype, itemsize, s)
+ a = W_NDimArray([count], dtype=dtype)
+ fromstring_loop(a, dtype, itemsize, s)
return space.wrap(a)
-fromstring_driver = jit.JitDriver(greens=[], reds=['count', 'i', 'itemsize',
- 'dtype', 's', 'a'])
+fromstring_driver = jit.JitDriver(greens=[], reds=['i', 'itemsize',
+ 'dtype', 'ai', 's', 'a'])
-def fromstring_loop(a, count, dtype, itemsize, s):
+def fromstring_loop(a, dtype, itemsize, s):
i = 0
- while i < count:
- fromstring_driver.jit_merge_point(a=a, count=count, dtype=dtype,
- itemsize=itemsize, s=s, i=i)
+ ai = a.create_iter()
+ while not ai.done():
+ fromstring_driver.jit_merge_point(a=a, dtype=dtype,
+ itemsize=itemsize, s=s, i=i,
+ ai=ai)
val = dtype.itemtype.runpack_str(s[i*itemsize:i*itemsize + itemsize])
- a.dtype.setitem(a.storage, i, val)
+ a.dtype.setitem(a, ai.offset, val)
+ ai = ai.next(1)
i += 1
@unwrap_spec(s=str, count=int, sep=str)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -156,7 +156,7 @@
shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
else:
shape = obj.shape[:dim] + obj.shape[dim + 1:]
- result = W_NDimArray(support.product(shape), shape, dtype)
+ result = W_NDimArray(shape, dtype)
arr = AxisReduce(self.func, self.name, self.identity, obj.shape, dtype,
result, obj, dim)
loop.compute(arr)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -4,6 +4,7 @@
ViewTransform, BroadcastTransform
from pypy.tool.pairtype import extendabletype
from pypy.module.micronumpy.loop import ComputationDone
+from pypy.rlib import jit
""" Signature specifies both the numpy expression that has been constructed
and the assembler to be compiled. This is a very important observation -
@@ -142,11 +143,10 @@
from pypy.module.micronumpy.interp_numarray import ConcreteArray
concr = arr.get_concrete()
assert isinstance(concr, ConcreteArray)
- storage = concr.storage
if self.iter_no >= len(iterlist):
iterlist.append(concr.create_iter(transforms))
if self.array_no >= len(arraylist):
- arraylist.append(storage)
+ arraylist.append(concr)
def eval(self, frame, arr):
iter = frame.iterators[self.iter_no]
@@ -318,6 +318,20 @@
offset = frame.get_final_iter().offset
arr.left.setitem(offset, self.right.eval(frame, arr.right))
+class ToStringSignature(Call1):
+ def __init__(self, dtype, child):
+ Call1.__init__(self, None, 'tostring', dtype, child)
+
+ @jit.unroll_safe
+ def eval(self, frame, arr):
+ from pypy.module.micronumpy.interp_numarray import ToStringArray
+
+ assert isinstance(arr, ToStringArray)
+ arr.res.setitem(0, self.child.eval(frame, arr.values).convert_to(
+ self.dtype))
+ for i in range(arr.item_size):
+ arr.s.append(arr.res_casted[i])
+
class BroadcastLeft(Call2):
def _invent_numbering(self, cache, allnumbers):
self.left._invent_numbering(new_cache(), allnumbers)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -46,22 +46,31 @@
rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides
return rstrides, rbackstrides
-def find_shape_and_elems(space, w_iterable):
+def is_single_elem(space, w_elem, is_rec_type):
+ if (is_rec_type and space.isinstance_w(w_elem, space.w_tuple)):
+ return True
+ if space.issequence_w(w_elem):
+ return False
+ return True
+
+def find_shape_and_elems(space, w_iterable, dtype):
shape = [space.len_w(w_iterable)]
batch = space.listview(w_iterable)
+ is_rec_type = dtype is not None and dtype.is_record_type()
while True:
new_batch = []
if not batch:
return shape, []
- if not space.issequence_w(batch[0]):
- for elem in batch:
- if space.issequence_w(elem):
+ if is_single_elem(space, batch[0], is_rec_type):
+ for w_elem in batch:
+ if not is_single_elem(space, w_elem, is_rec_type):
raise OperationError(space.w_ValueError, space.wrap(
"setting an array element with a sequence"))
return shape, batch
size = space.len_w(batch[0])
for w_elem in batch:
- if not space.issequence_w(w_elem) or space.len_w(w_elem) != size:
+ if (is_single_elem(space, w_elem, is_rec_type) or
+ space.len_w(w_elem) != size):
raise OperationError(space.w_ValueError, space.wrap(
"setting an array element with a sequence"))
new_batch += space.listview(w_elem)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -4,6 +4,8 @@
from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
find_unaryop_result_dtype)
from pypy.module.micronumpy.interp_boxes import W_Float64Box
+from pypy.module.micronumpy.interp_dtype import nonnative_byteorder_prefix,\
+ byteorder_prefix
from pypy.conftest import option
import sys
@@ -15,14 +17,16 @@
sys.modules['numpypy'] = numpy
sys.modules['_numpypy'] = numpy
cls.space = gettestobjspace(usemodules=['micronumpy'])
+ cls.w_non_native_prefix = cls.space.wrap(nonnative_byteorder_prefix)
+ cls.w_native_prefix = cls.space.wrap(byteorder_prefix)
class TestSignature(object):
def test_binop_signature(self, space):
float64_dtype = get_dtype_cache(space).w_float64dtype
bool_dtype = get_dtype_cache(space).w_booldtype
- ar = W_NDimArray(10, [10], dtype=float64_dtype)
- ar2 = W_NDimArray(10, [10], dtype=float64_dtype)
+ ar = W_NDimArray([10], dtype=float64_dtype)
+ ar2 = W_NDimArray([10], dtype=float64_dtype)
v1 = ar.descr_add(space, ar)
v2 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(2.0)))
sig1 = v1.find_sig()
@@ -40,7 +44,7 @@
v4 = ar.descr_add(space, ar)
assert v1.find_sig() is v4.find_sig()
- bool_ar = W_NDimArray(10, [10], dtype=bool_dtype)
+ bool_ar = W_NDimArray([10], dtype=bool_dtype)
v5 = ar.descr_add(space, bool_ar)
assert v5.find_sig() is not v1.find_sig()
assert v5.find_sig() is not v2.find_sig()
@@ -57,7 +61,7 @@
def test_slice_signature(self, space):
float64_dtype = get_dtype_cache(space).w_float64dtype
- ar = W_NDimArray(10, [10], dtype=float64_dtype)
+ ar = W_NDimArray([10], dtype=float64_dtype)
v1 = ar.descr_getitem(space, space.wrap(slice(1, 3, 1)))
v2 = ar.descr_getitem(space, space.wrap(slice(4, 6, 1)))
assert v1.find_sig() is v2.find_sig()
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1,5 +1,7 @@
+import py
+from pypy.conftest import option
from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
-
+from pypy.interpreter.gateway import interp2app
class AppTestDtypes(BaseNumpyAppTest):
def test_dtype(self):
@@ -12,7 +14,10 @@
assert dtype(d) is d
assert dtype(None) is dtype(float)
assert dtype('int8').name == 'int8'
+ assert dtype(int).fields is None
+ assert dtype(int).names is None
raises(TypeError, dtype, 1042)
+ raises(KeyError, 'dtype(int)["asdasd"]')
def test_dtype_eq(self):
from _numpypy import dtype
@@ -53,13 +58,13 @@
assert a[i] is True_
def test_copy_array_with_dtype(self):
- from _numpypy import array, False_, True_, int64
+ from _numpypy import array, False_, longlong
a = array([0, 1, 2, 3], dtype=long)
# int on 64-bit, long in 32-bit
- assert isinstance(a[0], int64)
+ assert isinstance(a[0], longlong)
b = a.copy()
- assert isinstance(b[0], int64)
+ assert isinstance(b[0], longlong)
a = array([0, 1, 2, 3], dtype=bool)
assert a[0] is False_
@@ -81,17 +86,17 @@
assert a[i] is True_
def test_zeros_long(self):
- from _numpypy import zeros, int64
+ from _numpypy import zeros, longlong
a = zeros(10, dtype=long)
for i in range(10):
- assert isinstance(a[i], int64)
+ assert isinstance(a[i], longlong)
assert a[1] == 0
def test_ones_long(self):
- from _numpypy import ones, int64
+ from _numpypy import ones, longlong
a = ones(10, dtype=long)
for i in range(10):
- assert isinstance(a[i], int64)
+ assert isinstance(a[i], longlong)
assert a[1] == 1
def test_overflow(self):
@@ -181,17 +186,18 @@
assert dtype("float") is dtype(float)
-class AppTestTypes(BaseNumpyAppTest):
+class AppTestTypes(BaseNumpyAppTest):
def test_abstract_types(self):
import _numpypy as numpy
raises(TypeError, numpy.generic, 0)
raises(TypeError, numpy.number, 0)
raises(TypeError, numpy.integer, 0)
exc = raises(TypeError, numpy.signedinteger, 0)
- assert str(exc.value) == "cannot create 'signedinteger' instances"
+ assert 'cannot create' in str(exc.value)
+ assert 'signedinteger' in str(exc.value)
exc = raises(TypeError, numpy.unsignedinteger, 0)
- assert str(exc.value) == "cannot create 'unsignedinteger' instances"
-
+ assert 'cannot create' in str(exc.value)
+ assert 'unsignedinteger' in str(exc.value)
raises(TypeError, numpy.floating, 0)
raises(TypeError, numpy.inexact, 0)
@@ -404,10 +410,29 @@
assert issubclass(int64, int)
assert int_ is int64
+ def test_various_types(self):
+ import _numpypy as numpy
+ import sys
+
+ assert numpy.int16 is numpy.short
+ assert numpy.int8 is numpy.byte
+ assert numpy.bool_ is numpy.bool8
+ if sys.maxint == (1 << 63) - 1:
+ assert numpy.intp is numpy.int64
+ else:
+ assert numpy.intp is numpy.int32
+
+ def test_mro(self):
+ import _numpypy as numpy
+
+ assert numpy.int16.__mro__ == (numpy.int16, numpy.signedinteger,
+ numpy.integer, numpy.number,
+ numpy.generic, object)
+ assert numpy.bool_.__mro__ == (numpy.bool_, numpy.generic, object)
+
def test_operators(self):
from operator import truediv
from _numpypy import float64, int_, True_, False_
-
assert 5 / int_(2) == int_(2)
assert truediv(int_(3), int_(2)) == float64(1.5)
assert truediv(3, int_(2)) == float64(1.5)
@@ -427,9 +452,115 @@
assert int_(3) ^ int_(5) == int_(6)
assert True_ ^ False_ is True_
assert 5 ^ int_(3) == int_(6)
-
assert +int_(3) == int_(3)
assert ~int_(3) == int_(-4)
-
raises(TypeError, lambda: float64(3) & 1)
+ def test_alternate_constructs(self):
+ from _numpypy import dtype
+ nnp = self.non_native_prefix
+ byteorder = self.native_prefix
+ assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') # XXX should be equal == dtype(long)
+ assert dtype(nnp + 'i8') != dtype('i8')
+ assert dtype(nnp + 'i8').byteorder == nnp
+ assert dtype('=i8').byteorder == '='
+ assert dtype(byteorder + 'i8').byteorder == '='
+
+ def test_alignment(self):
+ from _numpypy import dtype
+ assert dtype('i4').alignment == 4
+
+ def test_typeinfo(self):
+ from _numpypy import typeinfo, void, number, int64, bool_
+ assert typeinfo['Number'] == number
+ assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L, -9223372036854775808L, int64)
+ assert typeinfo['VOID'] == ('V', 20, 0, 1, void)
+ assert typeinfo['BOOL'] == ('?', 0, 8, 1, 1, 0, bool_)
+
+class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
+ def test_str_unicode(self):
+ from _numpypy import str_, unicode_, character, flexible, generic
+
+ assert str_.mro() == [str_, str, basestring, character, flexible, generic, object]
+ assert unicode_.mro() == [unicode_, unicode, basestring, character, flexible, generic, object]
+
+ def test_str_dtype(self):
+ from _numpypy import dtype, str_
+
+ raises(TypeError, "dtype('Sx')")
+ d = dtype('S8')
+ assert d.itemsize == 8
+ assert dtype(str) == dtype('S')
+ assert d.kind == 'S'
+ assert d.type is str_
+ assert d.name == "string64"
+ assert d.num == 18
+
+ def test_unicode_dtype(self):
+ from _numpypy import dtype, unicode_
+
+ raises(TypeError, "dtype('Ux')")
+ d = dtype('U8')
+ assert d.itemsize == 8 * 4
+ assert dtype(unicode) == dtype('U')
+ assert d.kind == 'U'
+ assert d.type is unicode_
+ assert d.name == "unicode256"
+ assert d.num == 19
+
+ def test_string_boxes(self):
+ from _numpypy import str_
+ assert isinstance(str_(3), str_)
+
+ def test_unicode_boxes(self):
+ from _numpypy import unicode_
+ assert isinstance(unicode_(3), unicode)
+
+class AppTestRecordDtypes(BaseNumpyAppTest):
+ def test_create(self):
+ from _numpypy import dtype, void
+
+ raises(ValueError, "dtype([('x', int), ('x', float)])")
+ d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
+ assert d.fields['x'] == (dtype('int32'), 0)
+ assert d.fields['value'] == (dtype(float), 12)
+ assert d['x'] == dtype('int32')
+ assert d.name == "void160"
+ assert d.num == 20
+ assert d.itemsize == 20
+ assert d.kind == 'V'
+ assert d.type is void
+ assert d.char == 'V'
+ assert d.names == ("x", "y", "z", "value")
+ raises(KeyError, 'd["xyz"]')
+ raises(KeyError, 'd.fields["xyz"]')
+
+ def test_create_from_dict(self):
+ skip("not yet")
+ from _numpypy import dtype
+ d = dtype({'names': ['a', 'b', 'c'],
+ })
+
+class AppTestNotDirect(BaseNumpyAppTest):
+ def setup_class(cls):
+ BaseNumpyAppTest.setup_class.im_func(cls)
+ def check_non_native(w_obj, w_obj2):
+ assert w_obj.storage[0] == w_obj2.storage[1]
+ assert w_obj.storage[1] == w_obj2.storage[0]
+ if w_obj.storage[0] == '\x00':
+ assert w_obj2.storage[1] == '\x00'
+ assert w_obj2.storage[0] == '\x01'
+ else:
+ assert w_obj2.storage[1] == '\x01'
+ assert w_obj2.storage[0] == '\x00'
+ cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native))
+ if option.runappdirect:
+ py.test.skip("not a direct test")
+
+ def test_non_native(self):
+ from _numpypy import array
+ a = array([1, 2, 3], dtype=self.non_native_prefix + 'i2')
+ assert a[0] == 1
+ assert (a + a)[1] == 4
+ self.check_non_native(a, array([1, 2, 3], 'i2'))
+
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -5,15 +5,23 @@
from pypy.interpreter.error import OperationError
from pypy.module.micronumpy import signature
from pypy.module.micronumpy.appbridge import get_appbridge_cache
-from pypy.module.micronumpy.interp_iter import Chunk
+from pypy.module.micronumpy.interp_iter import Chunk, Chunks
from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
class MockDtype(object):
- def malloc(self, size):
- return None
+ class itemtype(object):
+ @staticmethod
+ def malloc(size):
+ return None
+ def get_size(self):
+ return 1
+
+
+def create_slice(a, chunks):
+ return Chunks(chunks).apply(a)
class TestNumArrayDirect(object):
def newslice(self, *args):
@@ -29,116 +37,116 @@
return self.space.newtuple(args_w)
def test_strides_f(self):
- a = W_NDimArray(100, [10, 5, 3], MockDtype(), 'F')
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
assert a.strides == [1, 10, 50]
assert a.backstrides == [9, 40, 100]
def test_strides_c(self):
- a = W_NDimArray(100, [10, 5, 3], MockDtype(), 'C')
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'C')
assert a.strides == [15, 3, 1]
assert a.backstrides == [135, 12, 2]
def test_create_slice_f(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([Chunk(3, 0, 0, 1)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
+ s = create_slice(a, [Chunk(3, 0, 0, 1)])
assert s.start == 3
assert s.strides == [10, 50]
assert s.backstrides == [40, 100]
- s = a.create_slice([Chunk(1, 9, 2, 4)])
+ s = create_slice(a, [Chunk(1, 9, 2, 4)])
assert s.start == 1
assert s.strides == [2, 10, 50]
assert s.backstrides == [6, 40, 100]
- s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
+ s = create_slice(a, [Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
assert s.shape == [2, 1]
assert s.strides == [3, 10]
assert s.backstrides == [3, 0]
- s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
assert s.start == 20
assert s.shape == [10, 3]
def test_create_slice_c(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
- s = a.create_slice([Chunk(3, 0, 0, 1)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'C')
+ s = create_slice(a, [Chunk(3, 0, 0, 1)])
assert s.start == 45
assert s.strides == [3, 1]
assert s.backstrides == [12, 2]
- s = a.create_slice([Chunk(1, 9, 2, 4)])
+ s = create_slice(a, [Chunk(1, 9, 2, 4)])
assert s.start == 15
assert s.strides == [30, 3, 1]
assert s.backstrides == [90, 12, 2]
- s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+ s = create_slice(a, [Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
Chunk(1, 0, 0, 1)])
assert s.start == 19
assert s.shape == [2, 1]
assert s.strides == [45, 3]
assert s.backstrides == [45, 0]
- s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
assert s.start == 6
assert s.shape == [10, 3]
def test_slice_of_slice_f(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([Chunk(5, 0, 0, 1)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
+ s = create_slice(a, [Chunk(5, 0, 0, 1)])
assert s.start == 5
- s2 = s.create_slice([Chunk(3, 0, 0, 1)])
+ s2 = create_slice(s, [Chunk(3, 0, 0, 1)])
assert s2.shape == [3]
assert s2.strides == [50]
assert s2.parent is a
assert s2.backstrides == [100]
assert s2.start == 35
- s = a.create_slice([Chunk(1, 5, 3, 2)])
- s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(1, 5, 3, 2)])
+ s2 = create_slice(s, [Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
assert s2.shape == [2, 3]
assert s2.strides == [3, 50]
assert s2.backstrides == [3, 100]
assert s2.start == 1 * 15 + 2 * 3
def test_slice_of_slice_c(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
- s = a.create_slice([Chunk(5, 0, 0, 1)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), order='C')
+ s = create_slice(a, [Chunk(5, 0, 0, 1)])
assert s.start == 15 * 5
- s2 = s.create_slice([Chunk(3, 0, 0, 1)])
+ s2 = create_slice(s, [Chunk(3, 0, 0, 1)])
assert s2.shape == [3]
assert s2.strides == [1]
assert s2.parent is a
assert s2.backstrides == [2]
assert s2.start == 5 * 15 + 3 * 3
- s = a.create_slice([Chunk(1, 5, 3, 2)])
- s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(1, 5, 3, 2)])
+ s2 = create_slice(s, [Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
assert s2.shape == [2, 3]
assert s2.strides == [45, 1]
assert s2.backstrides == [45, 2]
assert s2.start == 1 * 15 + 2 * 3
def test_negative_step_f(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([Chunk(9, -1, -2, 5)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
+ s = create_slice(a, [Chunk(9, -1, -2, 5)])
assert s.start == 9
assert s.strides == [-2, 10, 50]
assert s.backstrides == [-8, 40, 100]
def test_negative_step_c(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
- s = a.create_slice([Chunk(9, -1, -2, 5)])
+ a = W_NDimArray([10, 5, 3], MockDtype(), order='C')
+ s = create_slice(a, [Chunk(9, -1, -2, 5)])
assert s.start == 135
assert s.strides == [-30, 3, 1]
assert s.backstrides == [-120, 12, 2]
def test_index_of_single_item_f(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'F')
r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
assert r == 1 + 2 * 10 + 2 * 50
- s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
r = s._index_of_single_item(self.space, self.newtuple(1, 0))
assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
r = s._index_of_single_item(self.space, self.newtuple(1, 1))
assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 1))
def test_index_of_single_item_c(self):
- a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
+ a = W_NDimArray([10, 5, 3], MockDtype(), 'C')
r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
assert r == 1 * 3 * 5 + 2 * 3 + 2
- s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
+ s = create_slice(a, [Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
r = s._index_of_single_item(self.space, self.newtuple(1, 0))
assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -416,6 +424,7 @@
from numpypy.core.numeric import newaxis
a = array(range(5))
b = a[newaxis]
+ assert b.shape == (1, 5)
assert (b[0,1:] == a[1:]).all()
def test_slice_then_newaxis(self):
@@ -1140,7 +1149,7 @@
assert array([True, False]).dtype is dtype(bool)
assert array([True, 1]).dtype is dtype(int)
assert array([1, 2, 3]).dtype is dtype(int)
- assert array([1L, 2, 3]).dtype is dtype(long)
+ #assert array([1L, 2, 3]).dtype is dtype(long)
assert array([1.2, True]).dtype is dtype(float)
assert array([1.2, 5]).dtype is dtype(float)
assert array([]).dtype is dtype(float)
@@ -1647,6 +1656,7 @@
a = arange(12).reshape(3,4)
b = a.T.flat
b[6::2] = [-1, -2]
+ print a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]
assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all()
b[0:2] = [[[100]]]
assert(a[0,0] == 100)
@@ -1921,6 +1931,12 @@
#5 bytes is larger than 3 bytes
raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8)
+ def test_tostring(self):
+ from _numpypy import array
+ assert array([1, 2, 3], 'i2').tostring() == '\x01\x00\x02\x00\x03\x00'
+ assert array([1, 2, 3], 'i2')[::2].tostring() == '\x01\x00\x03\x00'
+ assert array([1, 2, 3], '<i2')[::2].tostring() == '\x01\x00\x03\x00'
+ assert array([1, 2, 3], '>i2')[::2].tostring() == '\x00\x01\x00\x03'
class AppTestRanges(BaseNumpyAppTest):
def test_arange(self):
@@ -1966,3 +1982,57 @@
cache = get_appbridge_cache(cls.space)
cache.w_array_repr = cls.old_array_repr
cache.w_array_str = cls.old_array_str
+
+class AppTestRecordDtype(BaseNumpyAppTest):
+ def test_zeros(self):
+ from _numpypy import zeros
+ a = zeros(2, dtype=[('x', int), ('y', float)])
+ raises(IndexError, 'a[0]["xyz"]')
+ assert a[0]['x'] == 0
+ assert a[0]['y'] == 0
+ raises(ValueError, "a[0] = (1, 2, 3)")
+ a[0]['x'] = 13
+ assert a[0]['x'] == 13
+ a[1] = (1, 2)
+ assert a[1]['y'] == 2
+ b = zeros(2, dtype=[('x', int), ('y', float)])
+ b[1] = a[1]
+ assert a[1]['y'] == 2
+
+ def test_views(self):
+ from _numpypy import array
+ a = array([(1, 2), (3, 4)], dtype=[('x', int), ('y', float)])
+ raises(ValueError, 'array([1])["x"]')
+ raises(ValueError, 'a["z"]')
+ assert a['x'][1] == 3
+ assert a['y'][1] == 4
+ a['x'][0] = 15
+ assert a['x'][0] == 15
+ b = a['x'] + a['y']
+ assert (b == [15+2, 3+4]).all()
+ assert b.dtype == float
+
+ def test_assign_tuple(self):
+ from _numpypy import zeros
+ a = zeros((2, 3), dtype=[('x', int), ('y', float)])
+ a[1, 2] = (1, 2)
+ assert a['x'][1, 2] == 1
+ assert a['y'][1, 2] == 2
+
+ def test_creation_and_repr(self):
+ from _numpypy import array
+ a = array([(1, 2), (3, 4)], dtype=[('x', int), ('y', float)])
+ assert repr(a[0]) == '(1, 2.0)'
+
+ def test_nested_dtype(self):
+ from _numpypy import zeros
+ a = [('x', int), ('y', float)]
+ b = [('x', int), ('y', a)]
+ arr = zeros(3, dtype=b)
+ arr[1]['x'] = 15
+ assert arr[1]['x'] == 15
+ arr[1]['y']['y'] = 3.5
+ assert arr[1]['y']['y'] == 3.5
+ assert arr[1]['y']['x'] == 0.0
+ assert arr[1]['x'] == 15
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1,15 +1,20 @@
import functools
import math
+import struct
from pypy.interpreter.error import OperationError
from pypy.module.micronumpy import interp_boxes
from pypy.objspace.std.floatobject import float2string
from pypy.rlib import rfloat, libffi, clibffi
-from pypy.rlib.objectmodel import specialize
-from pypy.rlib.rarithmetic import LONG_BIT, widen
+from pypy.rlib.objectmodel import specialize, we_are_translated
+from pypy.rlib.rarithmetic import widen, byteswap
from pypy.rpython.lltypesystem import lltype, rffi
from pypy.rlib.rstruct.runpack import runpack
+from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
+VOID_STORAGE = lltype.Array(lltype.Char, hints={'nolength': True,
+ 'render_as_void': True})
degToRad = math.pi / 180.0
log2 = math.log(2)
@@ -62,6 +67,15 @@
def _unimplemented_ufunc(self, *args):
raise NotImplementedError
+ def malloc(self, size):
+ # XXX find out why test_zjit explodes with tracking of allocations
+ return lltype.malloc(VOID_STORAGE, size,
+ zero=True, flavor="raw",
+ track_allocation=False, add_memory_pressure=True)
+
+ def __repr__(self):
+ return self.__class__.__name__
+
class Primitive(object):
_mixin_ = True
@@ -76,7 +90,7 @@
assert isinstance(box, self.BoxType)
return box.value
- def coerce(self, space, w_item):
+ def coerce(self, space, dtype, w_item):
if isinstance(w_item, self.BoxType):
return w_item
return self.coerce_subtype(space, space.gettypefor(self.BoxType), w_item)
@@ -97,32 +111,41 @@
def default_fromstring(self, space):
raise NotImplementedError
- def read(self, storage, width, i, offset):
- return self.box(libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
- width, storage, i, offset
- ))
+ def _read(self, storage, width, i, offset):
+ if we_are_translated():
+ return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
+ width, storage, i, offset)
+ else:
+ return libffi.array_getitem_T(self.T, width, storage, i, offset)
- def read_bool(self, storage, width, i, offset):
- return bool(self.for_computation(
- libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
- width, storage, i, offset)))
+ def read(self, arr, width, i, offset, dtype=None):
+ return self.box(self._read(arr.storage, width, i, offset))
- def store(self, storage, width, i, offset, box):
- value = self.unbox(box)
- libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
- width, storage, i, offset, value
- )
+ def read_bool(self, arr, width, i, offset):
+ return bool(self.for_computation(self._read(arr.storage, width, i, offset)))
+
+ def _write(self, storage, width, i, offset, value):
+ if we_are_translated():
+ libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
+ width, storage, i, offset, value)
+ else:
+ libffi.array_setitem_T(self.T, width, storage, i, offset, value)
+
+
+ def store(self, arr, width, i, offset, box):
+ self._write(arr.storage, width, i, offset, self.unbox(box))
def fill(self, storage, width, box, start, stop, offset):
value = self.unbox(box)
- for i in xrange(start, stop):
- libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
- width, storage, i, offset, value
- )
+ for i in xrange(start, stop, width):
+ self._write(storage, 1, i, offset, value)
def runpack_str(self, s):
return self.box(runpack(self.format_code, s))
+ def pack_str(self, box):
+ return struct.pack(self.format_code, self.unbox(box))
+
@simple_binary_op
def add(self, v1, v2):
return v1 + v2
@@ -214,6 +237,17 @@
def min(self, v1, v2):
return min(v1, v2)
+class NonNativePrimitive(Primitive):
+ _mixin_ = True
+
+ def _read(self, storage, width, i, offset):
+ return byteswap(Primitive._read(self, storage, width, i, offset))
+
+ def _write(self, storage, width, i, offset, value):
+ Primitive._write(self, storage, width, i, offset, byteswap(value))
+
+ def pack_str(self, box):
+ return struct.pack(self.format_code, byteswap(self.unbox(box)))
class Bool(BaseType, Primitive):
T = lltype.Bool
@@ -242,8 +276,7 @@
return space.wrap(self.unbox(w_item))
def str_format(self, box):
- value = self.unbox(box)
- return "True" if value else "False"
+ return "True" if self.unbox(box) else "False"
def for_computation(self, v):
return int(v)
@@ -267,15 +300,18 @@
def invert(self, v):
return ~v
+NonNativeBool = Bool
+
class Integer(Primitive):
_mixin_ = True
+ def _base_coerce(self, space, w_item):
+ return self.box(space.int_w(space.call_function(space.w_int, w_item)))
def _coerce(self, space, w_item):
- return self.box(space.int_w(space.call_function(space.w_int, w_item)))
+ return self._base_coerce(space, w_item)
def str_format(self, box):
- value = self.unbox(box)
- return str(self.for_computation(value))
+ return str(self.for_computation(self.unbox(box)))
def for_computation(self, v):
return widen(v)
@@ -347,68 +383,117 @@
def invert(self, v):
return ~v
+class NonNativeInteger(NonNativePrimitive, Integer):
+ _mixin_ = True
+
class Int8(BaseType, Integer):
T = rffi.SIGNEDCHAR
BoxType = interp_boxes.W_Int8Box
format_code = "b"
+NonNativeInt8 = Int8
class UInt8(BaseType, Integer):
T = rffi.UCHAR
BoxType = interp_boxes.W_UInt8Box
format_code = "B"
+NonNativeUInt8 = UInt8
class Int16(BaseType, Integer):
T = rffi.SHORT
BoxType = interp_boxes.W_Int16Box
format_code = "h"
+class NonNativeInt16(BaseType, NonNativeInteger):
+ T = rffi.SHORT
+ BoxType = interp_boxes.W_Int16Box
+ format_code = "h"
+
class UInt16(BaseType, Integer):
T = rffi.USHORT
BoxType = interp_boxes.W_UInt16Box
format_code = "H"
+class NonNativeUInt16(BaseType, NonNativeInteger):
+ T = rffi.USHORT
+ BoxType = interp_boxes.W_UInt16Box
+ format_code = "H"
+
class Int32(BaseType, Integer):
T = rffi.INT
BoxType = interp_boxes.W_Int32Box
format_code = "i"
+class NonNativeInt32(BaseType, NonNativeInteger):
+ T = rffi.INT
+ BoxType = interp_boxes.W_Int32Box
+ format_code = "i"
+
class UInt32(BaseType, Integer):
T = rffi.UINT
BoxType = interp_boxes.W_UInt32Box
format_code = "I"
+class NonNativeUInt32(BaseType, NonNativeInteger):
+ T = rffi.UINT
+ BoxType = interp_boxes.W_UInt32Box
+ format_code = "I"
+
class Long(BaseType, Integer):
T = rffi.LONG
BoxType = interp_boxes.W_LongBox
format_code = "l"
+class NonNativeLong(BaseType, NonNativeInteger):
+ T = rffi.LONG
+ BoxType = interp_boxes.W_LongBox
+ format_code = "l"
+
class ULong(BaseType, Integer):
T = rffi.ULONG
BoxType = interp_boxes.W_ULongBox
format_code = "L"
+class NonNativeULong(BaseType, NonNativeInteger):
+ T = rffi.ULONG
+ BoxType = interp_boxes.W_ULongBox
+ format_code = "L"
+
class Int64(BaseType, Integer):
T = rffi.LONGLONG
BoxType = interp_boxes.W_Int64Box
format_code = "q"
+class NonNativeInt64(BaseType, NonNativeInteger):
+ T = rffi.LONGLONG
+ BoxType = interp_boxes.W_Int64Box
+ format_code = "q"
+
+def _uint64_coerce(self, space, w_item):
+ try:
+ return self._base_coerce(space, w_item)
+ except OperationError, e:
+ if not e.match(space, space.w_OverflowError):
+ raise
+ bigint = space.bigint_w(w_item)
+ try:
+ value = bigint.toulonglong()
+ except OverflowError:
+ raise OperationError(space.w_OverflowError, space.w_None)
+ return self.box(value)
+
class UInt64(BaseType, Integer):
T = rffi.ULONGLONG
BoxType = interp_boxes.W_UInt64Box
format_code = "Q"
- def _coerce(self, space, w_item):
- try:
- return Integer._coerce(self, space, w_item)
- except OperationError, e:
- if not e.match(space, space.w_OverflowError):
- raise
- bigint = space.bigint_w(w_item)
- try:
- value = bigint.toulonglong()
- except OverflowError:
- raise OperationError(space.w_OverflowError, space.w_None)
- return self.box(value)
+ _coerce = func_with_new_name(_uint64_coerce, '_coerce')
+
+class NonNativeUInt64(BaseType, NonNativeInteger):
+ T = rffi.ULONGLONG
+ BoxType = interp_boxes.W_UInt64Box
+ format_code = "Q"
+
+ _coerce = func_with_new_name(_uint64_coerce, '_coerce')
class Float(Primitive):
_mixin_ = True
@@ -417,8 +502,8 @@
return self.box(space.float_w(space.call_function(space.w_float, w_item)))
def str_format(self, box):
- value = self.unbox(box)
- return float2string(self.for_computation(value), "g", rfloat.DTSF_STR_PRECISION)
+ return float2string(self.for_computation(self.unbox(box)), "g",
+ rfloat.DTSF_STR_PRECISION)
def for_computation(self, v):
return float(v)
@@ -702,13 +787,128 @@
return -rfloat.INFINITY
return rfloat.NAN
+class NonNativeFloat(NonNativePrimitive, Float):
+ _mixin_ = True
class Float32(BaseType, Float):
T = rffi.FLOAT
BoxType = interp_boxes.W_Float32Box
format_code = "f"
+class NonNativeFloat32(BaseType, NonNativeFloat):
+ T = rffi.FLOAT
+ BoxType = interp_boxes.W_Float32Box
+ format_code = "f"
+
class Float64(BaseType, Float):
T = rffi.DOUBLE
BoxType = interp_boxes.W_Float64Box
format_code = "d"
+
+class NonNativeFloat64(BaseType, NonNativeFloat):
+ T = rffi.DOUBLE
+ BoxType = interp_boxes.W_Float64Box
+ format_code = "d"
+
+class CompositeType(BaseType):
+ def __init__(self, offsets_and_fields, size):
+ self.offsets_and_fields = offsets_and_fields
+ self.size = size
+
+ def get_element_size(self):
+ return self.size
+
+class BaseStringType(object):
+ _mixin_ = True
+
+ def __init__(self, size=0):
+ self.size = size
+
+ def get_element_size(self):
+ return self.size * rffi.sizeof(self.T)
+
+class StringType(BaseType, BaseStringType):
+ T = lltype.Char
+
+class VoidType(BaseType, BaseStringType):
+ T = lltype.Char
+
+NonNativeVoidType = VoidType
+NonNativeStringType = StringType
+
+class UnicodeType(BaseType, BaseStringType):
+ T = lltype.UniChar
+
+NonNativeUnicodeType = UnicodeType
+
+class RecordType(CompositeType):
+ T = lltype.Char
+
+ def read(self, arr, width, i, offset, dtype=None):
+ if dtype is None:
+ dtype = arr.dtype
+ return interp_boxes.W_VoidBox(arr, i + offset, dtype)
+
+ @jit.unroll_safe
+ def coerce(self, space, dtype, w_item):
+ from pypy.module.micronumpy.interp_numarray import W_NDimArray
+
+ if isinstance(w_item, interp_boxes.W_VoidBox):
+ return w_item
+ # we treat every sequence as sequence, no special support
+ # for arrays
+ if not space.issequence_w(w_item):
+ raise OperationError(space.w_TypeError, space.wrap(
+ "expected sequence"))
+ if len(self.offsets_and_fields) != space.int_w(space.len(w_item)):
+ raise OperationError(space.w_ValueError, space.wrap(
+ "wrong length"))
+ items_w = space.fixedview(w_item)
+ # XXX optimize it out one day, but for now we just allocate an
+ # array
+ arr = W_NDimArray([1], dtype)
+ for i in range(len(items_w)):
+ subdtype = dtype.fields[dtype.fieldnames[i]][1]
+ ofs, itemtype = self.offsets_and_fields[i]
+ w_item = items_w[i]
+ w_box = itemtype.coerce(space, subdtype, w_item)
+ itemtype.store(arr, 1, 0, ofs, w_box)
+ return interp_boxes.W_VoidBox(arr, 0, arr.dtype)
+
+ @jit.unroll_safe
+ def store(self, arr, _, i, ofs, box):
+ assert isinstance(box, interp_boxes.W_VoidBox)
+ for k in range(self.get_element_size()):
+ arr.storage[k + i] = box.arr.storage[k + box.ofs]
+
+ @jit.unroll_safe
+ def str_format(self, box):
+ assert isinstance(box, interp_boxes.W_VoidBox)
+ pieces = ["("]
+ first = True
+ for ofs, tp in self.offsets_and_fields:
+ if first:
+ first = False
+ else:
+ pieces.append(", ")
+ pieces.append(tp.str_format(tp.read(box.arr, 1, box.ofs, ofs)))
+ pieces.append(")")
+ return "".join(pieces)
+
+for tp in [Int32, Int64]:
+ if tp.T == lltype.Signed:
+ IntP = tp
+ break
+for tp in [UInt32, UInt64]:
+ if tp.T == lltype.Unsigned:
+ UIntP = tp
+ break
+del tp
+
+def _setup():
+ # compute alignment
+ for tp in globals().values():
+ if isinstance(tp, type) and hasattr(tp, 'T'):
+ tp.alignment = clibffi.cast_type_to_ffitype(tp.T).c_alignment
+_setup()
+del _setup
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
@@ -34,7 +34,7 @@
assert repr(a) == "array([], dtype=float64)"
a = zeros(1001)
assert repr(a) == "array([ 0., 0., 0., ..., 0., 0., 0.])"
- a = array(range(5), long)
+ a = array(range(5), int)
if a.dtype.itemsize == int_size:
assert repr(a) == "array([0, 1, 2, 3, 4])"
else:
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -207,6 +207,11 @@
is_arguments(args)
return w_some_obj()
+ def get_and_call_function(space, w_descr, w_obj, *args_w):
+ args = argument.Arguments(space, list(args_w))
+ w_impl = space.get(w_descr, w_obj)
+ return space.call_args(w_impl, args)
+
def gettypefor(self, cls):
return self.gettypeobject(cls.typedef)
diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py
--- a/pypy/rlib/clibffi.py
+++ b/pypy/rlib/clibffi.py
@@ -233,6 +233,7 @@
(rffi.LONGLONG, _signed_type_for(rffi.LONGLONG)),
(lltype.UniChar, _unsigned_type_for(lltype.UniChar)),
(lltype.Bool, _unsigned_type_for(lltype.Bool)),
+ (lltype.Char, _signed_type_for(lltype.Char)),
]
__float_type_map = [
diff --git a/pypy/rlib/libffi.py b/pypy/rlib/libffi.py
--- a/pypy/rlib/libffi.py
+++ b/pypy/rlib/libffi.py
@@ -429,6 +429,11 @@
return rffi.cast(rffi.CArrayPtr(TYPE), addr)[0]
assert False
+def array_getitem_T(TYPE, width, addr, index, offset):
+ addr = rffi.ptradd(addr, index * width)
+ addr = rffi.ptradd(addr, offset)
+ return rffi.cast(rffi.CArrayPtr(TYPE), addr)[0]
+
@specialize.call_location()
@jit.oopspec("libffi_array_setitem(ffitype, width, addr, index, offset, value)")
def array_setitem(ffitype, width, addr, index, offset, value):
@@ -439,3 +444,8 @@
rffi.cast(rffi.CArrayPtr(TYPE), addr)[0] = value
return
assert False
+
+def array_setitem_T(TYPE, width, addr, index, offset, value):
+ addr = rffi.ptradd(addr, index * width)
+ addr = rffi.ptradd(addr, offset)
+ rffi.cast(rffi.CArrayPtr(TYPE), addr)[0] = value
diff --git a/pypy/rlib/rarithmetic.py b/pypy/rlib/rarithmetic.py
--- a/pypy/rlib/rarithmetic.py
+++ b/pypy/rlib/rarithmetic.py
@@ -569,3 +569,30 @@
if not objectmodel.we_are_translated():
assert n <= p
return llop.int_between(lltype.Bool, n, m, p)
+
+ at objectmodel.specialize.ll()
+def byteswap(arg):
+ """ Convert little->big endian and the opposite
+ """
+ from pypy.rpython.lltypesystem import lltype, rffi
+
+ T = lltype.typeOf(arg)
+ # XXX we cannot do arithmetics on small ints
+ arg = widen(arg)
+ if rffi.sizeof(T) == 1:
+ res = arg
+ elif rffi.sizeof(T) == 2:
+ a, b = arg & 0xFF, arg & 0xFF00
+ res = (a << 8) | (b >> 8)
+ elif rffi.sizeof(T) == 4:
+ a, b, c, d = arg & 0xFF, arg & 0xFF00, arg & 0xFF0000, arg & 0xFF000000
+ res = (a << 24) | (b << 8) | (c >> 8) | (d >> 24)
+ elif rffi.sizeof(T) == 8:
+ a, b, c, d = arg & 0xFF, arg & 0xFF00, arg & 0xFF0000, arg & 0xFF000000
+ e, f, g, h = (arg & (0xFF << 32), arg & (0xFF << 40),
+ arg & (0xFF << 48), arg & (r_uint(0xFF) << 56))
+ res = ((a << 56) | (b << 40) | (c << 24) | (d << 8) | (e >> 8) |
+ (f >> 24) | (g >> 40) | (h >> 56))
+ else:
+ assert False # unreachable code
+ return rffi.cast(T, res)
diff --git a/pypy/rlib/rstruct/runpack.py b/pypy/rlib/rstruct/runpack.py
--- a/pypy/rlib/rstruct/runpack.py
+++ b/pypy/rlib/rstruct/runpack.py
@@ -4,11 +4,10 @@
"""
import py
-from struct import pack, unpack
+from struct import unpack
from pypy.rlib.rstruct.formatiterator import FormatIterator
from pypy.rlib.rstruct.error import StructError
from pypy.rlib.rstruct.nativefmttable import native_is_bigendian
-from pypy.rpython.extregistry import ExtRegistryEntry
class MasterReader(object):
def __init__(self, s):
diff --git a/pypy/rlib/test/test_rarithmetic.py b/pypy/rlib/test/test_rarithmetic.py
--- a/pypy/rlib/test/test_rarithmetic.py
+++ b/pypy/rlib/test/test_rarithmetic.py
@@ -383,3 +383,9 @@
assert not int_between(1, 2, 2)
assert not int_between(1, 1, 1)
+def test_byteswap():
+ from pypy.rpython.lltypesystem import rffi
+
+ assert byteswap(rffi.cast(rffi.USHORT, 0x0102)) == 0x0201
+ assert byteswap(rffi.cast(rffi.INT, 0x01020304)) == 0x04030201
+ assert byteswap(rffi.cast(rffi.ULONGLONG, 0x0102030405060708L)) == 0x0807060504030201L
More information about the pypy-commit
mailing list