[pypy-commit] pypy default: merge upstream
pjenvey
noreply at buildbot.pypy.org
Thu Apr 17 01:35:11 CEST 2014
Author: Philip Jenvey <pjenvey at underboss.org>
Branch:
Changeset: r70676:c7c7b54e61a0
Date: 2014-04-16 16:34 -0700
http://bitbucket.org/pypy/pypy/changeset/c7c7b54e61a0/
Log: merge upstream
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -2,7 +2,7 @@
Implementation of the interpreter-level default import logic.
"""
-import sys, os, stat
+import sys, os, stat, genericpath
from pypy.interpreter.module import Module
from pypy.interpreter.gateway import interp2app, unwrap_spec
@@ -522,7 +522,8 @@
path = space.str0_w(w_pathitem)
filepart = os.path.join(path, partname)
- if os.path.isdir(filepart) and case_ok(filepart):
+ # os.path.isdir on win32 is not rpython when pywin32 installed
+ if genericpath.isdir(filepart) and case_ok(filepart):
initfile = os.path.join(filepart, '__init__')
modtype, _, _ = find_modtype(space, initfile)
if modtype in (PY_SOURCE, PY_COMPILED):
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,7 @@
'set_string_function': 'appbridge.set_string_function',
'typeinfo': 'descriptor.get_dtype_cache(space).w_typeinfo',
+ 'nditer': 'nditer.nditer',
}
for c in ['MAXDIMS', 'CLIP', 'WRAP', 'RAISE']:
interpleveldefs[c] = 'space.wrap(constants.%s)' % c
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -458,6 +458,13 @@
return SliceArray(self.start, new_strides, new_backstrides, new_shape,
self, orig_array)
+ def readonly(self):
+ return NonWritableSlice(self.start, self.strides, self.backstrides, self.shape, self.parent, self.orig_arr, self.dtype)
+
+class NonWritableSlice(SliceArray):
+ def descr_setitem(self, space, orig_array, w_index, w_value):
+ raise OperationError(space.w_ValueError, space.wrap(
+ "assignment destination is read-only"))
class VoidBoxStorage(BaseConcreteArray):
def __init__(self, size, dtype):
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -42,6 +42,7 @@
"""
from rpython.rlib import jit
from pypy.module.micronumpy import support
+from pypy.module.micronumpy.strides import calc_strides
from pypy.module.micronumpy.base import W_NDimArray
@@ -142,6 +143,37 @@
def setitem(self, elem):
self.array.setitem(self.offset, elem)
+class SliceIterator(ArrayIter):
+ def __init__(self, arr, strides, backstrides, shape, order="C",
+ backward=False, dtype=None):
+ if dtype is None:
+ dtype = arr.implementation.dtype
+ self.dtype = dtype
+ self.arr = arr
+ if backward:
+ self.slicesize = shape[0]
+ self.gap = [support.product(shape[1:]) * dtype.elsize]
+ strides = strides[1:]
+ backstrides = backstrides[1:]
+ shape = shape[1:]
+ strides.reverse()
+ backstrides.reverse()
+ shape.reverse()
+ size = support.product(shape)
+ else:
+ shape = [support.product(shape)]
+ strides, backstrides = calc_strides(shape, dtype, order)
+ size = 1
+ self.slicesize = support.product(shape)
+ self.gap = strides
+
+ ArrayIter.__init__(self, arr.implementation, size, shape, strides, backstrides)
+
+ def getslice(self):
+ from pypy.module.micronumpy.concrete import SliceArray
+ retVal = SliceArray(self.offset, self.gap, self.backstrides,
+ [self.slicesize], self.arr.implementation, self.arr, self.dtype)
+ return retVal
def AxisIter(array, shape, axis, cumulative):
strides = array.get_strides()
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/nditer.py
@@ -0,0 +1,577 @@
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.interpreter.error import OperationError
+from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
+from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
+ shape_agreement, shape_agreement_multiple)
+from pypy.module.micronumpy.iterators import ArrayIter, SliceIterator
+from pypy.module.micronumpy.concrete import SliceArray
+from pypy.module.micronumpy.descriptor import decode_w_dtype
+from pypy.module.micronumpy import ufuncs
+
+
+class AbstractIterator(object):
+ def done(self):
+ raise NotImplementedError("Abstract Class")
+
+ def next(self):
+ raise NotImplementedError("Abstract Class")
+
+ def getitem(self, space, array):
+ raise NotImplementedError("Abstract Class")
+
+class IteratorMixin(object):
+ _mixin_ = True
+ def __init__(self, it, op_flags):
+ self.it = it
+ self.op_flags = op_flags
+
+ def done(self):
+ return self.it.done()
+
+ def next(self):
+ self.it.next()
+
+ def getitem(self, space, array):
+ return self.op_flags.get_it_item[self.index](space, array, self.it)
+
+ def setitem(self, space, array, val):
+ xxx
+
+class BoxIterator(IteratorMixin, AbstractIterator):
+ index = 0
+
+class ExternalLoopIterator(IteratorMixin, AbstractIterator):
+ index = 1
+
+def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
+ ret = []
+ if space.is_w(w_op_flags, space.w_None):
+ for i in range(n):
+ ret.append(OpFlag())
+ elif not space.isinstance_w(w_op_flags, space.w_tuple) and not \
+ space.isinstance_w(w_op_flags, space.w_list):
+ raise OperationError(space.w_ValueError, space.wrap(
+ '%s must be a tuple or array of per-op flag-tuples' % name))
+ else:
+ w_lst = space.listview(w_op_flags)
+ if space.isinstance_w(w_lst[0], space.w_tuple) or \
+ space.isinstance_w(w_lst[0], space.w_list):
+ if len(w_lst) != n:
+ raise OperationError(space.w_ValueError, space.wrap(
+ '%s must be a tuple or array of per-op flag-tuples' % name))
+ for item in w_lst:
+ ret.append(parse_one_arg(space, space.listview(item)))
+ else:
+ op_flag = parse_one_arg(space, w_lst)
+ for i in range(n):
+ ret.append(op_flag)
+ return ret
+
+class OpFlag(object):
+ def __init__(self):
+ self.rw = 'r'
+ self.broadcast = True
+ self.force_contig = False
+ self.force_align = False
+ self.native_byte_order = False
+ self.tmp_copy = ''
+ self.allocate = False
+ self.get_it_item = (get_readonly_item, get_readonly_slice)
+
+def get_readonly_item(space, array, it):
+ return space.wrap(it.getitem())
+
+def get_readwrite_item(space, array, it):
+ #create a single-value view (since scalars are not views)
+ res = SliceArray(it.array.start + it.offset, [0], [0], [1,], it.array, array)
+ #it.dtype.setitem(res, 0, it.getitem())
+ return W_NDimArray(res)
+
+def get_readonly_slice(space, array, it):
+ return W_NDimArray(it.getslice().readonly())
+
+def get_readwrite_slice(space, array, it):
+ return W_NDimArray(it.getslice())
+
+def parse_op_flag(space, lst):
+ op_flag = OpFlag()
+ for w_item in lst:
+ item = space.str_w(w_item)
+ if item == 'readonly':
+ op_flag.rw = 'r'
+ elif item == 'readwrite':
+ op_flag.rw = 'rw'
+ elif item == 'writeonly':
+ op_flag.rw = 'w'
+ elif item == 'no_broadcast':
+ op_flag.broadcast = False
+ elif item == 'contig':
+ op_flag.force_contig = True
+ elif item == 'aligned':
+ op_flag.force_align = True
+ elif item == 'nbo':
+ op_flag.native_byte_order = True
+ elif item == 'copy':
+ op_flag.tmp_copy = 'r'
+ elif item == 'updateifcopy':
+ op_flag.tmp_copy = 'rw'
+ elif item == 'allocate':
+ op_flag.allocate = True
+ elif item == 'no_subtype':
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ '"no_subtype" op_flag not implemented yet'))
+ elif item == 'arraymask':
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ '"arraymask" op_flag not implemented yet'))
+ elif item == 'writemask':
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ '"writemask" op_flag not implemented yet'))
+ else:
+ raise OperationError(space.w_ValueError, space.wrap(
+ 'op_flags must be a tuple or array of per-op flag-tuples'))
+ if op_flag.rw == 'r':
+ op_flag.get_it_item = (get_readonly_item, get_readonly_slice)
+ elif op_flag.rw == 'rw':
+ op_flag.get_it_item = (get_readwrite_item, get_readwrite_slice)
+ elif op_flag.rw == 'w':
+ # XXX Extra logic needed to make sure writeonly
+ op_flag.get_it_item = (get_readwrite_item, get_readwrite_slice)
+ return op_flag
+
+def parse_func_flags(space, nditer, w_flags):
+ if space.is_w(w_flags, space.w_None):
+ return
+ elif not space.isinstance_w(w_flags, space.w_tuple) and not \
+ space.isinstance_w(w_flags, space.w_list):
+ raise OperationError(space.w_ValueError, space.wrap(
+ 'Iter global flags must be a list or tuple of strings'))
+ lst = space.listview(w_flags)
+ for w_item in lst:
+ if not space.isinstance_w(w_item, space.w_str) and not \
+ space.isinstance_w(w_item, space.w_unicode):
+ typename = space.type(w_item).getname(space)
+ raise OperationError(space.w_TypeError, space.wrap(
+ 'expected string or Unicode object, %s found' % typename))
+ item = space.str_w(w_item)
+ if item == 'external_loop':
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'nditer external_loop not implemented yet'))
+ nditer.external_loop = True
+ elif item == 'buffered':
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'nditer buffered not implemented yet'))
+ # For numpy compatability
+ nditer.buffered = True
+ elif item == 'c_index':
+ nditer.tracked_index = 'C'
+ elif item == 'f_index':
+ nditer.tracked_index = 'F'
+ elif item == 'multi_index':
+ nditer.tracked_index = 'multi'
+ elif item == 'common_dtype':
+ nditer.common_dtype = True
+ elif item == 'delay_bufalloc':
+ nditer.delay_bufalloc = True
+ elif item == 'grow_inner':
+ nditer.grow_inner = True
+ elif item == 'ranged':
+ nditer.ranged = True
+ elif item == 'refs_ok':
+ nditer.refs_ok = True
+ elif item == 'reduce_ok':
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'nditer reduce_ok not implemented yet'))
+ nditer.reduce_ok = True
+ elif item == 'zerosize_ok':
+ nditer.zerosize_ok = True
+ else:
+ raise OperationError(space.w_ValueError, space.wrap(
+ 'Unexpected iterator global flag "%s"' % item))
+ if nditer.tracked_index and nditer.external_loop:
+ raise OperationError(space.w_ValueError, space.wrap(
+ 'Iterator flag EXTERNAL_LOOP cannot be used if an index or '
+ 'multi-index is being tracked'))
+
+def is_backward(imp, order):
+ if order == 'K' or (order == 'C' and imp.order == 'C'):
+ return False
+ elif order =='F' and imp.order == 'C':
+ return True
+ else:
+ raise NotImplementedError('not implemented yet')
+
+def get_iter(space, order, arr, shape, dtype):
+ imp = arr.implementation.astype(space, dtype)
+ backward = is_backward(imp, order)
+ if (imp.strides[0] < imp.strides[-1] and not backward) or \
+ (imp.strides[0] > imp.strides[-1] and backward):
+ # flip the strides. Is this always true for multidimension?
+ strides = imp.strides[:]
+ backstrides = imp.backstrides[:]
+ shape = imp.shape[:]
+ strides.reverse()
+ backstrides.reverse()
+ shape.reverse()
+ else:
+ strides = imp.strides
+ backstrides = imp.backstrides
+ r = calculate_broadcast_strides(strides, backstrides, imp.shape,
+ shape, backward)
+ return ArrayIter(imp, imp.get_size(), shape, r[0], r[1])
+
+def get_external_loop_iter(space, order, arr, shape):
+ imp = arr.implementation
+ backward = is_backward(imp, order)
+ return SliceIterator(arr, imp.strides, imp.backstrides, shape, order=order, backward=backward)
+
+def convert_to_array_or_none(space, w_elem):
+ '''
+ None will be passed through, all others will be converted
+ '''
+ if space.is_none(w_elem):
+ return None
+ return convert_to_array(space, w_elem)
+
+
+class IndexIterator(object):
+ def __init__(self, shape, backward=False):
+ self.shape = shape
+ self.index = [0] * len(shape)
+ self.backward = backward
+
+ def next(self):
+ # TODO It's probably possible to refactor all the "next" method from each iterator
+ for i in range(len(self.shape) - 1, -1, -1):
+ if self.index[i] < self.shape[i] - 1:
+ self.index[i] += 1
+ break
+ else:
+ self.index[i] = 0
+
+ def getvalue(self):
+ if not self.backward:
+ ret = self.index[-1]
+ for i in range(len(self.shape) - 2, -1, -1):
+ ret += self.index[i] * self.shape[i - 1]
+ else:
+ ret = self.index[0]
+ for i in range(1, len(self.shape)):
+ ret += self.index[i] * self.shape[i - 1]
+ return ret
+
+class W_NDIter(W_Root):
+
+ def __init__(self, space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting,
+ w_op_axes, w_itershape, w_buffersize, order):
+ self.order = order
+ self.external_loop = False
+ self.buffered = False
+ self.tracked_index = ''
+ self.common_dtype = False
+ self.delay_bufalloc = False
+ self.grow_inner = False
+ self.ranged = False
+ self.refs_ok = False
+ self.reduce_ok = False
+ self.zerosize_ok = False
+ self.index_iter = None
+ self.done = False
+ self.first_next = True
+ self.op_axes = []
+ if space.isinstance_w(w_seq, space.w_tuple) or \
+ space.isinstance_w(w_seq, space.w_list):
+ w_seq_as_list = space.listview(w_seq)
+ self.seq = [convert_to_array_or_none(space, w_elem) for w_elem in w_seq_as_list]
+ else:
+ self.seq =[convert_to_array(space, w_seq)]
+ parse_func_flags(space, self, w_flags)
+ self.op_flags = parse_op_arg(space, 'op_flags', w_op_flags,
+ len(self.seq), parse_op_flag)
+ if not space.is_none(w_op_axes):
+ self.set_op_axes(space, w_op_axes)
+ if not space.is_none(w_op_dtypes):
+ w_seq_as_list = space.listview(w_op_dtypes)
+ self.dtypes = [decode_w_dtype(space, w_elem) for w_elem in w_seq_as_list]
+ if len(self.dtypes) != len(self.seq):
+ raise OperationError(space.w_ValueError, space.wrap(
+ "op_dtypes must be a tuple/list matching the number of ops"))
+ else:
+ self.dtypes = []
+ self.iters=[]
+ outargs = [i for i in range(len(self.seq)) \
+ if self.seq[i] is None or self.op_flags[i].rw == 'w']
+ if len(outargs) > 0:
+ out_shape = shape_agreement_multiple(space, [self.seq[i] for i in outargs])
+ else:
+ out_shape = None
+ self.shape = iter_shape = shape_agreement_multiple(space, self.seq,
+ shape=out_shape)
+ if len(outargs) > 0:
+ # Make None operands writeonly and flagged for allocation
+ out_dtype = self.dtypes[0] if len(self.dtypes) > 0 else None
+ for i in range(len(self.seq)):
+ if self.seq[i] is None:
+ self.op_flags[i].get_it_item = (get_readwrite_item,
+ get_readwrite_slice)
+ self.op_flags[i].allocate = True
+ continue
+ if self.op_flags[i].rw == 'w':
+ continue
+ out_dtype = ufuncs.find_binop_result_dtype(space,
+ self.seq[i].get_dtype(), out_dtype)
+ for i in outargs:
+ if self.seq[i] is None:
+ # XXX can we postpone allocation to later?
+ self.seq[i] = W_NDimArray.from_shape(space, iter_shape, out_dtype)
+ else:
+ if not self.op_flags[i].broadcast:
+ # Raises if ooutput cannot be broadcast
+ shape_agreement(space, iter_shape, self.seq[i], False)
+ if self.tracked_index != "":
+ if self.order == "K":
+ self.order = self.seq[0].implementation.order
+ if self.tracked_index == "multi":
+ backward = False
+ else:
+ backward = self.order != self.tracked_index
+ self.index_iter = IndexIterator(iter_shape, backward=backward)
+ if len(self.dtypes) > 0:
+ # Make sure dtypes make sense
+ for i in range(len(self.seq)):
+ selfd = self.dtypes[i]
+ seq_d = self.seq[i].get_dtype()
+ if not selfd:
+ self.dtypes[i] = seq_d
+ elif selfd != seq_d and not 'r' in self.op_flags[i].tmp_copy:
+ raise OperationError(space.w_TypeError, space.wrap(
+ "Iterator operand required copying or buffering"))
+ else:
+ #copy them from seq
+ self.dtypes = [s.get_dtype() for s in self.seq]
+ if self.external_loop:
+ for i in range(len(self.seq)):
+ self.iters.append(ExternalLoopIterator(get_external_loop_iter(space, self.order,
+ self.seq[i], iter_shape), self.op_flags[i]))
+ else:
+ for i in range(len(self.seq)):
+ self.iters.append(BoxIterator(get_iter(space, self.order,
+ self.seq[i], iter_shape, self.dtypes[i]),
+ self.op_flags[i]))
+
+ def set_op_axes(self, space, w_op_axes):
+ if space.len_w(w_op_axes) != len(self.seq):
+ raise OperationError(space.w_ValueError, space.wrap("op_axes must be a tuple/list matching the number of ops"))
+ op_axes = space.listview(w_op_axes)
+ l = -1
+ for w_axis in op_axes:
+ if not space.is_none(w_axis):
+ axis_len = space.len_w(w_axis)
+ if l == -1:
+ l = axis_len
+ elif axis_len != l:
+ raise OperationError(space.w_ValueError, space.wrap("Each entry of op_axes must have the same size"))
+ self.op_axes.append([space.int_w(x) if not space.is_none(x) else -1 for x in space.listview(w_axis)])
+ if l == -1:
+ raise OperationError(space.w_ValueError, space.wrap("If op_axes is provided, at least one list of axes must be contained within it"))
+ raise Exception('xxx TODO')
+ # Check that values make sense:
+ # - in bounds for each operand
+ # ValueError: Iterator input op_axes[0][3] (==3) is not a valid axis of op[0], which has 2 dimensions
+ # - no repeat axis
+ # ValueError: The 'op_axes' provided to the iterator constructor for operand 1 contained duplicate value 0
+
+ def descr_iter(self, space):
+ return space.wrap(self)
+
+ def descr_getitem(self, space, w_idx):
+ idx = space.int_w(w_idx)
+ try:
+ ret = space.wrap(self.iters[idx].getitem(space, self.seq[idx]))
+ except IndexError:
+ raise OperationError(space.w_IndexError, space.wrap("Iterator operand index %d is out of bounds" % idx))
+ return ret
+
+ def descr_setitem(self, space, w_idx, w_value):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_len(self, space):
+ space.wrap(len(self.iters))
+
+ def descr_next(self, space):
+ for it in self.iters:
+ if not it.done():
+ break
+ else:
+ self.done = True
+ raise OperationError(space.w_StopIteration, space.w_None)
+ res = []
+ if self.index_iter:
+ if not self.first_next:
+ self.index_iter.next()
+ else:
+ self.first_next = False
+ for i in range(len(self.iters)):
+ res.append(self.iters[i].getitem(space, self.seq[i]))
+ self.iters[i].next()
+ if len(res) <2:
+ return res[0]
+ return space.newtuple(res)
+
+ def iternext(self):
+ if self.index_iter:
+ self.index_iter.next()
+ for i in range(len(self.iters)):
+ self.iters[i].next()
+ for it in self.iters:
+ if not it.done():
+ break
+ else:
+ self.done = True
+ return self.done
+ return self.done
+
+ def descr_iternext(self, space):
+ return space.wrap(self.iternext())
+
+ def descr_copy(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_debug_print(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_enable_external_loop(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ @unwrap_spec(axis=int)
+ def descr_remove_axis(self, space, axis):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_remove_multi_index(self, space, w_multi_index):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_reset(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_operands(self, space):
+ l_w = []
+ for op in self.seq:
+ l_w.append(op.descr_view(space))
+ return space.newlist(l_w)
+
+ def descr_get_dtypes(self, space):
+ res = [None] * len(self.seq)
+ for i in range(len(self.seq)):
+ res[i] = self.seq[i].descr_get_dtype(space)
+ return space.newtuple(res)
+
+ def descr_get_finished(self, space):
+ return space.wrap(self.done)
+
+ def descr_get_has_delayed_bufalloc(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_has_index(self, space):
+ return space.wrap(self.tracked_index in ["C", "F"])
+
+ def descr_get_index(self, space):
+ if not self.tracked_index in ["C", "F"]:
+ raise OperationError(space.w_ValueError, space.wrap("Iterator does not have an index"))
+ if self.done:
+ raise OperationError(space.w_ValueError, space.wrap("Iterator is past the end"))
+ return space.wrap(self.index_iter.getvalue())
+
+ def descr_get_has_multi_index(self, space):
+ return space.wrap(self.tracked_index == "multi")
+
+ def descr_get_multi_index(self, space):
+ if not self.tracked_index == "multi":
+ raise OperationError(space.w_ValueError, space.wrap("Iterator is not tracking a multi-index"))
+ if self.done:
+ raise OperationError(space.w_ValueError, space.wrap("Iterator is past the end"))
+ return space.newtuple([space.wrap(x) for x in self.index_iter.index])
+
+ def descr_get_iterationneedsapi(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_iterindex(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_itersize(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_itviews(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_ndim(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_nop(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_shape(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+ def descr_get_value(self, space):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ 'not implemented yet'))
+
+
+ at unwrap_spec(w_flags = WrappedDefault(None), w_op_flags=WrappedDefault(None),
+ w_op_dtypes = WrappedDefault(None), order=str,
+ w_casting=WrappedDefault(None), w_op_axes=WrappedDefault(None),
+ w_itershape=WrappedDefault(None), w_buffersize=WrappedDefault(None))
+def nditer(space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting, w_op_axes,
+ w_itershape, w_buffersize, order='K'):
+ return W_NDIter(space, w_seq, w_flags, w_op_flags, w_op_dtypes, w_casting, w_op_axes,
+ w_itershape, w_buffersize, order)
+
+W_NDIter.typedef = TypeDef(
+ 'nditer',
+ __iter__ = interp2app(W_NDIter.descr_iter),
+ __getitem__ = interp2app(W_NDIter.descr_getitem),
+ __setitem__ = interp2app(W_NDIter.descr_setitem),
+ __len__ = interp2app(W_NDIter.descr_len),
+
+ next = interp2app(W_NDIter.descr_next),
+ iternext = interp2app(W_NDIter.descr_iternext),
+ copy = interp2app(W_NDIter.descr_copy),
+ debug_print = interp2app(W_NDIter.descr_debug_print),
+ enable_external_loop = interp2app(W_NDIter.descr_enable_external_loop),
+ remove_axis = interp2app(W_NDIter.descr_remove_axis),
+ remove_multi_index = interp2app(W_NDIter.descr_remove_multi_index),
+ reset = interp2app(W_NDIter.descr_reset),
+
+ operands = GetSetProperty(W_NDIter.descr_get_operands),
+ dtypes = GetSetProperty(W_NDIter.descr_get_dtypes),
+ finished = GetSetProperty(W_NDIter.descr_get_finished),
+ has_delayed_bufalloc = GetSetProperty(W_NDIter.descr_get_has_delayed_bufalloc),
+ has_index = GetSetProperty(W_NDIter.descr_get_has_index),
+ index = GetSetProperty(W_NDIter.descr_get_index),
+ has_multi_index = GetSetProperty(W_NDIter.descr_get_has_multi_index),
+ multi_index = GetSetProperty(W_NDIter.descr_get_multi_index),
+ iterationneedsapi = GetSetProperty(W_NDIter.descr_get_iterationneedsapi),
+ iterindex = GetSetProperty(W_NDIter.descr_get_iterindex),
+ itersize = GetSetProperty(W_NDIter.descr_get_itersize),
+ itviews = GetSetProperty(W_NDIter.descr_get_itviews),
+ ndim = GetSetProperty(W_NDIter.descr_get_ndim),
+ nop = GetSetProperty(W_NDIter.descr_get_nop),
+ shape = GetSetProperty(W_NDIter.descr_get_shape),
+ value = GetSetProperty(W_NDIter.descr_get_value),
+)
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -282,14 +282,16 @@
@jit.unroll_safe
-def shape_agreement_multiple(space, array_list):
+def shape_agreement_multiple(space, array_list, shape=None):
""" call shape_agreement recursively, allow elements from array_list to
be None (like w_out)
"""
- shape = array_list[0].get_shape()
- for arr in array_list[1:]:
+ for arr in array_list:
if not space.is_none(arr):
- shape = shape_agreement(space, shape, arr)
+ if shape is None:
+ shape = arr.get_shape()
+ else:
+ shape = shape_agreement(space, shape, arr)
return shape
diff --git a/pypy/module/micronumpy/test/test_nditer.py b/pypy/module/micronumpy/test/test_nditer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -0,0 +1,294 @@
+import py
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+
+class AppTestNDIter(BaseNumpyAppTest):
+ def test_basic(self):
+ from numpy import arange, nditer
+ a = arange(6).reshape(2,3)
+ r = []
+ for x in nditer(a):
+ r.append(x)
+ assert r == [0, 1, 2, 3, 4, 5]
+ r = []
+
+ for x in nditer(a.T):
+ r.append(x)
+ assert r == [0, 1, 2, 3, 4, 5]
+
+ def test_order(self):
+ from numpy import arange, nditer
+ a = arange(6).reshape(2,3)
+ r = []
+ for x in nditer(a, order='C'):
+ r.append(x)
+ assert r == [0, 1, 2, 3, 4, 5]
+ r = []
+ for x in nditer(a, order='F'):
+ r.append(x)
+ assert r == [0, 3, 1, 4, 2, 5]
+
+ def test_readwrite(self):
+ from numpy import arange, nditer
+ a = arange(6).reshape(2,3)
+ for x in nditer(a, op_flags=['readwrite']):
+ x[...] = 2 * x
+ assert (a == [[0, 2, 4], [6, 8, 10]]).all()
+
+ def test_external_loop(self):
+ from numpy import arange, nditer, array
+ a = arange(24).reshape(2, 3, 4)
+ import sys
+ if '__pypy__' in sys.builtin_module_names:
+ raises(NotImplementedError, nditer, a, flags=['external_loop'])
+ skip('nditer external_loop not implmented')
+ r = []
+ n = 0
+ for x in nditer(a, flags=['external_loop']):
+ r.append(x)
+ n += 1
+ assert n == 1
+ assert (array(r) == range(24)).all()
+ r = []
+ n = 0
+ for x in nditer(a, flags=['external_loop'], order='F'):
+ r.append(x)
+ n += 1
+ assert n == 12
+ assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17], [ 9, 21], [ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19], [11, 23]]).all()
+ e = raises(ValueError, 'r[0][0] = 0')
+ assert str(e.value) == 'assignment destination is read-only'
+ r = []
+ for x in nditer(a.T, flags=['external_loop'], order='F'):
+ r.append(x)
+ array_r = array(r)
+ assert len(array_r.shape) == 2
+ assert array_r.shape == (1,24)
+ assert (array(r) == arange(24)).all()
+
+ def test_index(self):
+ from numpy import arange, nditer
+ a = arange(6).reshape(2,3)
+
+ r = []
+ it = nditer(a, flags=['c_index'])
+ assert it.has_index
+ for value in it:
+ r.append((value, it.index))
+ assert r == [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)]
+ exc = None
+ try:
+ it.index
+ except ValueError, e:
+ exc = e
+ assert exc
+
+ r = []
+ it = nditer(a, flags=['f_index'])
+ assert it.has_index
+ for value in it:
+ r.append((value, it.index))
+ assert r == [(0, 0), (1, 2), (2, 4), (3, 1), (4, 3), (5, 5)]
+
+ @py.test.mark.xfail(reason="Fortran order not implemented")
+ def test_iters_with_different_order(self):
+ from numpy import nditer, array
+
+ a = array([[1, 2], [3, 4]], order="C")
+ b = array([[1, 2], [3, 4]], order="F")
+
+ it = nditer([a, b])
+
+ assert list(it) == zip(range(1, 5), range(1, 5))
+
+ def test_interface(self):
+ from numpy import arange, nditer, zeros
+ import sys
+ a = arange(6).reshape(2,3)
+ r = []
+ it = nditer(a, flags=['f_index'])
+ while not it.finished:
+ r.append((it[0], it.index))
+ it.iternext()
+ assert r == [(0, 0), (1, 2), (2, 4), (3, 1), (4, 3), (5, 5)]
+ it = nditer(a, flags=['multi_index'], op_flags=['writeonly'])
+ if '__pypy__' in sys.builtin_module_names:
+ raises(NotImplementedError, 'it[0] = 3')
+ skip('nditer.__setitem__ not implmented')
+ while not it.finished:
+ it[0] = it.multi_index[1] - it.multi_index[0]
+ it.iternext()
+ assert (a == [[0, 1, 2], [-1, 0, 1]]).all()
+ # b = zeros((2, 3))
+ # exc = raises(ValueError, nditer, b, flags=['c_index', 'external_loop'])
+ # assert str(exc.value).startswith("Iterator flag EXTERNAL_LOOP cannot")
+
+ def test_buffered(self):
+ from numpy import arange, nditer, array
+ a = arange(6).reshape(2,3)
+ import sys
+ if '__pypy__' in sys.builtin_module_names:
+ raises(NotImplementedError, nditer, a, flags=['buffered'])
+ skip('nditer buffered not implmented')
+ r = []
+ for x in nditer(a, flags=['external_loop', 'buffered'], order='F'):
+ r.append(x)
+ array_r = array(r)
+ assert len(array_r.shape) == 2
+ assert array_r.shape == (1, 6)
+ assert (array_r == [0, 3, 1, 4, 2, 5]).all()
+
+ def test_op_dtype(self):
+ from numpy import arange, nditer, sqrt, array
+ a = arange(6).reshape(2,3) - 3
+ exc = raises(TypeError, nditer, a, op_dtypes=['complex'])
+ assert str(exc.value).startswith("Iterator operand required copying or buffering")
+ r = []
+ for x in nditer(a, op_flags=['readonly','copy'],
+ op_dtypes=['complex128']):
+ r.append(sqrt(x))
+ assert abs((array(r) - [1.73205080757j, 1.41421356237j, 1j, 0j,
+ 1+0j, 1.41421356237+0j]).sum()) < 1e-5
+ r = []
+ for x in nditer(a, op_flags=['copy'],
+ op_dtypes=['complex128']):
+ r.append(sqrt(x))
+ assert abs((array(r) - [1.73205080757j, 1.41421356237j, 1j, 0j,
+ 1+0j, 1.41421356237+0j]).sum()) < 1e-5
+
+ def test_casting(self):
+ from numpy import arange, nditer
+ import sys
+ a = arange(6.)
+ if '__pypy__' in sys.builtin_module_names:
+ raises(NotImplementedError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
+ skip('nditer casting not implemented yet')
+ exc = raises(TypeError, nditer, a, flags=['buffered'], op_dtypes=['float32'])
+ assert str(exc.value).startswith("Iterator operand 0 dtype could not be cast")
+ r = []
+ for x in nditer(a, flags=['buffered'], op_dtypes=['float32'],
+ casting='same_kind'):
+ r.append(x)
+ assert r == [0., 1., 2., 3., 4., 5.]
+ exc = raises(TypeError, nditer, a, flags=['buffered'],
+ op_dtypes=['int32'], casting='same_kind')
+ assert str(exc.value).startswith("Iterator operand 0 dtype could not be cast")
+ r = []
+ b = arange(6)
+ exc = raises(TypeError, nditer, b, flags=['buffered'], op_dtypes=['float64'],
+ op_flags=['readwrite'], casting='same_kind')
+ assert str(exc.value).startswith("Iterator requested dtype could not be cast")
+
+ def test_broadcast(self):
+ from numpy import arange, nditer
+ a = arange(3)
+ b = arange(6).reshape(2,3)
+ r = []
+ for x,y in nditer([a, b]):
+ r.append((x, y))
+ assert r == [(0, 0), (1, 1), (2, 2), (0, 3), (1, 4), (2, 5)]
+ a = arange(2)
+ exc = raises(ValueError, nditer, [a, b])
+ assert str(exc.value).find('shapes (2) (2,3)') > 0
+
+ def test_outarg(self):
+ from numpy import nditer, zeros, arange
+ import sys
+ if '__pypy__' in sys.builtin_module_names:
+ raises(NotImplementedError, nditer, [1, 2], flags=['external_loop'])
+ skip('nditer external_loop not implmented')
+
+ def square1(a):
+ it = nditer([a, None])
+ for x,y in it:
+ y[...] = x*x
+ return it.operands[1]
+ assert (square1([1, 2, 3]) == [1, 4, 9]).all()
+
+ def square2(a, out=None):
+ it = nditer([a, out], flags=['external_loop', 'buffered'],
+ op_flags=[['readonly'],
+ ['writeonly', 'allocate', 'no_broadcast']])
+ for x,y in it:
+ y[...] = x*x
+ return it.operands[1]
+ assert (square2([1, 2, 3]) == [1, 4, 9]).all()
+ b = zeros((3, ))
+ c = square2([1, 2, 3], out=b)
+ assert (c == [1., 4., 9.]).all()
+ assert (b == c).all()
+ exc = raises(ValueError, square2, arange(6).reshape(2, 3), out=b)
+ assert str(exc.value).find('cannot be broadcasted') > 0
+
+ def test_outer_product(self):
+ from numpy import nditer, arange
+ a = arange(3)
+ import sys
+ if '__pypy__' in sys.builtin_module_names:
+ raises(NotImplementedError, nditer, a, flags=['external_loop'])
+ skip('nditer external_loop not implmented')
+ b = arange(8).reshape(2,4)
+ it = nditer([a, b, None], flags=['external_loop'],
+ op_axes=[[0, -1, -1], [-1, 0, 1], None])
+ for x, y, z in it:
+ z[...] = x*y
+ assert it.operands[2].shape == (3, 2, 4)
+ for i in range(a.size):
+ assert (it.operands[2][i] == a[i]*b).all()
+
+ def test_reduction(self):
+ from numpy import nditer, arange, array
+ import sys
+ a = arange(24).reshape(2, 3, 4)
+ b = array(0)
+ if '__pypy__' in sys.builtin_module_names:
+ raises(NotImplementedError, nditer, [a, b], flags=['reduce_ok'])
+ skip('nditer reduce_ok not implemented yet')
+ #reduction operands must be readwrite
+ for x, y in nditer([a, b], flags=['reduce_ok', 'external_loop'],
+ op_flags=[['readonly'], ['readwrite']]):
+ y[...] += x
+ assert b == 276
+ assert b == a.sum()
+
+ # reduction and allocation requires op_axes and initialization
+ it = nditer([a, None], flags=['reduce_ok', 'external_loop'],
+ op_flags=[['readonly'], ['readwrite', 'allocate']],
+ op_axes=[None, [0,1,-1]])
+ it.operands[1][...] = 0
+ for x, y in it:
+ y[...] += x
+
+ assert (it.operands[1] == [[6, 22, 38], [54, 70, 86]]).all()
+ assert (it.operands[1] == a.sum(axis=2)).all()
+
+ # previous example with buffering, requires more flags and reset
+ it = nditer([a, None], flags=['reduce_ok', 'external_loop',
+ 'buffered', 'delay_bufalloc'],
+ op_flags=[['readonly'], ['readwrite', 'allocate']],
+ op_axes=[None, [0,1,-1]])
+ it.operands[1][...] = 0
+ it.reset()
+ for x, y in it:
+ y[...] += x
+
+ assert (it.operands[1] == [[6, 22, 38], [54, 70, 86]]).all()
+ assert (it.operands[1] == a.sum(axis=2)).all()
+
+ def test_get_dtypes(self):
+ from numpy import array, nditer
+ x = array([1, 2])
+ y = array([1.0, 2.0])
+ assert nditer([x, y]).dtypes == (x.dtype, y.dtype)
+
+ def test_multi_index(self):
+ import numpy as np
+ a = np.arange(6).reshape(2, 3)
+ it = np.nditer(a, flags=['multi_index'])
+ res = []
+ while not it.finished:
+ res.append((it[0], it.multi_index))
+ it.iternext()
+ assert res == [(0, (0, 0)), (1, (0, 1)),
+ (2, (0, 2)), (3, (1, 0)),
+ (4, (1, 1)), (5, (1, 2))]
More information about the pypy-commit
mailing list