[pypy-commit] pypy numpy-refactor: put Chunks in strides.py
bdkearns
noreply at buildbot.pypy.org
Thu Feb 27 02:00:00 CET 2014
Author: Brian Kearns <bdkearns at gmail.com>
Branch: numpy-refactor
Changeset: r69485:dd44430725b8
Date: 2014-02-26 19:32 -0500
http://bitbucket.org/pypy/pypy/changeset/dd44430725b8/
Log: put Chunks in strides.py
diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
--- a/pypy/module/micronumpy/appbridge.py
+++ b/pypy/module/micronumpy/appbridge.py
@@ -1,5 +1,6 @@
from rpython.rlib.objectmodel import specialize
+
class AppBridgeCache(object):
w__mean = None
w__var = None
@@ -20,6 +21,7 @@
setattr(self, 'w_' + name, w_method)
return space.call_args(w_method, args)
+
def set_string_function(space, w_f, w_repr):
cache = get_appbridge_cache(space)
if space.is_true(w_repr):
@@ -27,5 +29,6 @@
else:
cache.w_array_str = w_f
+
def get_appbridge_cache(space):
return space.fromcache(AppBridgeCache)
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -14,6 +14,7 @@
space.call_method(w_ret, '__array_finalize__', w_instance)
return w_ret
+
class ArrayArgumentException(Exception):
pass
@@ -46,7 +47,7 @@
strides, backstrides = calc_strides(shape, dtype, order)
if w_base is not None:
if owning:
- raise OperationError(space.w_ValueError,
+ raise OperationError(space.w_ValueError,
space.wrap("Cannot have owning=True when specifying a buffer"))
if writable:
impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides,
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,9 +1,9 @@
from pypy.module.micronumpy import support, loop, iter
from pypy.module.micronumpy.base import convert_to_array, W_NDimArray,\
ArrayArgumentException
-from pypy.module.micronumpy.strides import calc_new_strides, shape_agreement,\
- calculate_broadcast_strides, calculate_dot_strides
-from pypy.module.micronumpy.iter import Chunk, Chunks, NewAxisChunk, RecordChunk
+from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
+ RecordChunk, calc_new_strides, shape_agreement, calculate_broadcast_strides,
+ calculate_dot_strides)
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.buffer import RWBuffer
from rpython.rlib import jit
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -1,8 +1,7 @@
from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
from pypy.module.micronumpy import loop, interp_dtype, interp_ufuncs
-from pypy.module.micronumpy.iter import Chunk, Chunks
-from pypy.module.micronumpy.strides import shape_agreement,\
- shape_agreement_multiple
+from pypy.module.micronumpy.strides import Chunk, Chunks, shape_agreement, \
+ shape_agreement_multiple
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import unwrap_spec
from pypy.module.micronumpy.conversion_utils import clipmode_converter
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
--- a/pypy/module/micronumpy/iter.py
+++ b/pypy/module/micronumpy/iter.py
@@ -1,4 +1,3 @@
-
""" This is a mini-tutorial on iterators, strides, and
memory layout. It assumes you are familiar with the terms, see
http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html
@@ -42,99 +41,10 @@
dimension, perhaps we could overflow times in one big step.
"""
-from pypy.module.micronumpy.strides import enumerate_chunks,\
- calculate_slice_strides
from pypy.module.micronumpy.base import W_NDimArray
from pypy.module.micronumpy import support
from rpython.rlib import jit
-# structures to describe slicing
-
-class BaseChunk(object):
- pass
-
-class RecordChunk(BaseChunk):
- def __init__(self, name):
- self.name = name
-
- def apply(self, space, orig_arr):
- arr = orig_arr.implementation
- ofs, subdtype = arr.dtype.fields[self.name]
- # ofs only changes start
- # create a view of the original array by extending
- # the shape, strides, backstrides of the array
- from pypy.module.micronumpy.support import calc_strides
- strides, backstrides = calc_strides(subdtype.shape,
- subdtype.subdtype, arr.order)
- final_shape = arr.shape + subdtype.shape
- final_strides = arr.get_strides() + strides
- final_backstrides = arr.get_backstrides() + backstrides
- final_dtype = subdtype
- if subdtype.subdtype:
- final_dtype = subdtype.subdtype
- return W_NDimArray.new_slice(space, arr.start + ofs, final_strides,
- final_backstrides,
- final_shape, arr, orig_arr, final_dtype)
-
-class Chunks(BaseChunk):
- def __init__(self, l):
- self.l = l
-
- @jit.unroll_safe
- def extend_shape(self, old_shape):
- shape = []
- i = -1
- for i, c in enumerate_chunks(self.l):
- if c.step != 0:
- shape.append(c.lgt)
- s = i + 1
- assert s >= 0
- return shape[:] + old_shape[s:]
-
- def apply(self, space, orig_arr):
- arr = orig_arr.implementation
- shape = self.extend_shape(arr.shape)
- r = calculate_slice_strides(arr.shape, arr.start, arr.get_strides(),
- arr.get_backstrides(), self.l)
- _, start, strides, backstrides = r
- return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
- shape[:], arr, orig_arr)
-
-
-class Chunk(BaseChunk):
- axis_step = 1
-
- def __init__(self, start, stop, step, lgt):
- self.start = start
- self.stop = stop
- self.step = step
- self.lgt = lgt
-
- def __repr__(self):
- return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
- self.lgt)
-
-class NewAxisChunk(Chunk):
- start = 0
- stop = 1
- step = 1
- lgt = 1
- axis_step = 0
-
- def __init__(self):
- pass
-
-class BaseTransform(object):
- pass
-
-class ViewTransform(BaseTransform):
- def __init__(self, chunks):
- # 4-tuple specifying slicing
- self.chunks = chunks
-
-class BroadcastTransform(BaseTransform):
- def __init__(self, res_shape):
- self.res_shape = res_shape
class PureShapeIterator(object):
def __init__(self, shape, idx_w):
@@ -168,18 +78,21 @@
def get_index(self, space, shapelen):
return [space.wrap(self.indexes[i]) for i in range(shapelen)]
+
class BaseArrayIterator(object):
def next(self):
- raise NotImplementedError # purely abstract base class
+ raise NotImplementedError # purely abstract base class
def setitem(self, elem):
raise NotImplementedError
def set_scalar_object(self, value):
- raise NotImplementedError # works only on scalars
+ raise NotImplementedError # works only on scalars
+
class ConcreteArrayIterator(BaseArrayIterator):
_immutable_fields_ = ['array', 'skip', 'size']
+
def __init__(self, array):
self.array = array
self.offset = 0
@@ -207,6 +120,7 @@
def reset(self):
self.offset %= self.size
+
class OneDimViewIterator(ConcreteArrayIterator):
def __init__(self, array, start, strides, shape):
self.array = array
@@ -238,6 +152,7 @@
def get_index(self, d):
return self.index
+
class MultiDimViewIterator(ConcreteArrayIterator):
def __init__(self, array, start, strides, backstrides, shape):
self.indexes = [0] * len(shape)
@@ -276,7 +191,7 @@
remaining_step = (self.indexes[i] + step) // self.shape[i]
this_i_step = step - remaining_step * self.shape[i]
self.offset += self.strides[i] * this_i_step
- self.indexes[i] = self.indexes[i] + this_i_step
+ self.indexes[i] = self.indexes[i] + this_i_step
step = remaining_step
else:
self._done = True
@@ -290,6 +205,7 @@
def get_index(self, d):
return self.indexes[d]
+
class AxisIterator(BaseArrayIterator):
def __init__(self, array, shape, dim, cumulative):
self.shape = shape
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -1,4 +1,3 @@
-
""" This file is the main run loop as well as evaluation loops for various
operations. This is the place to look for all the computations that iterate
over all the array elements.
@@ -660,4 +659,3 @@
out_iter.setitem(arr.getitem_index(space, indexes))
iter.next()
out_iter.next()
-
diff --git a/pypy/module/micronumpy/sort.py b/pypy/module/micronumpy/sort.py
--- a/pypy/module/micronumpy/sort.py
+++ b/pypy/module/micronumpy/sort.py
@@ -17,6 +17,7 @@
INT_SIZE = rffi.sizeof(lltype.Signed)
+
def make_argsort_function(space, itemtype, comp_type, count=1):
TP = itemtype.T
step = rffi.sizeof(TP)
@@ -167,6 +168,7 @@
return argsort
+
def argsort_array(arr, space, w_axis):
cache = space.fromcache(ArgSortCache) # that populates ArgSortClasses
itemtype = arr.dtype.itemtype
@@ -179,10 +181,6 @@
"sorting of non-numeric types '%s' is not implemented",
arr.dtype.get_name())
-all_types = (types.all_float_types + types.all_complex_types +
- types.all_int_types)
-all_types = [i for i in all_types if not '_mixin_' in i[0].__dict__]
-all_types = unrolling_iterable(all_types)
def make_sort_function(space, itemtype, comp_type, count=1):
TP = itemtype.T
@@ -307,8 +305,9 @@
return sort
+
def sort_array(arr, space, w_axis, w_order):
- cache = space.fromcache(SortCache) # that populates SortClasses
+ cache = space.fromcache(SortCache) # that populates SortClasses
itemtype = arr.dtype.itemtype
if arr.dtype.byteorder == NPY.OPPBYTE:
raise oefmt(space.w_NotImplementedError,
@@ -327,6 +326,7 @@
all_types = [i for i in all_types if not issubclass(i[0], types.Float16)]
all_types = unrolling_iterable(all_types)
+
class ArgSortCache(object):
built = False
@@ -341,7 +341,7 @@
else:
cache[cls] = make_argsort_function(space, cls, it)
self.cache = cache
- self._lookup = specialize.memo()(lambda tp : cache[tp[0]])
+ self._lookup = specialize.memo()(lambda tp: cache[tp[0]])
class SortCache(object):
@@ -358,4 +358,4 @@
else:
cache[cls] = make_sort_function(space, cls, it)
self.cache = cache
- self._lookup = specialize.memo()(lambda tp : cache[tp[0]])
+ self._lookup = specialize.memo()(lambda tp: cache[tp[0]])
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,9 +1,105 @@
from rpython.rlib import jit
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
from pypy.module.micronumpy.base import W_NDimArray
from pypy.module.micronumpy import support
from pypy.module.micronumpy import constants as NPY
+
+# structures to describe slicing
+
+class BaseChunk(object):
+ pass
+
+
+class RecordChunk(BaseChunk):
+ def __init__(self, name):
+ self.name = name
+
+ def apply(self, space, orig_arr):
+ arr = orig_arr.implementation
+ ofs, subdtype = arr.dtype.fields[self.name]
+ # ofs only changes start
+ # create a view of the original array by extending
+ # the shape, strides, backstrides of the array
+ from pypy.module.micronumpy.support import calc_strides
+ strides, backstrides = calc_strides(subdtype.shape,
+ subdtype.subdtype, arr.order)
+ final_shape = arr.shape + subdtype.shape
+ final_strides = arr.get_strides() + strides
+ final_backstrides = arr.get_backstrides() + backstrides
+ final_dtype = subdtype
+ if subdtype.subdtype:
+ final_dtype = subdtype.subdtype
+ return W_NDimArray.new_slice(space, arr.start + ofs, final_strides,
+ final_backstrides,
+ final_shape, arr, orig_arr, final_dtype)
+
+
+class Chunks(BaseChunk):
+ def __init__(self, l):
+ self.l = l
+
+ @jit.unroll_safe
+ def extend_shape(self, old_shape):
+ shape = []
+ i = -1
+ for i, c in enumerate_chunks(self.l):
+ if c.step != 0:
+ shape.append(c.lgt)
+ s = i + 1
+ assert s >= 0
+ return shape[:] + old_shape[s:]
+
+ def apply(self, space, orig_arr):
+ arr = orig_arr.implementation
+ shape = self.extend_shape(arr.shape)
+ r = calculate_slice_strides(arr.shape, arr.start, arr.get_strides(),
+ arr.get_backstrides(), self.l)
+ _, start, strides, backstrides = r
+ return W_NDimArray.new_slice(space, start, strides[:], backstrides[:],
+ shape[:], arr, orig_arr)
+
+
+class Chunk(BaseChunk):
+ axis_step = 1
+
+ def __init__(self, start, stop, step, lgt):
+ self.start = start
+ self.stop = stop
+ self.step = step
+ self.lgt = lgt
+
+ def __repr__(self):
+ return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
+ self.lgt)
+
+
+class NewAxisChunk(Chunk):
+ start = 0
+ stop = 1
+ step = 1
+ lgt = 1
+ axis_step = 0
+
+ def __init__(self):
+ pass
+
+
+class BaseTransform(object):
+ pass
+
+
+class ViewTransform(BaseTransform):
+ def __init__(self, chunks):
+ # 4-tuple specifying slicing
+ self.chunks = chunks
+
+
+class BroadcastTransform(BaseTransform):
+ def __init__(self, res_shape):
+ self.res_shape = res_shape
+
+
@jit.look_inside_iff(lambda chunks: jit.isconstant(len(chunks)))
def enumerate_chunks(chunks):
result = []
@@ -13,9 +109,9 @@
result.append((i, chunk))
return result
+
@jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
- jit.isconstant(len(chunks))
-)
+ jit.isconstant(len(chunks)))
def calculate_slice_strides(shape, start, strides, backstrides, chunks):
size = 0
for chunk in chunks:
@@ -46,6 +142,7 @@
rshape += shape[s:]
return rshape, rstart, rstrides, rbackstrides
+
def calculate_broadcast_strides(strides, backstrides, orig_shape, res_shape, backwards=False):
rstrides = []
rbackstrides = []
@@ -64,16 +161,18 @@
rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides
return rstrides, rbackstrides
+
def is_single_elem(space, w_elem, is_rec_type):
if (is_rec_type and space.isinstance_w(w_elem, space.w_tuple)):
return True
if (space.isinstance_w(w_elem, space.w_tuple) or
- space.isinstance_w(w_elem, space.w_list)):
+ space.isinstance_w(w_elem, space.w_list)):
return False
if isinstance(w_elem, W_NDimArray) and not w_elem.is_scalar():
return False
return True
+
def find_shape_and_elems(space, w_iterable, dtype):
isstr = space.isinstance_w(w_iterable, space.w_str)
if not support.issequence_w(space, w_iterable) or isstr:
@@ -99,7 +198,7 @@
size = space.len_w(batch[0])
for w_elem in batch:
if (is_single_elem(space, w_elem, is_rec_type) or
- space.len_w(w_elem) != size):
+ space.len_w(w_elem) != size):
raise OperationError(space.w_ValueError, space.wrap(
"setting an array element with a sequence"))
w_array = space.lookup(w_elem, '__array__')
@@ -112,12 +211,13 @@
shape.append(size)
batch = new_batch
+
def to_coords(space, shape, size, order, w_item_or_slice):
'''Returns a start coord, step, and length.
'''
start = lngth = step = 0
if not (space.isinstance_w(w_item_or_slice, space.w_int) or
- space.isinstance_w(w_item_or_slice, space.w_slice)):
+ space.isinstance_w(w_item_or_slice, space.w_slice)):
raise OperationError(space.w_IndexError,
space.wrap('unsupported iterator index'))
@@ -135,6 +235,7 @@
i //= shape[s]
return coords, step, lngth
+
@jit.unroll_safe
def shape_agreement(space, shape1, w_arr2, broadcast_down=True):
if w_arr2 is None:
@@ -158,6 +259,7 @@
)
return ret
+
@jit.unroll_safe
def shape_agreement_multiple(space, array_list):
""" call shape_agreement recursively, allow elements from array_list to
@@ -169,6 +271,7 @@
shape = shape_agreement(space, shape, arr)
return shape
+
def _shape_agreement(shape1, shape2):
""" Checks agreement about two shapes with respect to broadcasting. Returns
the resulting shape.
@@ -207,6 +310,7 @@
endshape[i] = remainder[i]
return endshape
+
def get_shape_from_iterable(space, old_size, w_iterable):
new_size = 0
new_shape = []
@@ -225,8 +329,8 @@
s = space.int_w(elem)
if s < 0:
if neg_dim >= 0:
- raise OperationError(space.w_ValueError, space.wrap(
- "can only specify one unknown dimension"))
+ raise oefmt(space.w_ValueError,
+ "can only specify one unknown dimension")
s = 1
neg_dim = i
new_size *= s
@@ -240,6 +344,7 @@
space.wrap("total size of new array must be unchanged"))
return new_shape
+
# Recalculating strides. Find the steps that the iteration does for each
# dimension, given the stride and shape. Then try to create a new stride that
# fits the new shape, using those steps. If there is a shape/step mismatch
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,5 +1,6 @@
from rpython.rlib import jit
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
+
def issequence_w(space, w_obj):
from pypy.module.micronumpy.base import W_NDimArray
@@ -7,6 +8,7 @@
space.isinstance_w(w_obj, space.w_list) or
isinstance(w_obj, W_NDimArray))
+
def index_w(space, w_obj):
try:
return space.int_w(space.index(w_obj))
@@ -14,8 +16,8 @@
try:
return space.int_w(space.int(w_obj))
except OperationError:
- raise OperationError(space.w_IndexError, space.wrap(
- "cannot convert index to integer"))
+ raise oefmt(space.w_IndexError, "cannot convert index to integer")
+
@jit.unroll_safe
def product(s):
@@ -24,6 +26,7 @@
i *= x
return i
+
@jit.unroll_safe
def calc_strides(shape, dtype, order):
strides = []
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -3,7 +3,7 @@
from pypy.conftest import option
from pypy.module.micronumpy.appbridge import get_appbridge_cache
-from pypy.module.micronumpy.iter import Chunk, Chunks
+from pypy.module.micronumpy.strides import Chunk, Chunks
from pypy.module.micronumpy.interp_numarray import W_NDimArray
from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -11,7 +11,8 @@
from rpython.rlib.rawstorage import (alloc_raw_storage,
raw_storage_getitem_unaligned, raw_storage_setitem_unaligned)
from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, most_neg_value_of, LONG_BIT
+from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, \
+ most_neg_value_of, LONG_BIT
from rpython.rtyper.lltypesystem import lltype, rffi
from rpython.rlib.rstruct.runpack import runpack
from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
More information about the pypy-commit
mailing list