[pypy-commit] pypy default: add product_check() to test overflow, be more careful about where this is needed
mattip
noreply at buildbot.pypy.org
Mon Sep 21 21:28:06 CEST 2015
Author: mattip <matti.picus at gmail.com>
Branch:
Changeset: r79746:8e3a27cadc69
Date: 2015-09-21 22:27 +0300
http://bitbucket.org/pypy/pypy/changeset/8e3a27cadc69/
Log: add product_check() to test overflow, be more careful about where
this is needed
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -1,6 +1,7 @@
from pypy.interpreter.baseobjspace import W_Root
from pypy.interpreter.error import OperationError, oefmt
from rpython.tool.pairtype import extendabletype
+from rpython.rlib.rarithmetic import ovfcheck
from pypy.module.micronumpy import support
from pypy.module.micronumpy import constants as NPY
@@ -44,9 +45,9 @@
raise oefmt(space.w_ValueError,
"sequence too large; must be smaller than %d", NPY.MAXDIMS)
try:
- support.product(shape) * dtype.elsize
+ ovfcheck(support.product_check(shape) * dtype.elsize)
except OverflowError as e:
- raise oefmt(space.w_ValueError, "array is too big")
+ raise oefmt(space.w_ValueError, "array is too big.")
strides, backstrides = calc_strides(shape, dtype.base, order)
impl = concrete.ConcreteArray(shape, dtype.base, order, strides,
backstrides, zero=zero)
@@ -68,9 +69,9 @@
raise oefmt(space.w_ValueError,
"sequence too large; must be smaller than %d", NPY.MAXDIMS)
try:
- totalsize = support.product(shape) * isize
+ totalsize = ovfcheck(support.product_check(shape) * isize)
except OverflowError as e:
- raise oefmt(space.w_ValueError, "array is too big")
+ raise oefmt(space.w_ValueError, "array is too big.")
if storage_bytes > 0 :
if totalsize > storage_bytes:
raise OperationError(space.w_TypeError, space.wrap(
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,5 +1,6 @@
from pypy.interpreter.error import OperationError, oefmt
from rpython.rlib import jit, rgc
+from rpython.rlib.rarithmetic import ovfcheck
from rpython.rlib.buffer import Buffer
from rpython.rlib.debug import make_sure_not_resized
from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
@@ -409,6 +410,7 @@
make_sure_not_resized(strides)
make_sure_not_resized(backstrides)
self.shape = shape
+ # already tested for overflow in from_shape_and_storage
self.size = support.product(shape) * dtype.elsize
self.order = order
self.dtype = dtype
@@ -428,9 +430,9 @@
raise oefmt(space.w_ValueError,
"sequence too large; must be smaller than %d", NPY.MAXDIMS)
try:
- support.product(new_shape) * self.dtype.elsize
+ ovfcheck(support.product_check(new_shape) * self.dtype.elsize)
except OverflowError as e:
- raise oefmt(space.w_ValueError, "array is too big")
+ raise oefmt(space.w_ValueError, "array is too big.")
strides, backstrides = calc_strides(new_shape, self.dtype,
self.order)
return SliceArray(self.start, strides, backstrides, new_shape, self,
@@ -457,8 +459,11 @@
storage=lltype.nullptr(RAW_STORAGE), zero=True):
gcstruct = V_OBJECTSTORE
flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
- length = support.product(shape)
- self.size = length * dtype.elsize
+ try:
+ length = support.product_check(shape)
+ self.size = ovfcheck(length * dtype.elsize)
+ except OverflowError:
+ raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big.")
if storage == lltype.nullptr(RAW_STORAGE):
if dtype.num == NPY.OBJECT:
storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True)
@@ -542,7 +547,10 @@
self.gcstruct = parent.gcstruct
self.order = parent.order
self.dtype = dtype
- self.size = support.product(shape) * self.dtype.elsize
+ try:
+ self.size = ovfcheck(support.product_check(shape) * self.dtype.elsize)
+ except OverflowError:
+ raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big.")
self.start = start
self.orig_arr = orig_arr
flags = parent.flags & NPY.ARRAY_ALIGNED
@@ -564,9 +572,9 @@
raise oefmt(space.w_ValueError,
"sequence too large; must be smaller than %d", NPY.MAXDIMS)
try:
- support.product(new_shape) * self.dtype.elsize
+ ovfcheck(support.product_check(new_shape) * self.dtype.elsize)
except OverflowError as e:
- raise oefmt(space.w_ValueError, "array is too big")
+ raise oefmt(space.w_ValueError, "array is too big.")
if len(self.get_shape()) < 2 or self.size == 0:
# TODO: this code could be refactored into calc_strides
# but then calc_strides would have to accept a stepping factor
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -153,7 +153,7 @@
dtype = descriptor.variable_dtype(space, dtype.char + '1')
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
- if support.product(shape) == 1:
+ if support.product(shape) == 1: # safe from overflow since from_shape checks
w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
else:
loop.assign(space, w_arr, elems_w)
@@ -213,10 +213,9 @@
raise OperationError(space.w_ValueError, space.wrap(
"negative dimensions are not allowed"))
try:
- support.product(shape)
+ support.product_check(shape)
except OverflowError:
- raise OperationError(space.w_ValueError, space.wrap(
- "array is too big."))
+ raise oefmt(space.w_ValueError, "array is too big.")
return W_NDimArray.from_shape(space, shape, dtype=dtype, zero=zero)
def empty(space, w_shape, w_dtype=None, w_order=None):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -6,6 +6,7 @@
from rpython.rlib import jit
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rawstorage import RAW_STORAGE_PTR
+from rpython.rlib.rarithmetic import ovfcheck
from rpython.rtyper.lltypesystem import rffi
from rpython.tool.sourcetools import func_with_new_name
from pypy.module.micronumpy import descriptor, ufuncs, boxes, arrayops, loop, \
@@ -611,6 +612,7 @@
"__array__(dtype) not implemented"))
if type(self) is W_NDimArray:
return self
+ # sz cannot overflow since self is valid
sz = support.product(self.get_shape()) * self.get_dtype().elsize
return W_NDimArray.from_shape_and_storage(
space, self.get_shape(), self.implementation.storage,
@@ -1405,9 +1407,9 @@
return W_NDimArray.from_shape(space, shape, dtype, order)
strides, backstrides = calc_strides(shape, dtype.base, order)
try:
- totalsize = support.product(shape) * dtype.base.elsize
+ totalsize = ovfcheck(support.product_check(shape) * dtype.base.elsize)
except OverflowError as e:
- raise oefmt(space.w_ValueError, "array is too big")
+ raise oefmt(space.w_ValueError, "array is too big.")
impl = ConcreteArray(shape, dtype.base, order, strides, backstrides)
w_ret = space.allocate_instance(W_NDimArray, w_subtype)
W_NDimArray.__init__(w_ret, impl)
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -32,10 +32,16 @@
def product(s):
i = 1
for x in s:
+ i *= x
+ return i
+
+ at jit.unroll_safe
+def product_check(s):
+ i = 1
+ for x in s:
i = ovfcheck(i * x)
return i
-
def check_and_adjust_index(space, index, size, axis):
if index < -size or index >= size:
if axis >= 0:
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -270,7 +270,7 @@
exc = raises(ValueError, ndarray, [1,2,256]*10000)
assert exc.value[0] == 'sequence too large; must be smaller than 32'
exc = raises(ValueError, ndarray, [1,2,256]*10)
- assert exc.value[0] == 'array is too big'
+ assert exc.value[0] == 'array is too big.'
def test_ndmin(self):
from numpy import array
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1006,7 +1006,6 @@
assert isinstance(curarg, W_NDimArray)
if len(arg_shapes[i]) != curarg.ndims():
# reshape
-
sz = product(curarg.get_shape()) * curarg.get_dtype().elsize
with curarg.implementation as storage:
inargs[i] = W_NDimArray.from_shape_and_storage(
More information about the pypy-commit
mailing list