[pypy-commit] pypy default: more carefully guard access to storage by using a context manager
mattip
noreply at buildbot.pypy.org
Wed Mar 4 20:14:21 CET 2015
Author: mattip <matti.picus at gmail.com>
Branch:
Changeset: r76246:6cba1a6d7ad1
Date: 2015-03-04 21:15 +0200
http://bitbucket.org/pypy/pypy/changeset/6cba1a6d7ad1/
Log: more carefully guard access to storage by using a context manager
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -129,6 +129,9 @@
def get_order(self):
return self.implementation.order
+ def get_start(self):
+ return self.implementation.start
+
def ndims(self):
return len(self.get_shape())
ndims._always_inline_ = True
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -313,9 +313,6 @@
l_w = [w_res.descr_getitem(space, space.wrap(d)) for d in range(nd)]
return space.newtuple(l_w)
- def get_storage_as_int(self):
- return rffi.cast(lltype.Signed, self.storage) + self.start
-
##def get_storage(self):
## return self.storage
## use a safer context manager
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -100,10 +100,11 @@
elems_w[i] = w_object.implementation.getitem(i * elsize)
else:
imp = w_object.implementation
- sz = support.product(w_object.get_shape()) * dtype.elsize
- return W_NDimArray.from_shape_and_storage(space,
- w_object.get_shape(),imp.storage, dtype, storage_bytes=sz,
- w_base=w_object, start=imp.start)
+ with imp as storage:
+ sz = support.product(w_object.get_shape()) * dtype.elsize
+ return W_NDimArray.from_shape_and_storage(space,
+ w_object.get_shape(), storage, dtype, storage_bytes=sz,
+ w_base=w_object, start=imp.start)
else:
# not an array
shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -604,14 +604,15 @@
iter, state = arr.create_iter()
w_res_str = W_NDimArray.from_shape(space, [1], arr.get_dtype(), order='C')
itemsize = arr.get_dtype().elsize
- res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
- w_res_str.implementation.get_storage_as_int())
- while not iter.done(state):
- w_res_str.implementation.setitem(0, iter.getitem(state))
- for i in range(itemsize):
- builder.append(res_str_casted[i])
- state = iter.next(state)
- return builder.build()
+ with w_res_str.implementation as storage:
+ res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
+ support.get_storage_as_int(storage))
+ while not iter.done(state):
+ w_res_str.implementation.setitem(0, iter.getitem(state))
+ for i in range(itemsize):
+ builder.append(res_str_casted[i])
+ state = iter.next(state)
+ return builder.build()
getitem_int_driver = jit.JitDriver(name = 'numpy_getitem_int',
greens = ['shapelen', 'indexlen',
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -532,20 +532,25 @@
self.get_dtype(), storage_bytes=sz, w_base=self)
def descr_array_iface(self, space):
- addr = self.implementation.get_storage_as_int()
- # will explode if it can't
- w_d = space.newdict()
- space.setitem_str(w_d, 'data',
- space.newtuple([space.wrap(addr), space.w_False]))
- space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
- space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
- if self.implementation.order == 'C':
- # Array is contiguous, no strides in the interface.
- strides = space.w_None
- else:
- strides = self.descr_get_strides(space)
- space.setitem_str(w_d, 'strides', strides)
- return w_d
+ '''
+ Note: arr.__array__.data[0] is a pointer so arr must be kept alive
+ while it is in use
+ '''
+ with self.implementation as storage:
+ addr = support.get_storage_as_int(storage, self.get_start())
+ # will explode if it can't
+ w_d = space.newdict()
+ space.setitem_str(w_d, 'data',
+ space.newtuple([space.wrap(addr), space.w_False]))
+ space.setitem_str(w_d, 'shape', self.descr_get_shape(space))
+ space.setitem_str(w_d, 'typestr', self.get_dtype().descr_get_str(space))
+ if self.implementation.order == 'C':
+ # Array is contiguous, no strides in the interface.
+ strides = space.w_None
+ else:
+ strides = self.descr_get_strides(space)
+ space.setitem_str(w_d, 'strides', strides)
+ return w_d
w_pypy_data = None
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,6 +1,7 @@
from pypy.interpreter.error import OperationError, oefmt
from rpython.rlib import jit
from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rtyper.lltypesystem import rffi, lltype
def issequence_w(space, w_obj):
@@ -147,3 +148,7 @@
if cur_core_dim == 0:
ufunc.core_enabled = False
return 0 # for historical reasons, any failures will raise
+
+def get_storage_as_int(storage, start=0):
+ return rffi.cast(lltype.Signed, storage) + start
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -180,13 +180,16 @@
raw_storage_setitem_unaligned(storage, i + offset, value)
def read(self, arr, i, offset, dtype=None):
- return self.box(self._read(arr.storage, i, offset))
+ with arr as storage:
+ return self.box(self._read(storage, i, offset))
def read_bool(self, arr, i, offset):
- return bool(self.for_computation(self._read(arr.storage, i, offset)))
+ with arr as storage:
+ return bool(self.for_computation(self._read(storage, i, offset)))
def store(self, arr, i, offset, box):
- self._write(arr.storage, i, offset, self.unbox(box))
+ with arr as storage:
+ self._write(storage, i, offset, self.unbox(box))
def fill(self, storage, width, box, start, stop, offset):
value = self.unbox(box)
@@ -1080,8 +1083,9 @@
return bool(real) or bool(imag)
def read_bool(self, arr, i, offset):
- v = self.for_computation(self._read(arr.storage, i, offset))
- return bool(v[0]) or bool(v[1])
+ with arr as storage:
+ v = self.for_computation(self._read(storage, i, offset))
+ return bool(v[0]) or bool(v[1])
def get_element_size(self):
return 2 * rffi.sizeof(self.T)
@@ -1132,8 +1136,9 @@
return real, imag
def read(self, arr, i, offset, dtype=None):
- real, imag = self._read(arr.storage, i, offset)
- return self.box_complex(real, imag)
+ with arr as storage:
+ real, imag = self._read(storage, i, offset)
+ return self.box_complex(real, imag)
def _write(self, storage, i, offset, value):
real, imag = value
@@ -1144,7 +1149,8 @@
raw_storage_setitem_unaligned(storage, i + offset + rffi.sizeof(self.T), imag)
def store(self, arr, i, offset, box):
- self._write(arr.storage, i, offset, self.unbox(box))
+ with arr as storage:
+ self._write(storage, i, offset, self.unbox(box))
def fill(self, storage, width, box, start, stop, offset):
value = self.unbox(box)
@@ -1633,13 +1639,14 @@
assert isinstance(item, boxes.W_FlexibleBox)
i = item.ofs
end = i + item.dtype.elsize
- while i < end:
- assert isinstance(item.arr.storage[i], str)
- if item.arr.storage[i] == '\x00':
- break
- builder.append(item.arr.storage[i])
- i += 1
- return builder.build()
+ with item.arr as storage:
+ while i < end:
+ assert isinstance(storage[i], str)
+ if storage[i] == '\x00':
+ break
+ builder.append(storage[i])
+ i += 1
+ return builder.build()
def str_unary_op(func):
specialize.argtype(1)(func)
@@ -1669,23 +1676,26 @@
w_item = space.wrap('')
arg = space.str_w(space.str(w_item))
arr = VoidBoxStorage(dtype.elsize, dtype)
- j = min(len(arg), dtype.elsize)
- for i in range(j):
- arr.storage[i] = arg[i]
- for j in range(j, dtype.elsize):
- arr.storage[j] = '\x00'
- return boxes.W_StringBox(arr, 0, arr.dtype)
+ with arr as storage:
+ j = min(len(arg), dtype.elsize)
+ for i in range(j):
+ storage[i] = arg[i]
+ for j in range(j, dtype.elsize):
+ storage[j] = '\x00'
+ return boxes.W_StringBox(arr, 0, arr.dtype)
def store(self, arr, i, offset, box):
assert isinstance(box, boxes.W_StringBox)
size = min(arr.dtype.elsize - offset, box.arr.size - box.ofs)
- return self._store(arr.storage, i, offset, box, size)
+ with arr as storage:
+ return self._store(storage, i, offset, box, size)
@jit.unroll_safe
def _store(self, storage, i, offset, box, size):
assert isinstance(box, boxes.W_StringBox)
- for k in range(size):
- storage[k + offset + i] = box.arr.storage[k + box.ofs]
+ with box.arr as box_storage:
+ for k in range(size):
+ storage[k + offset + i] = box_storage[k + box.ofs]
def read(self, arr, i, offset, dtype=None):
if dtype is None:
@@ -1802,8 +1812,9 @@
assert i == 0
assert isinstance(box, boxes.W_VoidBox)
assert box.dtype is box.arr.dtype
- for k in range(box.arr.dtype.elsize):
- arr.storage[k + ofs] = box.arr.storage[k + box.ofs]
+ with arr as arr_storage, box.arr as box_storage:
+ for k in range(box.arr.dtype.elsize):
+ arr_storage[k + ofs] = box_storage[k + box.ofs]
def readarray(self, arr, i, offset, dtype=None):
from pypy.module.micronumpy.base import W_NDimArray
@@ -1893,12 +1904,14 @@
def store(self, arr, i, ofs, box):
assert isinstance(box, boxes.W_VoidBox)
- self._store(arr.storage, i, ofs, box, box.dtype.elsize)
+ with arr as storage:
+ self._store(storage, i, ofs, box, box.dtype.elsize)
@jit.unroll_safe
def _store(self, storage, i, ofs, box, size):
- for k in range(size):
- storage[k + i + ofs] = box.arr.storage[k + box.ofs]
+ with box.arr as box_storage:
+ for k in range(size):
+ storage[k + i + ofs] = box_storage[k + box.ofs]
def fill(self, storage, width, box, start, stop, offset):
assert isinstance(box, boxes.W_VoidBox)
@@ -1944,9 +1957,10 @@
s1 = v1.dtype.elsize
s2 = v2.dtype.elsize
assert s1 == s2
- for i in range(s1):
- if v1.arr.storage[v1.ofs + i] != v2.arr.storage[v2.ofs + i]:
- return False
+ with v1.arr as v1_storage, v2.arr as v2_storage:
+ for i in range(s1):
+ if v1_storage[v1.ofs + i] != v2_storage[v2.ofs + i]:
+ return False
return True
def ne(self, v1, v2):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -13,11 +13,12 @@
from pypy.module.micronumpy.ctors import numpify
from pypy.module.micronumpy.nditer import W_NDIter, coalesce_iter
from pypy.module.micronumpy.strides import shape_agreement
-from pypy.module.micronumpy.support import _parse_signature, product
+from pypy.module.micronumpy.support import _parse_signature, product, get_storage_as_int
from rpython.rlib.rawstorage import (raw_storage_setitem, free_raw_storage,
alloc_raw_storage)
from rpython.rtyper.lltypesystem import rffi, lltype
from rpython.rlib.rarithmetic import LONG_BIT, _get_bitsize
+from rpython.rlib.objectmodel import keepalive_until_here
def done_if_true(dtype, val):
@@ -98,7 +99,9 @@
if out is not None and not isinstance(out, W_NDimArray):
raise OperationError(space.w_TypeError, space.wrap(
'output must be an array'))
- return self.call(space, args_w, sig, casting, extobj)
+ retval = self.call(space, args_w, sig, casting, extobj)
+ keepalive_until_here(args_w)
+ return retval
def descr_accumulate(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None):
if space.is_none(w_axis):
@@ -804,11 +807,12 @@
assert isinstance(curarg, W_NDimArray)
if len(arg_shapes[i]) != curarg.ndims():
# reshape
+
sz = product(curarg.get_shape()) * curarg.get_dtype().elsize
- inargs[i] = W_NDimArray.from_shape_and_storage(
- space, arg_shapes[i], curarg.implementation.storage,
- curarg.get_dtype(), storage_bytes=sz, w_base=curarg)
- pass
+ with curarg.implementation as storage:
+ inargs[i] = W_NDimArray.from_shape_and_storage(
+ space, arg_shapes[i], storage,
+ curarg.get_dtype(), storage_bytes=sz, w_base=curarg)
need_to_cast.append(curarg.get_dtype() != dtypes[i])
for i in range(len(outargs)):
j = self.nin + i
@@ -819,9 +823,10 @@
elif len(arg_shapes[i]) != curarg.ndims():
# reshape
sz = product(curarg.get_shape()) * curarg.get_dtype().elsize
- outargs[i] = W_NDimArray.from_shape_and_storage(
- space, arg_shapes[i], curarg.implementation.storage,
- curarg.get_dtype(), storage_bytes=sz, w_base=curarg)
+ with curarg.implementation as storage:
+ outargs[i] = W_NDimArray.from_shape_and_storage(
+ space, arg_shapes[i], storage,
+ curarg.get_dtype(), storage_bytes=sz, w_base=curarg)
curarg = outargs[i]
assert isinstance(curarg, W_NDimArray)
need_to_cast.append(curarg.get_dtype() != dtypes[j])
@@ -1406,8 +1411,9 @@
raise OperationError(space.w_NotImplementedError,
space.wrap("cannot mix ndarray and %r (arg %d) in call to ufunc" % (
arg_i, i)))
- raw_storage_setitem(dataps, CCHARP_SIZE * i,
- rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int()))
+ with arg_i.implementation as storage:
+ addr = get_storage_as_int(storage, arg_i.get_start())
+ raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr))
#This assumes we iterate over the whole array (it should be a view...)
raw_storage_setitem(self.dims, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_size()))
raw_storage_setitem(self.steps, LONG_SIZE * i, rffi.cast(rffi.LONG, arg_i.get_dtype().elsize))
@@ -1415,8 +1421,9 @@
for i in range(len(args_w)):
arg_i = args_w[i]
assert isinstance(arg_i, W_NDimArray)
- raw_storage_setitem(dataps, CCHARP_SIZE * i,
- rffi.cast(rffi.CCHARP, arg_i.implementation.get_storage_as_int()))
+ with arg_i.implementation as storage:
+ addr = get_storage_as_int(storage, arg_i.get_start())
+ raw_storage_setitem(dataps, CCHARP_SIZE * i, rffi.cast(rffi.CCHARP, addr))
try:
arg1 = rffi.cast(rffi.CArrayPtr(rffi.CCHARP), dataps)
arg2 = rffi.cast(npy_intpp, self.dims)
@@ -1424,6 +1431,7 @@
self.func(arg1, arg2, arg3, self.data)
finally:
free_raw_storage(dataps, track_allocation=False)
+ keepalive_until_here(args_w)
def set_dims_and_steps(self, space, dims, steps):
if not isinstance(dims, list) or not isinstance(steps, list):
More information about the pypy-commit
mailing list