[pypy-commit] pypy default: merge buffer-interface, which implements buffer-protocol parts of cpyext and numpypy
mattip
pypy.commits at gmail.com
Tue Sep 6 06:59:16 EDT 2016
Author: Matti Picus <matti.picus at gmail.com>
Branch:
Changeset: r86906:03fc4cb79e37
Date: 2016-09-06 13:52 +0300
http://bitbucket.org/pypy/pypy/changeset/03fc4cb79e37/
Log: merge buffer-interface, which implements buffer-protocol parts of
cpyext and numpypy
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1428,6 +1428,9 @@
BUF_FORMAT = 0x0004
BUF_ND = 0x0008
BUF_STRIDES = 0x0010 | BUF_ND
+ BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+ BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+ BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
BUF_INDIRECT = 0x0100 | BUF_STRIDES
BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -597,6 +597,18 @@
def getlength(self):
return self.array.len * self.array.itemsize
+ def getformat(self):
+ return self.array.typecode
+
+ def getitemsize(self):
+ return self.array.itemsize
+
+ def getndim(self):
+ return 1
+
+ def getstrides(self):
+ return [self.getitemsize()]
+
def getitem(self, index):
array = self.array
data = array._charbuf_start()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -122,7 +122,7 @@
METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
""".split()
for name in constant_names:
setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,6 +645,9 @@
('format', rffi.CCHARP),
('shape', Py_ssize_tP),
('strides', Py_ssize_tP),
+ ('_format', rffi.UCHAR),
+ ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+ ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
('suboffsets', Py_ssize_tP),
#('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,8 +1,9 @@
from pypy.interpreter.error import oefmt
from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.rarithmetic import widen
from pypy.module.cpyext.api import (
- cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
-from pypy.module.cpyext.pyobject import PyObject
+ cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
@cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
def PyObject_CheckBuffer(space, pyobj):
@@ -33,13 +34,82 @@
raise an error if the object can't support a simpler view of its memory.
0 is returned on success and -1 on error."""
- raise oefmt(space.w_TypeError,
- "PyPy does not yet implement the new buffer interface")
+ flags = widen(flags)
+ buf = space.buffer_w(w_obj, flags)
+ try:
+ view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+ except ValueError:
+ raise BufferError("could not create buffer from object")
+ view.c_len = buf.getlength()
+ view.c_obj = make_ref(space, w_obj)
+ ndim = buf.getndim()
+ view.c_itemsize = buf.getitemsize()
+ rffi.setintfield(view, 'c_readonly', int(buf.readonly))
+ rffi.setintfield(view, 'c_ndim', ndim)
+ view.c_format = rffi.str2charp(buf.getformat())
+ view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+ view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
+ shape = buf.getshape()
+ strides = buf.getstrides()
+ for i in range(ndim):
+ view.c_shape[i] = shape[i]
+ view.c_strides[i] = strides[i]
+ view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+ view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+ return 0
+
+def _IsFortranContiguous(view):
+ ndim = widen(view.c_ndim)
+ if ndim == 0:
+ return 1
+ if not view.c_strides:
+ return ndim == 1
+ sd = view.c_itemsize
+ if ndim == 1:
+ return view.c_shape[0] == 1 or sd == view.c_strides[0]
+ for i in range(view.c_ndim):
+ dim = view.c_shape[i]
+ if dim == 0:
+ return 1
+ if view.c_strides[i] != sd:
+ return 0
+ sd *= dim
+ return 1
+
+def _IsCContiguous(view):
+ ndim = widen(view.c_ndim)
+ if ndim == 0:
+ return 1
+ if not view.c_strides:
+ return ndim == 1
+ sd = view.c_itemsize
+ if ndim == 1:
+ return view.c_shape[0] == 1 or sd == view.c_strides[0]
+ for i in range(ndim - 1, -1, -1):
+ dim = view.c_shape[i]
+ if dim == 0:
+ return 1
+ if view.c_strides[i] != sd:
+ return 0
+ sd *= dim
+ return 1
+
@cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
+def PyBuffer_IsContiguous(space, view, fort):
"""Return 1 if the memory defined by the view is C-style (fortran is
'C') or Fortran-style (fortran is 'F') contiguous or either one
(fortran is 'A'). Return 0 otherwise."""
- # PyPy only supports contiguous Py_buffers for now.
- return 1
+ # traverse the strides, checking for consistent stride increases from
+ # right-to-left (c) or left-to-right (fortran). Copied from cpython
+ if not view.c_suboffsets:
+ return 0
+ if (fort == 'C'):
+ return _IsCContiguous(view)
+ elif (fort == 'F'):
+ return _IsFortranContiguous(view)
+ elif (fort == 'A'):
+ return (_IsCContiguous(view) or _IsFortranContiguous(view))
+ return 0
+
+
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -29,19 +29,17 @@
## Solution
## --------
##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
+## PyBytesObject contains two additional members: the ob_size and an array
+## char ob_sval which holds a \x0 terminated string.
##
## - A string allocated by pypy will be converted into a PyBytesObject with a
-## NULL buffer. The first time PyString_AsString() is called, memory is
-## allocated (with flavor='raw') and content is copied.
+## buffer holding \x0. The first time PyString_AsString() is called, the
+## PyStringObject is reallocated, and the string copied into the buffer. The
+## ob_size reflects the length of the string.
##
## - A string allocated with PyString_FromStringAndSize(NULL, size) will
## allocate a PyBytesObject structure, and a buffer with the specified
-## size+1, but the reference won't be stored in the global map; there is no
-## corresponding object in pypy. When from_ref() or Py_INCREF() is called,
-## the pypy string is created, and added to the global map of tracked
-## objects. The buffer is then supposed to be immutable.
+## size+1, as part of the object. The buffer is then supposed to be immutable.
##
##- A buffer obtained from PyString_AS_STRING() could be mutable iff
## there is no corresponding pypy object for the string
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -142,7 +142,8 @@
typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
typedef struct bufferinfo {
void *buf;
PyObject *obj; /* owned reference */
@@ -156,12 +157,14 @@
char *format;
Py_ssize_t *shape;
Py_ssize_t *strides;
- Py_ssize_t *suboffsets;
-
+ Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+ unsigned char _format;
+ Py_ssize_t _strides[Py_MAX_NDIMS];
+ Py_ssize_t _shape[Py_MAX_NDIMS];
/* static store for shape and strides of
mono-dimensional buffers. */
/* Py_ssize_t smalltable[2]; */
- void *internal;
+ void *internal; /* always NULL for app-level objects */
} Py_buffer;
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,7 +1,8 @@
from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
- build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+ Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.objspace.std.memoryobject import W_MemoryView
PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
@@ -12,6 +13,7 @@
@cpython_api([PyObject], PyObject)
def PyMemoryView_GET_BASE(space, w_obj):
# return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+ # XXX needed for numpy on py3k
raise NotImplementedError('PyMemoryView_GET_BUFFER')
@cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -20,21 +22,35 @@
object. The object must be a memoryview instance; this macro doesn't
check its type, you must do it yourself or you will risk crashes."""
view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
- # TODO - fill in fields
- '''
- view.c_buf = buf
- view.c_len = length
- view.c_obj = obj
- Py_IncRef(space, obj)
- view.c_itemsize = 1
- rffi.setintfield(view, 'c_readonly', readonly)
- rffi.setintfield(view, 'c_ndim', 0)
- view.c_format = lltype.nullptr(rffi.CCHARP.TO)
- view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
- view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+ if not isinstance(w_obj, W_MemoryView):
+ return view
+ ndim = w_obj.buf.getndim()
+ if ndim >= Py_MAX_NDIMS:
+ # XXX warn?
+ return view
+ try:
+ view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+ view.c_obj = make_ref(space, w_obj)
+ rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+ isstr = False
+ except ValueError:
+ w_s = w_obj.descr_tobytes(space)
+ view.c_obj = make_ref(space, w_s)
+ rffi.setintfield(view, 'c_readonly', 1)
+ isstr = True
+ view.c_len = w_obj.getlength()
+ view.c_itemsize = w_obj.buf.getitemsize()
+ rffi.setintfield(view, 'c_ndim', ndim)
+ view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
+ view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+ view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+ view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+ shape = w_obj.buf.getshape()
+ strides = w_obj.buf.getstrides()
+ for i in range(ndim):
+ view.c_shape[i] = shape[i]
+ view.c_strides[i] = strides[i]
view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
- '''
return view
-
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -508,10 +508,9 @@
@cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
def PyBuffer_Release(space, view):
"""
- Releases a Py_buffer obtained from getbuffer ParseTuple's s*.
-
- This is not a complete re-implementation of the CPython API; it only
- provides a subset of CPython's behavior.
+ Release the buffer view. This should be called when the buffer is
+ no longer being used as it may free memory from it
"""
Py_DecRef(space, view.c_obj)
view.c_obj = lltype.nullptr(PyObject.TO)
+ # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
def getshape(self):
return self.shape
+ def getstrides(self):
+ return self.strides
+
def getitemsize(self):
return self.itemsize
+ def getndim(self):
+ return self.ndim
+
def wrap_getreadbuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
{
PyMyArray* self = (PyMyArray*)obj;
- fprintf(stdout, "in PyMyArray_getbuffer\n");
if (view == NULL) {
- fprintf(stdout, "view is NULL\n");
PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
return -1;
}
if (flags == 0) {
- fprintf(stdout, "flags is 0\n");
PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
return -1;
}
@@ -188,7 +185,131 @@
(initproc)PyMyArray_init, /* tp_init */
};
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+ Py_buffer* view = NULL;
+ PyObject* obj = PyTuple_GetItem(args, 0);
+ PyObject* memoryview = PyMemoryView_FromObject(obj);
+ if (memoryview == NULL)
+ return PyInt_FromLong(-1);
+ view = PyMemoryView_GET_BUFFER(memoryview);
+ Py_DECREF(memoryview);
+ return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG) \
+ buf_flag = PyUnicode_FromString(#FLAG); \
+ flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ); \
+ Py_DECREF(buf_flag); \
+ if (flag_matches == 1) { \
+ Py_DECREF(tmp); \
+ flags |= PyBUF_##FLAG; \
+ continue; \
+ } \
+ else if (flag_matches == -1) { \
+ Py_DECREF(tmp); \
+ return NULL; \
+ }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+ PyObject *buffer_obj, *pyflags;
+ PyObject *tmp, *buf_flag;
+ Py_buffer buffer;
+ PyObject *shape, *strides;
+ Py_ssize_t i, n;
+ int flag_matches;
+ int flags = 0;
+
+ if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+ return NULL;
+ }
+
+ n = PySequence_Length(pyflags);
+ if (n < 0) {
+ return NULL;
+ }
+
+ for (i=0; i < n; i++) {
+ tmp = PySequence_GetItem(pyflags, i);
+ if (tmp == NULL) {
+ return NULL;
+ }
+
+ GET_PYBUF_FLAG(SIMPLE);
+ GET_PYBUF_FLAG(WRITABLE);
+ GET_PYBUF_FLAG(STRIDES);
+ GET_PYBUF_FLAG(ND);
+ GET_PYBUF_FLAG(C_CONTIGUOUS);
+ GET_PYBUF_FLAG(F_CONTIGUOUS);
+ GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+ GET_PYBUF_FLAG(INDIRECT);
+ GET_PYBUF_FLAG(FORMAT);
+ GET_PYBUF_FLAG(STRIDED);
+ GET_PYBUF_FLAG(STRIDED_RO);
+ GET_PYBUF_FLAG(RECORDS);
+ GET_PYBUF_FLAG(RECORDS_RO);
+ GET_PYBUF_FLAG(FULL);
+ GET_PYBUF_FLAG(FULL_RO);
+ GET_PYBUF_FLAG(CONTIG);
+ GET_PYBUF_FLAG(CONTIG_RO);
+
+ Py_DECREF(tmp);
+
+ /* One of the flags must match */
+ PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+ return NULL;
+ }
+
+ if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+ return NULL;
+ }
+
+ if (buffer.shape == NULL) {
+ Py_INCREF(Py_None);
+ shape = Py_None;
+ }
+ else {
+ shape = PyTuple_New(buffer.ndim);
+ for (i=0; i < buffer.ndim; i++) {
+ PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+ }
+ }
+
+ if (buffer.strides == NULL) {
+ Py_INCREF(Py_None);
+ strides = Py_None;
+ }
+ else {
+ strides = PyTuple_New(buffer.ndim);
+ for (i=0; i < buffer.ndim; i++) {
+ PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+ }
+ }
+
+ PyBuffer_Release(&buffer);
+ return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
static PyMethodDef buffer_functions[] = {
+ {"test_buffer", (PyCFunction)test_buffer, METH_VARARGS, NULL},
+ {"get_buffer_info", (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
{NULL, NULL} /* Sentinel */
};
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,6 @@
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
class TestMemoryViewObject(BaseApiTest):
def test_fromobject(self, space, api):
@@ -12,6 +12,12 @@
w_bytes = space.call_method(w_view, "tobytes")
assert space.unwrap(w_bytes) == "hello"
+ def test_frombuffer(self, space, api):
+ w_buf = space.newbuffer(StringBuffer("hello"))
+ w_memoryview = api.PyMemoryView_FromObject(w_buf)
+ w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+ ndim = w_view.c_ndim
+ assert ndim == 1
class AppTestBufferProtocol(AppTestCpythonExtensionBase):
def test_buffer_protocol(self):
@@ -21,6 +27,25 @@
y = memoryview(arr)
assert y.format == 'i'
assert y.shape == (10,)
+ assert len(y) == 10
s = y[3]
assert len(s) == struct.calcsize('i')
assert s == struct.pack('i', 3)
+ viewlen = module.test_buffer(arr)
+ assert viewlen == y.itemsize * len(y)
+
+ def test_buffer_info(self):
+ from _numpypy import multiarray as np
+ module = self.import_module(name='buffer_test')
+ get_buffer_info = module.get_buffer_info
+ # test_export_flags from numpy test_multiarray
+ raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+ # test_relaxed_strides from numpy test_multiarray
+ arr = np.zeros((1, 10))
+ if arr.flags.f_contiguous:
+ shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+ assert strides[0] == 8
+ arr = np.ones((10, 1), order='F')
+ shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+ assert strides[-1] == 8
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
STRUCT_TYPE = PyNumberMethods
elif slot_names[0] == 'c_tp_as_sequence':
STRUCT_TYPE = PySequenceMethods
+ elif slot_names[0] == 'c_tp_as_buffer':
+ STRUCT_TYPE = PyBufferProcs
else:
raise AssertionError(
"Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
def getdictvalue(self, space, key):
return self.items[key]
+ def descr_memoryview(self, space, buf):
+ raise oefmt(space.w_TypeError, "error")
+
class IterDictObject(W_Root):
def __init__(self, space, w_dict):
self.space = space
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -377,7 +377,25 @@
def __exit__(self, typ, value, traceback):
keepalive_until_here(self)
- def get_buffer(self, space, readonly):
+ def get_buffer(self, space, flags):
+ errtype = space.w_ValueError # should be BufferError, numpy does this instead
+ if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and
+ not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+ raise oefmt(errtype, "ndarray is not C-contiguous")
+ if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and
+ not self.flags & NPY.ARRAY_F_CONTIGUOUS):
+ raise oefmt(errtype, "ndarray is not Fortran contiguous")
+ if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
+ not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
+ self.flags & NPY.ARRAY_C_CONTIGUOUS)):
+ raise oefmt(errtype, "ndarray is not contiguous")
+ if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
+ not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+ raise oefmt(errtype, "ndarray is not C-contiguous")
+ if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and
+ not self.flags & NPY.ARRAY_WRITEABLE):
+ raise oefmt(errtype, "buffer source array is read-only")
+ readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
return ArrayBuffer(self, readonly)
def astype(self, space, dtype, order, copy=True):
@@ -695,6 +713,7 @@
index + self.impl.start)
def setitem(self, index, v):
+ # XXX what if self.readonly?
raw_storage_setitem(self.impl.storage, index + self.impl.start,
rffi.cast(lltype.Char, v))
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,4 +1,5 @@
from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
from rpython.rlib.buffer import SubBuffer
from rpython.rlib.rstring import strip_spaces
@@ -42,7 +43,7 @@
raise oefmt(space.w_ValueError,
"object __array__ method not producing an array")
-def try_interface_method(space, w_object):
+def try_interface_method(space, w_object, copy):
try:
w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
if w_interface is None:
@@ -81,17 +82,20 @@
raise oefmt(space.w_ValueError,
"__array_interface__ could not decode dtype %R", w_dtype
)
- if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
+ if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
+ space.isinstance_w(w_data, space.w_list)):
data_w = space.listview(w_data)
- data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
- read_only = True # XXX why not space.is_true(data_w[1])
+ w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+ read_only = space.is_true(data_w[1]) or copy
offset = 0
- return W_NDimArray.from_shape_and_storage(space, shape, data,
- dtype, strides=strides, start=offset), read_only
+ w_base = w_object
+ if read_only:
+ w_base = None
+ return W_NDimArray.from_shape_and_storage(space, shape, w_data,
+ dtype, w_base=w_base, strides=strides,
+ start=offset), read_only
if w_data is None:
- data = w_object
- else:
- data = w_data
+ w_data = w_object
w_offset = space.finditem(w_interface, space.wrap('offset'))
if w_offset is None:
offset = 0
@@ -101,7 +105,7 @@
if strides is not None:
raise oefmt(space.w_NotImplementedError,
"__array_interface__ strides not fully supported yet")
- arr = frombuffer(space, data, dtype, support.product(shape), offset)
+ arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
new_impl = arr.implementation.reshape(arr, shape)
return W_NDimArray(new_impl), False
@@ -110,6 +114,78 @@
return None, False
raise
+def _descriptor_from_pep3118_format(space, c_format):
+ descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+ if descr:
+ return descr
+ msg = "invalid PEP 3118 format string: '%s'" % c_format
+ space.warn(space.wrap(msg), space.w_RuntimeWarning)
+ return None
+
+def _array_from_buffer_3118(space, w_object, dtype):
+ try:
+ w_buf = space.call_method(space.builtin, "memoryview", w_object)
+ except OperationError as e:
+ if e.match(space, space.w_TypeError):
+ # object does not have buffer interface
+ return w_object
+ raise
+ format = space.getattr(w_buf,space.newbytes('format'))
+ if format:
+ descr = _descriptor_from_pep3118_format(space, space.str_w(format))
+ if not descr:
+ return w_object
+ if dtype and descr:
+ raise oefmt(space.w_NotImplementedError,
+ "creating an array from a memoryview while specifying dtype "
+ "not supported")
+ if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))):
+ msg = ("Item size computed from the PEP 3118 buffer format "
+ "string does not match the actual item size.")
+ space.warn(space.wrap(msg), space.w_RuntimeWarning)
+ return w_object
+ dtype = descr
+ elif not dtype:
+ dtype = descriptor.get_dtype_cache(space).w_stringdtype
+ dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
+ nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim')))
+ shape = [space.int_w(d) for d in space.listview(
+ space.getattr(w_buf, space.newbytes('shape')))]
+ strides = []
+ buflen = space.len_w(w_buf) * dtype.elsize
+ if shape:
+ strides = [space.int_w(d) for d in space.listview(
+ space.getattr(w_buf, space.newbytes('strides')))]
+ if not strides:
+ d = buflen
+ strides = [0] * nd
+ for k in range(nd):
+ if shape[k] > 0:
+ d /= shape[k]
+ strides[k] = d
+ else:
+ if nd == 1:
+ shape = [buflen / dtype.elsize, ]
+ strides = [dtype.elsize, ]
+ elif nd > 1:
+ msg = ("ndim computed from the PEP 3118 buffer format "
+ "is greater than 1, but shape is NULL.")
+ space.warn(space.wrap(msg), space.w_RuntimeWarning)
+ return w_object
+ try:
+ w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
+ except OperationError as e:
+ if e.match(space, space.w_ValueError):
+ return w_object
+ else:
+ raise e
+ writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
+ w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
+ storage_bytes=buflen, dtype=dtype, w_base=w_object,
+ writable=writable, strides=strides)
+ if w_ret:
+ return w_ret
+ return w_object
@unwrap_spec(ndmin=int, copy=bool, subok=bool)
def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -127,6 +203,7 @@
def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
+ from pypy.module.micronumpy.boxes import W_GenericBox
# numpy testing calls array(type(array([]))) and expects a ValueError
if space.isinstance_w(w_object, space.w_type):
raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
@@ -134,13 +211,19 @@
dtype = descriptor.decode_w_dtype(space, w_dtype)
if not isinstance(w_object, W_NDimArray):
w_array = try_array_method(space, w_object, w_dtype)
- if w_array is not None:
+ if w_array is None:
+ if ( not space.isinstance_w(w_object, space.w_str) and
+ not space.isinstance_w(w_object, space.w_unicode) and
+ not isinstance(w_object, W_GenericBox)):
+ # use buffer interface
+ w_object = _array_from_buffer_3118(space, w_object, dtype)
+ else:
# continue with w_array, but do further operations in place
w_object = w_array
copy = False
dtype = w_object.get_dtype()
if not isinstance(w_object, W_NDimArray):
- w_array, _copy = try_interface_method(space, w_object)
+ w_array, _copy = try_interface_method(space, w_object, copy)
if w_array is not None:
w_object = w_array
copy = _copy
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -805,19 +805,19 @@
return w_result
def buffer_w(self, space, flags):
- return self.implementation.get_buffer(space, True)
+ return self.implementation.get_buffer(space, flags)
def readbuf_w(self, space):
- return self.implementation.get_buffer(space, True)
+ return self.implementation.get_buffer(space, space.BUF_FULL_RO)
def writebuf_w(self, space):
- return self.implementation.get_buffer(space, False)
+ return self.implementation.get_buffer(space, space.BUF_FULL)
def charbuf_w(self, space):
- return self.implementation.get_buffer(space, True).as_str()
+ return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
def descr_get_data(self, space):
- return space.newbuffer(self.implementation.get_buffer(space, False))
+ return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
@unwrap_spec(offset=int, axis1=int, axis2=int)
def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3215,7 +3215,9 @@
raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)}))
a = array([1, 2, 3])
- b = array(Dummy(a.__array_interface__))
+ d = Dummy(a.__array_interface__)
+ b = array(d)
+ assert b.base is None
b[1] = 200
assert a[1] == 2 # upstream compatibility, is this a bug?
interface_a = a.__array_interface__
@@ -3226,6 +3228,8 @@
interface_b.pop('data')
interface_a.pop('data')
assert interface_a == interface_b
+ b = array(d, copy=False)
+ assert b.base is d
b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
'data': 'a'*100}))
@@ -3594,6 +3598,7 @@
cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+ cls.w_one = cls.space.wrap(struct.pack('i', 1))
def test_frombuffer(self):
import numpy as np
@@ -3645,8 +3650,6 @@
else:
EMPTY = None
x = np.array([1, 2, 3, 4, 5], dtype='i')
- y = memoryview('abc')
- assert y.format == 'B'
y = memoryview(x)
assert y.format == 'i'
assert y.shape == (5,)
@@ -3654,6 +3657,16 @@
assert y.strides == (4,)
assert y.suboffsets == EMPTY
assert y.itemsize == 4
+ assert isinstance(y, memoryview)
+ assert y[0] == self.one
+ assert (np.array(y) == x).all()
+
+ x = np.array([0, 0, 0, 0], dtype='O')
+ y = memoryview(x)
+ # handles conversion of address to pinned object?
+ z = np.array(y)
+ assert z.dtype == 'O'
+ assert (z == x).all()
def test_fromstring(self):
import sys
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -702,3 +702,32 @@
ret = obj.sum()
print type(ret)
assert ret.info == 'spam'
+
+ def test_ndarray_subclass_assigns_base(self):
+ import numpy as np
+ init_called = []
+ class _DummyArray(object):
+ """ Dummy object that just exists to hang __array_interface__ dictionaries
+ and possibly keep alive a reference to a base array.
+ """
+ def __init__(self, interface, base=None):
+ self.__array_interface__ = interface
+ init_called.append(1)
+ self.base = base
+
+ x = np.zeros(10)
+ d = _DummyArray(x.__array_interface__, base=x)
+ y = np.array(d, copy=False)
+ assert sum(init_called) == 1
+ assert y.base is d
+
+ x = np.zeros((0,), dtype='float32')
+ intf = x.__array_interface__.copy()
+ intf["strides"] = x.strides
+ x.__array_interface__["strides"] = x.strides
+ d = _DummyArray(x.__array_interface__, base=x)
+ y = np.array(d, copy=False)
+ assert sum(init_called) == 2
+ assert y.base is d
+
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1851,7 +1851,7 @@
arr.gcstruct)
def read(self, arr, i, offset, dtype):
- if arr.gcstruct is V_OBJECTSTORE:
+ if arr.gcstruct is V_OBJECTSTORE and not arr.base():
raise oefmt(self.space.w_NotImplementedError,
"cannot read object from array with no gc hook")
return self.box(self._read(arr.storage, i, offset))
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -14,6 +14,7 @@
"""Implement the built-in 'memoryview' type as a wrapper around
an interp-level buffer.
"""
+ _attrs_ = ['buf']
def __init__(self, buf):
assert isinstance(buf, Buffer)
@@ -115,7 +116,7 @@
self.buf.setslice(start, value.as_str())
def descr_len(self, space):
- return space.wrap(self.buf.getlength())
+ return space.wrap(self.buf.getlength() / self.buf.getitemsize())
def w_get_format(self, space):
return space.wrap(self.buf.getformat())
More information about the pypy-commit
mailing list