[pypy-commit] pypy force-virtual-state: Merge with default
sbauman
pypy.commits at gmail.com
Tue Sep 13 11:38:54 EDT 2016
Author: Spenser Andrew Bauman <sabauma at gmail.com>
Branch: force-virtual-state
Changeset: r87079:5d49fd764471
Date: 2016-09-13 11:38 -0400
http://bitbucket.org/pypy/pypy/changeset/5d49fd764471/
Log: Merge with default
diff too long, truncating to 2000 out of 3169 lines
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -31,3 +31,5 @@
68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0
77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0
050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
+0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.8.1
+Version: 1.8.2
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "1.8.1"
-__version_info__ = (1, 8, 1)
+__version__ = "1.8.2"
+__version_info__ = (1, 8, 2)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
f = PySys_GetObject((char *)"stderr");
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.8.1"
+ "\ncompiled with cffi version: 1.8.2"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst
--- a/pypy/doc/release-pypy2.7-v5.4.1.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.1.rst
@@ -5,26 +5,29 @@
We have released a bugfix for PyPy2.7-v5.4.0, released last week,
due to the following issues:
- * Update list of contributors in documentation and LICENSE file,
- this was unfortunately left out of 5.4.0. My apoligies to the new
+ * Update list of contributors in documentation and LICENSE file,
+ this was unfortunately left out of 5.4.0. My apologies to the new
contributors
- * Allow tests run with `-A` to find `libm.so` even if it is a script not a
+ * Allow tests run with ``-A`` to find ``libm.so`` even if it is a script not a
dynamically loadable file
- * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython
+ * Bump ``sys.setrecursionlimit()`` when translating PyPy, for translating with CPython
- * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors
+ * Tweak a float comparison with 0 in ``backendopt.inline`` to avoid rounding errors
- * Fix for an issue where os.access() accepted a float for mode
+ * Fix for an issue for translating the sandbox
- * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up
+ * Fix for and issue where ``unicode.decode('utf8', 'custom_replace')`` messed up
the last byte of a unicode string sometimes
- * Update built-in cffi_ to the soon-to-be-released 1.8.1 version
+ * Update built-in cffi_ to version 1.8.1
* Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash
- translation with a message asking for help porting it
+ translation with a message asking for help porting it
+
+ * Fix a regression where a PyBytesObject was forced (converted to a RPython
+ object) when not required, reported as issue #2395
Thanks to those who reported the issues.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -7,3 +7,12 @@
.. branch: rpython-resync
Backport rpython changes made directly on the py3k and py3.5 branches.
+
+.. branch: buffer-interface
+Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
+in numpypy
+
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1428,6 +1428,9 @@
BUF_FORMAT = 0x0004
BUF_ND = 0x0008
BUF_STRIDES = 0x0010 | BUF_ND
+ BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES
+ BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES
+ BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES
BUF_INDIRECT = 0x0100 | BUF_STRIDES
BUF_CONTIG_RO = BUF_ND
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi, entrypoint
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.8.1"
+VERSION = "1.8.2"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -105,9 +105,6 @@
return True
return False
- def _check_only_one_argument_for_union(self, w_ob):
- pass
-
def convert_from_object(self, cdata, w_ob):
if not self._copy_from_same(cdata, w_ob):
self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@@ -117,19 +114,24 @@
)
def convert_struct_from_object(self, cdata, w_ob, optvarsize):
self.force_lazy_struct()
- self._check_only_one_argument_for_union(w_ob)
space = self.space
if (space.isinstance_w(w_ob, space.w_list) or
space.isinstance_w(w_ob, space.w_tuple)):
lst_w = space.listview(w_ob)
- if len(lst_w) > len(self._fields_list):
- raise oefmt(space.w_ValueError,
- "too many initializers for '%s' (got %d)",
- self.name, len(lst_w))
- for i in range(len(lst_w)):
- optvarsize = self._fields_list[i].write_v(cdata, lst_w[i],
+ j = 0
+ for w_obj in lst_w:
+ try:
+ while (self._fields_list[j].flags &
+ W_CField.BF_IGNORE_IN_CTOR):
+ j += 1
+ except IndexError:
+ raise oefmt(space.w_ValueError,
+ "too many initializers for '%s' (got %d)",
+ self.name, len(lst_w))
+ optvarsize = self._fields_list[j].write_v(cdata, w_obj,
optvarsize)
+ j += 1
return optvarsize
elif space.isinstance_w(w_ob, space.w_dict):
@@ -185,14 +187,6 @@
class W_CTypeUnion(W_CTypeStructOrUnion):
kind = "union"
- def _check_only_one_argument_for_union(self, w_ob):
- space = self.space
- n = space.int_w(space.len(w_ob))
- if n > 1:
- raise oefmt(space.w_ValueError,
- "initializer for '%s': %d items given, but only one "
- "supported (use a dict if needed)", self.name, n)
-
class W_CField(W_Root):
_immutable_ = True
@@ -200,18 +194,21 @@
BS_REGULAR = -1
BS_EMPTY_ARRAY = -2
- def __init__(self, ctype, offset, bitshift, bitsize):
+ BF_IGNORE_IN_CTOR = 0x01
+
+ def __init__(self, ctype, offset, bitshift, bitsize, flags):
self.ctype = ctype
self.offset = offset
self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY
self.bitsize = bitsize
+ self.flags = flags # BF_xxx
def is_bitfield(self):
return self.bitshift >= 0
- def make_shifted(self, offset):
+ def make_shifted(self, offset, fflags):
return W_CField(self.ctype, offset + self.offset,
- self.bitshift, self.bitsize)
+ self.bitshift, self.bitsize, self.flags | fflags)
def read(self, cdata):
cdata = rffi.ptradd(cdata, self.offset)
@@ -341,5 +338,6 @@
offset = interp_attrproperty('offset', W_CField),
bitshift = interp_attrproperty('bitshift', W_CField),
bitsize = interp_attrproperty('bitsize', W_CField),
+ flags = interp_attrproperty('flags', W_CField),
)
W_CField.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -345,6 +345,11 @@
if alignment < falign and do_align:
alignment = falign
#
+ if is_union and i > 0:
+ fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR
+ else:
+ fflags = 0
+ #
if fbitsize < 0:
# not a bitfield: common case
@@ -372,7 +377,7 @@
for name, srcfld in ftype._fields_dict.items():
srcfield2names[srcfld] = name
for srcfld in ftype._fields_list:
- fld = srcfld.make_shifted(boffset // 8)
+ fld = srcfld.make_shifted(boffset // 8, fflags)
fields_list.append(fld)
try:
fields_dict[srcfield2names[srcfld]] = fld
@@ -382,7 +387,8 @@
w_ctype._custom_field_pos = True
else:
# a regular field
- fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1)
+ fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1,
+ fflags)
fields_list.append(fld)
fields_dict[fname] = fld
@@ -489,7 +495,7 @@
bitshift = 8 * ftype.size - fbitsize- bitshift
fld = ctypestruct.W_CField(ftype, field_offset_bytes,
- bitshift, fbitsize)
+ bitshift, fbitsize, fflags)
fields_list.append(fld)
fields_dict[fname] = fld
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.2", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
@@ -2525,6 +2525,25 @@
assert d[2][1].bitshift == -1
assert d[2][1].bitsize == -1
+def test_nested_anonymous_struct_2():
+ BInt = new_primitive_type("int")
+ BStruct = new_struct_type("struct foo")
+ BInnerUnion = new_union_type("union bar")
+ complete_struct_or_union(BInnerUnion, [('a1', BInt, -1),
+ ('a2', BInt, -1)])
+ complete_struct_or_union(BStruct, [('b1', BInt, -1),
+ ('', BInnerUnion, -1),
+ ('b2', BInt, -1)])
+ assert sizeof(BInnerUnion) == sizeof(BInt)
+ assert sizeof(BStruct) == sizeof(BInt) * 3
+ fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields]
+ assert fields == [
+ ('b1', 0 * sizeof(BInt), 0),
+ ('a1', 1 * sizeof(BInt), 0),
+ ('a2', 1 * sizeof(BInt), 1),
+ ('b2', 2 * sizeof(BInt), 0),
+ ]
+
def test_sizeof_union():
# a union has the largest alignment of its members, and a total size
# that is the largest of its items *possibly further aligned* if
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -450,7 +450,12 @@
# For compatibility
assert exc.value.errno == _ssl.SSL_ERROR_WANT_READ
finally:
- c.shutdown()
+ try:
+ c.shutdown()
+ except _ssl.SSLError:
+ # If the expected exception was raised, the SSLContext
+ # can't be shut down yet
+ pass
finally:
s.close()
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -597,6 +597,18 @@
def getlength(self):
return self.array.len * self.array.itemsize
+ def getformat(self):
+ return self.array.typecode
+
+ def getitemsize(self):
+ return self.array.itemsize
+
+ def getndim(self):
+ return 1
+
+ def getstrides(self):
+ return [self.getitemsize()]
+
def getitem(self, index):
array = self.array
data = array._charbuf_start()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,10 +119,10 @@
constant_names = """
Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
-Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES
+Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
""".split()
for name in constant_names:
setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
@@ -645,6 +645,9 @@
('format', rffi.CCHARP),
('shape', Py_ssize_tP),
('strides', Py_ssize_tP),
+ ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
+ ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
+ ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
('suboffsets', Py_ssize_tP),
#('smalltable', rffi.CFixedArray(Py_ssize_t, 2)),
('internal', rffi.VOIDP)
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,7 +1,6 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
from pypy.module.cpyext.api import (
- cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER)
+ cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
from pypy.module.cpyext.pyobject import PyObject
@cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
@@ -13,33 +12,4 @@
return 1
return 0
- at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
- rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
- """Export obj into a Py_buffer, view. These arguments must
- never be NULL. The flags argument is a bit field indicating what
- kind of buffer the caller is prepared to deal with and therefore what
- kind of buffer the exporter is allowed to return. The buffer interface
- allows for complicated memory sharing possibilities, but some caller may
- not be able to handle all the complexity but may want to see if the
- exporter will let them take a simpler view to its memory.
-
- Some exporters may not be able to share memory in every possible way and
- may need to raise errors to signal to some consumers that something is
- just not possible. These errors should be a BufferError unless
- there is another error that is actually causing the problem. The
- exporter can use flags information to simplify how much of the
- Py_buffer structure is filled in with non-default values and/or
- raise an error if the object can't support a simpler view of its memory.
-
- 0 is returned on success and -1 on error."""
- raise oefmt(space.w_TypeError,
- "PyPy does not yet implement the new buffer interface")
-
- at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fortran):
- """Return 1 if the memory defined by the view is C-style (fortran is
- 'C') or Fortran-style (fortran is 'F') contiguous or either one
- (fortran is 'A'). Return 0 otherwise."""
- # PyPy only supports contiguous Py_buffers for now.
- return 1
+
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -14,45 +14,33 @@
## Implementation of PyBytesObject
## ================================
##
-## The problem
-## -----------
+## PyBytesObject has its own ob_sval buffer, so we have two copies of a string;
+## one in the PyBytesObject returned from various C-API functions and another
+## in the corresponding RPython object.
##
-## PyString_AsString() must return a (non-movable) pointer to the underlying
-## ob_sval, whereas pypy strings are movable. C code may temporarily store
-## this address and use it, as long as it owns a reference to the PyObject.
-## There is no "release" function to specify that the pointer is not needed
-## any more.
+## The following calls can create a PyBytesObject without a correspoinding
+## RPython object:
##
-## Also, the pointer may be used to fill the initial value of string. This is
-## valid only when the string was just allocated, and is not used elsewhere.
+## PyBytes_FromStringAndSize(NULL, n) / PyString_FromStringAndSize(NULL, n)
##
-## Solution
-## --------
+## In the PyBytesObject returned, the ob_sval buffer may be modified as
+## long as the freshly allocated PyBytesObject is not "forced" via a call
+## to any of the more sophisticated C-API functions.
##
-## PyBytesObject contains two additional members: the ob_size and a pointer to a
-## char ob_sval; it may be NULL.
-##
-## - A string allocated by pypy will be converted into a PyBytesObject with a
-## NULL buffer. The first time PyString_AsString() is called, memory is
-## allocated (with flavor='raw') and content is copied.
-##
-## - A string allocated with PyString_FromStringAndSize(NULL, size) will
-## allocate a PyBytesObject structure, and a buffer with the specified
-## size+1, but the reference won't be stored in the global map; there is no
-## corresponding object in pypy. When from_ref() or Py_INCREF() is called,
-## the pypy string is created, and added to the global map of tracked
-## objects. The buffer is then supposed to be immutable.
-##
-##- A buffer obtained from PyString_AS_STRING() could be mutable iff
-## there is no corresponding pypy object for the string
-##
-## - _PyString_Resize() works only on not-yet-pypy'd strings, and returns a
-## similar object.
-##
-## - PyString_Size() doesn't need to force the object.
+## Care has been taken in implementing the functions below, so that
+## if they are called with a non-forced PyBytesObject, they will not
+## unintentionally force the creation of a RPython object. As long as only these
+## are used, the ob_sval buffer is still modifiable:
+##
+## PyBytes_AsString / PyString_AsString
+## PyBytes_AS_STRING / PyString_AS_STRING
+## PyBytes_AsStringAndSize / PyString_AsStringAndSize
+## PyBytes_Size / PyString_Size
+## PyBytes_Resize / PyString_Resize
+## _PyBytes_Resize / _PyString_Resize (raises if called with a forced object)
##
## - There could be an (expensive!) check in from_ref() that the buffer still
-## corresponds to the pypy gc-managed string.
+## corresponds to the pypy gc-managed string,
##
PyBytesObjectStruct = lltype.ForwardReference()
@@ -156,9 +144,6 @@
"expected string or Unicode object, %T found",
from_ref(space, ref))
ref_str = rffi.cast(PyBytesObject, ref)
- if not pyobj_has_w_obj(ref):
- # XXX Force the ref?
- bytes_realize(space, ref)
return ref_str.c_ob_sval
@cpython_api([rffi.VOIDP], rffi.CCHARP, error=0)
@@ -182,9 +167,6 @@
raise oefmt(space.w_TypeError,
"expected string or Unicode object, %T found",
from_ref(space, ref))
- if not pyobj_has_w_obj(ref):
- # force the ref
- bytes_realize(space, ref)
ref_str = rffi.cast(PyBytesObject, ref)
data[0] = ref_str.c_ob_sval
if length:
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -142,7 +142,9 @@
typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
-/* Py3k buffer interface */
+/* Py3k buffer interface, adapted for PyPy */
+#define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
typedef struct bufferinfo {
void *buf;
PyObject *obj; /* owned reference */
@@ -156,12 +158,14 @@
char *format;
Py_ssize_t *shape;
Py_ssize_t *strides;
- Py_ssize_t *suboffsets;
-
+ Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
+ unsigned char _format[Py_MAX_FMT];
+ Py_ssize_t _strides[Py_MAX_NDIMS];
+ Py_ssize_t _shape[Py_MAX_NDIMS];
/* static store for shape and strides of
mono-dimensional buffers. */
/* Py_ssize_t smalltable[2]; */
- void *internal;
+ void *internal; /* always NULL for app-level objects */
} Py_buffer;
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,10 +1,122 @@
from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
- build_type_checkers)
-from pypy.module.cpyext.pyobject import PyObject
-from rpython.rtyper.lltypesystem import lltype
+ Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
+from pypy.objspace.std.memoryobject import W_MemoryView
PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview")
+ at cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+ rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+ """Export obj into a Py_buffer, view. These arguments must
+ never be NULL. The flags argument is a bit field indicating what
+ kind of buffer the caller is prepared to deal with and therefore what
+ kind of buffer the exporter is allowed to return. The buffer interface
+ allows for complicated memory sharing possibilities, but some caller may
+ not be able to handle all the complexity but may want to see if the
+ exporter will let them take a simpler view to its memory.
+
+ Some exporters may not be able to share memory in every possible way and
+ may need to raise errors to signal to some consumers that something is
+ just not possible. These errors should be a BufferError unless
+ there is another error that is actually causing the problem. The
+ exporter can use flags information to simplify how much of the
+ Py_buffer structure is filled in with non-default values and/or
+ raise an error if the object can't support a simpler view of its memory.
+
+ 0 is returned on success and -1 on error."""
+ flags = widen(flags)
+ buf = space.buffer_w(w_obj, flags)
+ try:
+ view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+ except ValueError:
+ raise BufferError("could not create buffer from object")
+ view.c_obj = make_ref(space, w_obj)
+ return fill_Py_buffer(space, buf, view)
+
+def fill_Py_buffer(space, buf, view):
+ # c_buf, c_obj have been filled in
+ ndim = buf.getndim()
+ view.c_len = buf.getlength()
+ view.c_itemsize = buf.getitemsize()
+ rffi.setintfield(view, 'c_ndim', ndim)
+ view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+ view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+ view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+ fmt = buf.getformat()
+ n = Py_MAX_FMT - 1 # NULL terminated buffer
+ if len(fmt) > n:
+ ### WARN?
+ pass
+ else:
+ n = len(fmt)
+ for i in range(n):
+ view.c_format[i] = fmt[i]
+ view.c_format[n] = '\x00'
+ shape = buf.getshape()
+ strides = buf.getstrides()
+ for i in range(ndim):
+ view.c_shape[i] = shape[i]
+ view.c_strides[i] = strides[i]
+ view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+ view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+ return 0
+
+def _IsFortranContiguous(view):
+ ndim = widen(view.c_ndim)
+ if ndim == 0:
+ return 1
+ if not view.c_strides:
+ return ndim == 1
+ sd = view.c_itemsize
+ if ndim == 1:
+ return view.c_shape[0] == 1 or sd == view.c_strides[0]
+ for i in range(view.c_ndim):
+ dim = view.c_shape[i]
+ if dim == 0:
+ return 1
+ if view.c_strides[i] != sd:
+ return 0
+ sd *= dim
+ return 1
+
+def _IsCContiguous(view):
+ ndim = widen(view.c_ndim)
+ if ndim == 0:
+ return 1
+ if not view.c_strides:
+ return ndim == 1
+ sd = view.c_itemsize
+ if ndim == 1:
+ return view.c_shape[0] == 1 or sd == view.c_strides[0]
+ for i in range(ndim - 1, -1, -1):
+ dim = view.c_shape[i]
+ if dim == 0:
+ return 1
+ if view.c_strides[i] != sd:
+ return 0
+ sd *= dim
+ return 1
+
+ at cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+ """Return 1 if the memory defined by the view is C-style (fortran is
+ 'C') or Fortran-style (fortran is 'F') contiguous or either one
+ (fortran is 'A'). Return 0 otherwise."""
+ # traverse the strides, checking for consistent stride increases from
+ # right-to-left (c) or left-to-right (fortran). Copied from cpython
+ if not view.c_suboffsets:
+ return 0
+ if (fort == 'C'):
+ return _IsCContiguous(view)
+ elif (fort == 'F'):
+ return _IsFortranContiguous(view)
+ elif (fort == 'A'):
+ return (_IsCContiguous(view) or _IsFortranContiguous(view))
+ return 0
+
@cpython_api([PyObject], PyObject)
def PyMemoryView_FromObject(space, w_obj):
return space.call_method(space.builtin, "memoryview", w_obj)
@@ -12,6 +124,7 @@
@cpython_api([PyObject], PyObject)
def PyMemoryView_GET_BASE(space, w_obj):
# return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
+ # XXX needed for numpy on py3k
raise NotImplementedError('PyMemoryView_GET_BUFFER')
@cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
@@ -20,21 +133,22 @@
object. The object must be a memoryview instance; this macro doesn't
check its type, you must do it yourself or you will risk crashes."""
view = lltype.malloc(Py_buffer, flavor='raw', zero=True)
- # TODO - fill in fields
- '''
- view.c_buf = buf
- view.c_len = length
- view.c_obj = obj
- Py_IncRef(space, obj)
- view.c_itemsize = 1
- rffi.setintfield(view, 'c_readonly', readonly)
- rffi.setintfield(view, 'c_ndim', 0)
- view.c_format = lltype.nullptr(rffi.CCHARP.TO)
- view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
- view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
- view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
- view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
- '''
+ if not isinstance(w_obj, W_MemoryView):
+ return view
+ ndim = w_obj.buf.getndim()
+ if ndim >= Py_MAX_NDIMS:
+ # XXX warn?
+ return view
+ try:
+ view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+ view.c_obj = make_ref(space, w_obj)
+ rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+ isstr = False
+ except ValueError:
+ w_s = w_obj.descr_tobytes(space)
+ view.c_obj = make_ref(space, w_s)
+ rffi.setintfield(view, 'c_readonly', 1)
+ isstr = True
+ fill_Py_buffer(space, w_obj.buf, view)
return view
-
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -508,10 +508,9 @@
@cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL)
def PyBuffer_Release(space, view):
"""
- Releases a Py_buffer obtained from getbuffer ParseTuple's s*.
-
- This is not a complete re-implementation of the CPython API; it only
- provides a subset of CPython's behavior.
+ Release the buffer view. This should be called when the buffer is
+ no longer being used as it may free memory from it
"""
Py_DecRef(space, view.c_obj)
view.c_obj = lltype.nullptr(PyObject.TO)
+ # XXX do other fields leak memory?
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
def PySequence_Fast(space, w_obj, m):
"""Returns the sequence o as a tuple, unless it is already a tuple or list, in
which case o is returned. Use PySequence_Fast_GET_ITEM() to access the
- members of the result. Returns NULL on failure. If the object is not a
- sequence, raises TypeError with m as the message text."""
+ members of the result. Returns NULL on failure. If the object cannot be
+ converted to a sequence, and raises a TypeError, raise a new TypeError with
+ m as the message text. If the conversion otherwise, fails, reraise the
+ original exception"""
if isinstance(w_obj, W_ListObject):
# make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM
w_obj.convert_to_cpy_strategy(space)
return w_obj
try:
return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
- except OperationError:
- raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+ except OperationError as e:
+ if e.match(space, space.w_TypeError):
+ raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+ raise e
@cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -335,9 +335,15 @@
def getshape(self):
return self.shape
+ def getstrides(self):
+ return self.strides
+
def getitemsize(self):
return self.itemsize
+ def getndim(self):
+ return self.ndim
+
def wrap_getreadbuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -107,14 +107,11 @@
PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags)
{
PyMyArray* self = (PyMyArray*)obj;
- fprintf(stdout, "in PyMyArray_getbuffer\n");
if (view == NULL) {
- fprintf(stdout, "view is NULL\n");
PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
return -1;
}
if (flags == 0) {
- fprintf(stdout, "flags is 0\n");
PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer");
return -1;
}
@@ -188,7 +185,131 @@
(initproc)PyMyArray_init, /* tp_init */
};
+static PyObject*
+test_buffer(PyObject* self, PyObject* args)
+{
+ Py_buffer* view = NULL;
+ PyObject* obj = PyTuple_GetItem(args, 0);
+ PyObject* memoryview = PyMemoryView_FromObject(obj);
+ if (memoryview == NULL)
+ return PyInt_FromLong(-1);
+ view = PyMemoryView_GET_BUFFER(memoryview);
+ Py_DECREF(memoryview);
+ return PyInt_FromLong(view->len);
+}
+
+/* Copied from numpy tests */
+/*
+ * Create python string from a FLAG and or the corresponding PyBuf flag
+ * for the use in get_buffer_info.
+ */
+#define GET_PYBUF_FLAG(FLAG) \
+ buf_flag = PyUnicode_FromString(#FLAG); \
+ flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ); \
+ Py_DECREF(buf_flag); \
+ if (flag_matches == 1) { \
+ Py_DECREF(tmp); \
+ flags |= PyBUF_##FLAG; \
+ continue; \
+ } \
+ else if (flag_matches == -1) { \
+ Py_DECREF(tmp); \
+ return NULL; \
+ }
+
+
+/*
+ * Get information for a buffer through PyBuf_GetBuffer with the
+ * corresponding flags or'ed. Note that the python caller has to
+ * make sure that or'ing those flags actually makes sense.
+ * More information should probably be returned for future tests.
+ */
+static PyObject *
+get_buffer_info(PyObject *self, PyObject *args)
+{
+ PyObject *buffer_obj, *pyflags;
+ PyObject *tmp, *buf_flag;
+ Py_buffer buffer;
+ PyObject *shape, *strides;
+ Py_ssize_t i, n;
+ int flag_matches;
+ int flags = 0;
+
+ if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) {
+ return NULL;
+ }
+
+ n = PySequence_Length(pyflags);
+ if (n < 0) {
+ return NULL;
+ }
+
+ for (i=0; i < n; i++) {
+ tmp = PySequence_GetItem(pyflags, i);
+ if (tmp == NULL) {
+ return NULL;
+ }
+
+ GET_PYBUF_FLAG(SIMPLE);
+ GET_PYBUF_FLAG(WRITABLE);
+ GET_PYBUF_FLAG(STRIDES);
+ GET_PYBUF_FLAG(ND);
+ GET_PYBUF_FLAG(C_CONTIGUOUS);
+ GET_PYBUF_FLAG(F_CONTIGUOUS);
+ GET_PYBUF_FLAG(ANY_CONTIGUOUS);
+ GET_PYBUF_FLAG(INDIRECT);
+ GET_PYBUF_FLAG(FORMAT);
+ GET_PYBUF_FLAG(STRIDED);
+ GET_PYBUF_FLAG(STRIDED_RO);
+ GET_PYBUF_FLAG(RECORDS);
+ GET_PYBUF_FLAG(RECORDS_RO);
+ GET_PYBUF_FLAG(FULL);
+ GET_PYBUF_FLAG(FULL_RO);
+ GET_PYBUF_FLAG(CONTIG);
+ GET_PYBUF_FLAG(CONTIG_RO);
+
+ Py_DECREF(tmp);
+
+ /* One of the flags must match */
+ PyErr_SetString(PyExc_ValueError, "invalid flag used.");
+ return NULL;
+ }
+
+ if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) {
+ return NULL;
+ }
+
+ if (buffer.shape == NULL) {
+ Py_INCREF(Py_None);
+ shape = Py_None;
+ }
+ else {
+ shape = PyTuple_New(buffer.ndim);
+ for (i=0; i < buffer.ndim; i++) {
+ PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i]));
+ }
+ }
+
+ if (buffer.strides == NULL) {
+ Py_INCREF(Py_None);
+ strides = Py_None;
+ }
+ else {
+ strides = PyTuple_New(buffer.ndim);
+ for (i=0; i < buffer.ndim; i++) {
+ PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i]));
+ }
+ }
+
+ PyBuffer_Release(&buffer);
+ return Py_BuildValue("(NN)", shape, strides);
+}
+
+
+
static PyMethodDef buffer_functions[] = {
+ {"test_buffer", (PyCFunction)test_buffer, METH_VARARGS, NULL},
+ {"get_buffer_info", (PyCFunction)get_buffer_info, METH_VARARGS, NULL},
{NULL, NULL} /* Sentinel */
};
diff --git a/pypy/module/cpyext/test/foo3.c b/pypy/module/cpyext/test/foo3.c
--- a/pypy/module/cpyext/test/foo3.c
+++ b/pypy/module/cpyext/test/foo3.c
@@ -4,9 +4,7 @@
PyObject* foo3type_tp_new(PyTypeObject* metatype, PyObject* args, PyObject* kwds)
{
PyObject* newType;
- /*printf("in foo3type_tp_new, preprocessing...\n"); */
newType = PyType_Type.tp_new(metatype, args, kwds);
- /*printf("in foo3type_tp_new, postprocessing...\n"); */
return newType;
}
@@ -81,4 +79,5 @@
return;
if (PyDict_SetItemString(d, "footype", (PyObject *)&footype) < 0)
return;
+ Py_INCREF(&footype);
}
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
from pypy.interpreter.baseobjspace import W_Root
from pypy.module.cpyext.state import State
from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
from pypy.module.cpyext.api import PyObject
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -183,8 +183,27 @@
Py_INCREF(Py_None);
return Py_None;
"""),
+ ("c_only", "METH_NOARGS",
+ """
+ int ret;
+ char * buf2;
+ PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024);
+ if (!obj)
+ return NULL;
+ buf2 = PyBytes_AsString(obj);
+ if (!buf2)
+ return NULL;
+ /* buf should not have been forced, issue #2395 */
+ ret = _PyBytes_Resize(&obj, 512);
+ if (ret < 0)
+ return NULL;
+ Py_DECREF(obj);
+ Py_INCREF(Py_None);
+ return Py_None;
+ """),
])
module.getbytes()
+ module.c_only()
def test_py_string_as_string_Unicode(self):
module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
from .support import c_compile
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
+
@api.cpython_api([], api.PyObject)
def PyPy_Crash1(space):
1/0
@@ -53,7 +55,48 @@
libraries=libraries)
return soname
-def compile_extension_module(space, modname, include_dirs=[],
+class SystemCompilationInfo(object):
+ """Bundles all the generic information required to compile extensions.
+
+ Note: here, 'system' means OS + target interpreter + test config + ...
+ """
+ def __init__(self, include_extra=None, compile_extra=None, link_extra=None,
+ extra_libs=None, ext=None):
+ self.include_extra = include_extra or []
+ self.compile_extra = compile_extra
+ self.link_extra = link_extra
+ self.extra_libs = extra_libs
+ self.ext = ext
+
+def get_cpyext_info(space):
+ from pypy.module.imp.importing import get_so_extension
+ state = space.fromcache(State)
+ api_library = state.api_lib
+ if sys.platform == 'win32':
+ libraries = [api_library]
+ # '%s' undefined; assuming extern returning int
+ compile_extra = ["/we4013"]
+ # prevent linking with PythonXX.lib
+ w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+ link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+ (space.int_w(w_maj), space.int_w(w_min))]
+ else:
+ libraries = []
+ if sys.platform.startswith('linux'):
+ compile_extra = [
+ "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+ link_extra = ["-g"]
+ else:
+ compile_extra = link_extra = None
+ return SystemCompilationInfo(
+ include_extra=api.include_dirs,
+ compile_extra=compile_extra,
+ link_extra=link_extra,
+ extra_libs=libraries,
+ ext=get_so_extension(space))
+
+
+def compile_extension_module(sys_info, modname, include_dirs=[],
source_files=None, source_strings=None):
"""
Build an extension module and return the filename of the resulting native
@@ -65,35 +108,15 @@
Any extra keyword arguments are passed on to ExternalCompilationInfo to
build the module (so specify your source with one of those).
"""
- state = space.fromcache(State)
- api_library = state.api_lib
- if sys.platform == 'win32':
- libraries = [api_library]
- # '%s' undefined; assuming extern returning int
- compile_extra = ["/we4013"]
- # prevent linking with PythonXX.lib
- w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
- link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
- (space.int_w(w_maj), space.int_w(w_min))]
- else:
- libraries = []
- if sys.platform.startswith('linux'):
- compile_extra = ["-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
- link_extra = ["-g"]
- else:
- compile_extra = link_extra = None
-
modname = modname.split('.')[-1]
soname = create_so(modname,
- include_dirs=api.include_dirs + include_dirs,
- source_files=source_files,
- source_strings=source_strings,
- compile_extra=compile_extra,
- link_extra=link_extra,
- libraries=libraries)
- from pypy.module.imp.importing import get_so_extension
- ext = get_so_extension(space)
- pydname = soname.new(purebasename=modname, ext=ext)
+ include_dirs=sys_info.include_extra + include_dirs,
+ source_files=source_files,
+ source_strings=source_strings,
+ compile_extra=sys_info.compile_extra,
+ link_extra=sys_info.link_extra,
+ libraries=sys_info.extra_libs)
+ pydname = soname.new(purebasename=modname, ext=sys_info.ext)
soname.rename(pydname)
return str(pydname)
@@ -106,18 +129,8 @@
raise RuntimeError("This interpreter does not define a filename "
"suffix for C extensions!")
-def compile_extension_module_applevel(space, modname, include_dirs=[],
- source_files=None, source_strings=None):
- """
- Build an extension module and return the filename of the resulting native
- code file.
-
- modname is the name of the module, possibly including dots if it is a module
- inside a package.
-
- Any extra keyword arguments are passed on to ExternalCompilationInfo to
- build the module (so specify your source with one of those).
- """
+def get_sys_info_app():
+ from distutils.sysconfig import get_python_inc
if sys.platform == 'win32':
compile_extra = ["/we4013"]
link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')]
@@ -128,18 +141,13 @@
compile_extra = [
"-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
link_extra = None
+ ext = get_so_suffix()
+ return SystemCompilationInfo(
+ include_extra=[get_python_inc()],
+ compile_extra=compile_extra,
+ link_extra=link_extra,
+ ext=get_so_suffix())
- modname = modname.split('.')[-1]
- soname = create_so(modname,
- include_dirs=[space.include_dir] + include_dirs,
- source_files=source_files,
- source_strings=source_strings,
- compile_extra=compile_extra,
- link_extra=link_extra)
- ext = get_so_suffix()
- pydname = soname.new(purebasename=modname, ext=ext)
- soname.rename(pydname)
- return str(pydname)
def freeze_refcnts(self):
rawrefcount._dont_free_any_more()
@@ -154,9 +162,7 @@
class FakeSpace(object):
"""Like TinyObjSpace, but different"""
def __init__(self, config):
- from distutils.sysconfig import get_python_inc
self.config = config
- self.include_dir = get_python_inc()
def passthrough(self, arg):
return arg
@@ -271,11 +277,11 @@
"the test actually passed in the first place; if it failed "
"it is likely to reach this place.")
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_only_import(self):
import cpyext
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_load_error(self):
import cpyext
raises(ImportError, cpyext.load_module, "missing.file", "foo")
@@ -303,27 +309,23 @@
def setup_method(self, func):
@gateway.unwrap_spec(name=str)
def compile_module(space, name,
- w_separate_module_files=None,
- w_separate_module_sources=None):
+ w_source_files=None,
+ w_source_strings=None):
"""
Build an extension module linked against the cpyext api library.
"""
- if not space.is_none(w_separate_module_files):
- separate_module_files = space.listview_bytes(
- w_separate_module_files)
- assert separate_module_files is not None
+ if not space.is_none(w_source_files):
+ source_files = space.listview_bytes(w_source_files)
else:
- separate_module_files = []
- if not space.is_none(w_separate_module_sources):
- separate_module_sources = space.listview_bytes(
- w_separate_module_sources)
- assert separate_module_sources is not None
+ source_files = None
+ if not space.is_none(w_source_strings):
+ source_strings = space.listview_bytes(w_source_strings)
else:
- separate_module_sources = []
- pydname = self.compile_extension_module(
- space, name,
- source_files=separate_module_files,
- source_strings=separate_module_sources)
+ source_strings = None
+ pydname = compile_extension_module(
+ self.sys_info, name,
+ source_files=source_files,
+ source_strings=source_strings)
return space.wrap(pydname)
@gateway.unwrap_spec(name=str, init='str_or_None', body=str,
@@ -376,7 +378,7 @@
filename = py.path.local(pypydir) / 'module' \
/ 'cpyext'/ 'test' / (filename + ".c")
kwds = dict(source_files=[filename])
- mod = self.compile_extension_module(space, name,
+ mod = compile_extension_module(self.sys_info, name,
include_dirs=include_dirs, **kwds)
if load_it:
@@ -468,11 +470,11 @@
return run
def wrap(func):
return func
- self.compile_extension_module = compile_extension_module_applevel
+ self.sys_info = get_sys_info_app()
else:
interp2app = gateway.interp2app
wrap = self.space.wrap
- self.compile_extension_module = compile_extension_module
+ self.sys_info = get_cpyext_info(self.space)
self.w_compile_module = wrap(interp2app(compile_module))
self.w_import_module = wrap(interp2app(import_module))
self.w_reimport_module = wrap(interp2app(reimport_module))
@@ -634,10 +636,10 @@
skip('record_imported_module not supported in runappdirect mode')
# Build the extensions.
banana = self.compile_module(
- "apple.banana", separate_module_files=[self.here + 'banana.c'])
+ "apple.banana", source_files=[self.here + 'banana.c'])
self.record_imported_module("apple.banana")
date = self.compile_module(
- "cherry.date", separate_module_files=[self.here + 'date.c'])
+ "cherry.date", source_files=[self.here + 'date.c'])
self.record_imported_module("cherry.date")
# Set up some package state so that the extensions can actually be
@@ -894,7 +896,7 @@
])
raises(SystemError, mod.newexc, "name", Exception, {})
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy specific test')
+ @pytest.mark.skipif(only_pypy, reason='pypy specific test')
def test_hash_pointer(self):
mod = self.import_extension('foo', [
('get_hash', 'METH_NOARGS',
@@ -945,7 +947,7 @@
print p
assert 'py' in p
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_get_version(self):
mod = self.import_extension('foo', [
('get_version', 'METH_NOARGS',
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,6 +1,7 @@
+from rpython.rtyper.lltypesystem import rffi
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from rpython.rlib.buffer import StringBuffer
class TestMemoryViewObject(BaseApiTest):
def test_fromobject(self, space, api):
@@ -12,6 +13,16 @@
w_bytes = space.call_method(w_view, "tobytes")
assert space.unwrap(w_bytes) == "hello"
+ def test_frombuffer(self, space, api):
+ w_buf = space.newbuffer(StringBuffer("hello"))
+ w_memoryview = api.PyMemoryView_FromObject(w_buf)
+ w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
+ assert w_view.c_ndim == 1
+ f = rffi.charp2str(w_view.c_format)
+ assert f == 'B'
+ assert w_view.c_shape[0] == 5
+ assert w_view.c_strides[0] == 1
+ assert w_view.c_len == 5
class AppTestBufferProtocol(AppTestCpythonExtensionBase):
def test_buffer_protocol(self):
@@ -21,6 +32,21 @@
y = memoryview(arr)
assert y.format == 'i'
assert y.shape == (10,)
+ assert len(y) == 10
s = y[3]
assert len(s) == struct.calcsize('i')
assert s == struct.pack('i', 3)
+ viewlen = module.test_buffer(arr)
+ assert viewlen == y.itemsize * len(y)
+
+ def test_buffer_info(self):
+ from _numpypy import multiarray as np
+ module = self.import_module(name='buffer_test')
+ get_buffer_info = module.get_buffer_info
+ raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+ arr = np.zeros((1, 10), order='F')
+ shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+ assert strides[0] == 8
+ arr = np.zeros((10, 1), order='C')
+ shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+ assert strides[-1] == 8
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
assert module.test_fast_sequence(s[0:-1])
assert module.test_fast_sequence(s[::-1])
+ def test_fast_keyerror(self):
+ module = self.import_extension('foo', [
+ ("test_fast_sequence", "METH_VARARGS",
+ """
+ PyObject *foo;
+ PyObject * seq = PyTuple_GetItem(args, 0);
+ if (seq == NULL)
+ Py_RETURN_NONE;
+ foo = PySequence_Fast(seq, "Could not convert object to sequence");
+ if (foo != NULL)
+ {
+ return foo;
+ }
+ if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+ PyErr_Clear();
+ return PyBool_FromLong(1);
+ }
+ return NULL;
+ """)])
+ class Map(object):
+ def __len__(self):
+ return 1
+
+ def __getitem__(self, index):
+ raise KeyError()
+
+ assert module.test_fast_sequence(Map()) is True
+
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -1,12 +1,13 @@
import sys
-import py, pytest
+import pytest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
class AppTestThread(AppTestCpythonExtensionBase):
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_get_thread_ident(self):
module = self.import_extension('foo', [
("get_thread_ident", "METH_NOARGS",
@@ -33,7 +34,7 @@
assert results[0][0] != results[1][0]
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_acquire_lock(self):
module = self.import_extension('foo', [
("test_acquire_lock", "METH_NOARGS",
@@ -57,7 +58,7 @@
])
module.test_acquire_lock()
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_release_lock(self):
module = self.import_extension('foo', [
("test_release_lock", "METH_NOARGS",
@@ -79,7 +80,7 @@
])
module.test_release_lock()
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_tls(self):
module = self.import_extension('foo', [
("create_key", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -961,7 +961,6 @@
def test_tp_new_in_subclass_of_type(self):
module = self.import_module(name='foo3')
- #print('calling module.footype()...')
module.footype("X", (object,), {})
def test_app_subclass_of_c_type(self):
diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,6 +3,7 @@
import py, pytest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
def test_pragma_version():
from pypy.module.sys.version import CPYTHON_VERSION
@@ -32,11 +33,9 @@
assert module.py_minor_version == sys.version_info.minor
assert module.py_micro_version == sys.version_info.micro
- #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_pypy_versions(self):
import sys
- if '__pypy__' not in sys.builtin_module_names:
- py.test.skip("pypy only test")
init = """
if (Py_IsInitialized()) {
PyObject *m = Py_InitModule("foo", NULL);
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -293,6 +293,8 @@
STRUCT_TYPE = PyNumberMethods
elif slot_names[0] == 'c_tp_as_sequence':
STRUCT_TYPE = PySequenceMethods
+ elif slot_names[0] == 'c_tp_as_buffer':
+ STRUCT_TYPE = PyBufferProcs
else:
raise AssertionError(
"Structure not allocated: %s" % (slot_names[0],))
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -259,10 +259,8 @@
raise oefmt(space.w_ValueError, "Empty module name")
w = space.wrap
- if w_fromlist is not None and space.is_true(w_fromlist):
- fromlist_w = space.fixedview(w_fromlist)
- else:
- fromlist_w = None
+ if w_fromlist is not None and not space.is_true(w_fromlist):
+ w_fromlist = None
rel_modulename = None
if (level != 0 and w_globals is not None and
@@ -284,19 +282,19 @@
w_mod = None
else:
w_mod = absolute_import(space, rel_modulename, rel_level,
- fromlist_w, tentative=True)
+ w_fromlist, tentative=True)
else:
w_mod = absolute_import(space, rel_modulename, rel_level,
- fromlist_w, tentative=False)
+ w_fromlist, tentative=False)
if w_mod is not None:
return w_mod
- w_mod = absolute_import(space, modulename, 0, fromlist_w, tentative=0)
+ w_mod = absolute_import(space, modulename, 0, w_fromlist, tentative=0)
if rel_modulename is not None:
space.setitem(space.sys.get('modules'), w(rel_modulename), space.w_None)
return w_mod
-def absolute_import(space, modulename, baselevel, fromlist_w, tentative):
+def absolute_import(space, modulename, baselevel, w_fromlist, tentative):
# Short path: check in sys.modules, but only if there is no conflict
# on the import lock. In the situation of 'import' statements
# inside tight loops, this should be true, and absolute_import_try()
@@ -304,25 +302,25 @@
# if the import lock is currently held by another thread, then we
# have to wait, and so shouldn't use the fast path.
if not getimportlock(space).lock_held_by_someone_else():
- w_mod = absolute_import_try(space, modulename, baselevel, fromlist_w)
+ w_mod = absolute_import_try(space, modulename, baselevel, w_fromlist)
if w_mod is not None and not space.is_w(w_mod, space.w_None):
return w_mod
return absolute_import_with_lock(space, modulename, baselevel,
- fromlist_w, tentative)
+ w_fromlist, tentative)
@jit.dont_look_inside
def absolute_import_with_lock(space, modulename, baselevel,
- fromlist_w, tentative):
+ w_fromlist, tentative):
lock = getimportlock(space)
lock.acquire_lock()
try:
return _absolute_import(space, modulename, baselevel,
- fromlist_w, tentative)
+ w_fromlist, tentative)
finally:
lock.release_lock(silent_after_fork=True)
@jit.unroll_safe
-def absolute_import_try(space, modulename, baselevel, fromlist_w):
+def absolute_import_try(space, modulename, baselevel, w_fromlist):
""" Only look up sys.modules, not actually try to load anything
"""
w_path = None
@@ -330,7 +328,7 @@
if '.' not in modulename:
w_mod = check_sys_modules_w(space, modulename)
first = w_mod
- if fromlist_w is not None and w_mod is not None:
+ if w_fromlist is not None and w_mod is not None:
w_path = try_getattr(space, w_mod, space.wrap('__path__'))
else:
level = 0
@@ -345,28 +343,37 @@
return None
if level == baselevel:
first = w_mod
- if fromlist_w is not None:
+ if w_fromlist is not None:
w_path = try_getattr(space, w_mod, space.wrap('__path__'))
level += 1
- if fromlist_w is not None:
+ if w_fromlist is not None:
+ # bit artificial code but important to not just unwrap w_fromlist
+ # to get a better trace. if it is unwrapped, the immutability of the
+ # tuple is lost
if w_path is not None:
- if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],
- space.wrap('*')):
+ length = space.len_w(w_fromlist)
+ if length == 1 and space.eq_w(
+ space.getitem(w_fromlist, space.wrap(0)),
+ space.wrap('*')):
w_all = try_getattr(space, w_mod, space.wrap('__all__'))
if w_all is not None:
- fromlist_w = space.fixedview(w_all)
+ w_fromlist = w_all
+ length = space.len_w(w_fromlist)
else:
- fromlist_w = []
+ w_fromlist = None
# "from x import *" with x already imported and no x.__all__
# always succeeds without doing more imports. It will
# just copy everything from x.__dict__ as it is now.
- for w_name in fromlist_w:
- if try_getattr(space, w_mod, w_name) is None:
- return None
+
+ if w_fromlist is not None:
+ for i in range(length):
+ w_name = space.getitem(w_fromlist, space.wrap(i))
+ if try_getattr(space, w_mod, w_name) is None:
+ return None
return w_mod
return first
-def _absolute_import(space, modulename, baselevel, fromlist_w, tentative):
+def _absolute_import(space, modulename, baselevel, w_fromlist, tentative):
w = space.wrap
if '/' in modulename or '\\' in modulename:
@@ -394,18 +401,24 @@
w_path = try_getattr(space, w_mod, w('__path__'))
level += 1
- if fromlist_w is not None:
+ if w_fromlist is not None:
if w_path is not None:
- if len(fromlist_w) == 1 and space.eq_w(fromlist_w[0],w('*')):
+ length = space.len_w(w_fromlist)
+ if length == 1 and space.eq_w(
+ space.getitem(w_fromlist, space.wrap(0)),
+ space.wrap('*')):
w_all = try_getattr(space, w_mod, w('__all__'))
if w_all is not None:
- fromlist_w = space.fixedview(w_all)
+ w_fromlist = w_all
+ length = space.len_w(w_fromlist)
else:
- fromlist_w = []
- for w_name in fromlist_w:
- if try_getattr(space, w_mod, w_name) is None:
- load_part(space, w_path, prefix, space.str0_w(w_name),
- w_mod, tentative=1)
+ w_fromlist = None
+ if w_fromlist is not None:
+ for i in range(length):
+ w_name = space.getitem(w_fromlist, space.wrap(i))
+ if try_getattr(space, w_mod, w_name) is None:
+ load_part(space, w_path, prefix, space.str0_w(w_name),
+ w_mod, tentative=1)
return w_mod
else:
return first
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -65,8 +65,9 @@
)
setuppkg("pkg.pkg2", a='', b='')
setuppkg("pkg.withall",
- __init__ = "__all__ = ['foobar']",
- foobar = "found = 123")
+ __init__ = "__all__ = ['foobar', 'barbaz']",
+ foobar = "found = 123",
+ barbaz = "other = 543")
setuppkg("pkg.withoutall",
__init__ = "",
foobar = "found = 123")
@@ -707,6 +708,7 @@
d = {}
exec "from pkg.withall import *" in d
assert d["foobar"].found == 123
+ assert d["barbaz"].other == 543
def test_import_star_does_not_find_submodules_without___all__(self):
for case in ["not-imported-yet", "already-imported"]:
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -460,6 +460,9 @@
def getdictvalue(self, space, key):
return self.items[key]
+ def descr_memoryview(self, space, buf):
+ raise oefmt(space.w_TypeError, "error")
+
class IterDictObject(W_Root):
def __init__(self, space, w_dict):
self.space = space
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -377,7 +377,25 @@
def __exit__(self, typ, value, traceback):
keepalive_until_here(self)
- def get_buffer(self, space, readonly):
+ def get_buffer(self, space, flags):
+ errtype = space.w_ValueError # should be BufferError, numpy does this instead
+ if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and
+ not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+ raise oefmt(errtype, "ndarray is not C-contiguous")
+ if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and
+ not self.flags & NPY.ARRAY_F_CONTIGUOUS):
+ raise oefmt(errtype, "ndarray is not Fortran contiguous")
+ if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
+ not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
+ self.flags & NPY.ARRAY_C_CONTIGUOUS)):
+ raise oefmt(errtype, "ndarray is not contiguous")
+ if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
+ not self.flags & NPY.ARRAY_C_CONTIGUOUS):
+ raise oefmt(errtype, "ndarray is not C-contiguous")
+ if ((flags & space.BUF_WRITABLE) == space.BUF_WRITABLE and
+ not self.flags & NPY.ARRAY_WRITEABLE):
+ raise oefmt(errtype, "buffer source array is read-only")
+ readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
return ArrayBuffer(self, readonly)
def astype(self, space, dtype, order, copy=True):
@@ -695,6 +713,7 @@
index + self.impl.start)
def setitem(self, index, v):
+ # XXX what if self.readonly?
raw_storage_setitem(self.impl.storage, index + self.impl.start,
rffi.cast(lltype.Char, v))
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -1,4 +1,5 @@
from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import BufferInterfaceNotFound
from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
from rpython.rlib.buffer import SubBuffer
from rpython.rlib.rstring import strip_spaces
@@ -42,7 +43,7 @@
raise oefmt(space.w_ValueError,
"object __array__ method not producing an array")
-def try_interface_method(space, w_object):
+def try_interface_method(space, w_object, copy):
try:
w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
if w_interface is None:
@@ -81,17 +82,20 @@
raise oefmt(space.w_ValueError,
"__array_interface__ could not decode dtype %R", w_dtype
)
- if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or space.isinstance_w(w_data, space.w_list)):
+ if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
+ space.isinstance_w(w_data, space.w_list)):
data_w = space.listview(w_data)
- data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
- read_only = True # XXX why not space.is_true(data_w[1])
+ w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
+ read_only = space.is_true(data_w[1]) or copy
offset = 0
- return W_NDimArray.from_shape_and_storage(space, shape, data,
- dtype, strides=strides, start=offset), read_only
+ w_base = w_object
+ if read_only:
+ w_base = None
+ return W_NDimArray.from_shape_and_storage(space, shape, w_data,
+ dtype, w_base=w_base, strides=strides,
+ start=offset), read_only
if w_data is None:
- data = w_object
- else:
- data = w_data
+ w_data = w_object
w_offset = space.finditem(w_interface, space.wrap('offset'))
if w_offset is None:
offset = 0
@@ -101,7 +105,7 @@
if strides is not None:
raise oefmt(space.w_NotImplementedError,
"__array_interface__ strides not fully supported yet")
- arr = frombuffer(space, data, dtype, support.product(shape), offset)
+ arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
new_impl = arr.implementation.reshape(arr, shape)
return W_NDimArray(new_impl), False
@@ -110,6 +114,78 @@
return None, False
raise
+def _descriptor_from_pep3118_format(space, c_format):
+ descr = descriptor.decode_w_dtype(space, space.wrap(c_format))
+ if descr:
+ return descr
+ msg = "invalid PEP 3118 format string: '%s'" % c_format
+ space.warn(space.wrap(msg), space.w_RuntimeWarning)
+ return None
+
+def _array_from_buffer_3118(space, w_object, dtype):
+ try:
+ w_buf = space.call_method(space.builtin, "memoryview", w_object)
+ except OperationError as e:
+ if e.match(space, space.w_TypeError):
+ # object does not have buffer interface
+ return w_object
+ raise
+ format = space.getattr(w_buf,space.newbytes('format'))
+ if format:
+ descr = _descriptor_from_pep3118_format(space, space.str_w(format))
+ if not descr:
+ return w_object
+ if dtype and descr:
+ raise oefmt(space.w_NotImplementedError,
+ "creating an array from a memoryview while specifying dtype "
+ "not supported")
+ if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))):
+ msg = ("Item size computed from the PEP 3118 buffer format "
+ "string does not match the actual item size.")
+ space.warn(space.wrap(msg), space.w_RuntimeWarning)
+ return w_object
+ dtype = descr
+ elif not dtype:
+ dtype = descriptor.get_dtype_cache(space).w_stringdtype
+ dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
+ nd = space.int_w(space.getattr(w_buf, space.newbytes('ndim')))
+ shape = [space.int_w(d) for d in space.listview(
+ space.getattr(w_buf, space.newbytes('shape')))]
+ strides = []
+ buflen = space.len_w(w_buf) * dtype.elsize
+ if shape:
+ strides = [space.int_w(d) for d in space.listview(
+ space.getattr(w_buf, space.newbytes('strides')))]
+ if not strides:
+ d = buflen
+ strides = [0] * nd
+ for k in range(nd):
+ if shape[k] > 0:
+ d /= shape[k]
+ strides[k] = d
+ else:
+ if nd == 1:
+ shape = [buflen / dtype.elsize, ]
+ strides = [dtype.elsize, ]
+ elif nd > 1:
+ msg = ("ndim computed from the PEP 3118 buffer format "
+ "is greater than 1, but shape is NULL.")
+ space.warn(space.wrap(msg), space.w_RuntimeWarning)
+ return w_object
+ try:
+ w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(space.call_method(w_buf, '_pypy_raw_address')))
+ except OperationError as e:
+ if e.match(space, space.w_ValueError):
+ return w_object
+ else:
+ raise e
+ writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
+ w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
+ storage_bytes=buflen, dtype=dtype, w_base=w_object,
+ writable=writable, strides=strides)
+ if w_ret:
+ return w_ret
+ return w_object
@unwrap_spec(ndmin=int, copy=bool, subok=bool)
def array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False,
@@ -127,6 +203,7 @@
def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
+ from pypy.module.micronumpy.boxes import W_GenericBox
# numpy testing calls array(type(array([]))) and expects a ValueError
if space.isinstance_w(w_object, space.w_type):
raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
@@ -134,13 +211,19 @@
dtype = descriptor.decode_w_dtype(space, w_dtype)
if not isinstance(w_object, W_NDimArray):
w_array = try_array_method(space, w_object, w_dtype)
- if w_array is not None:
+ if w_array is None:
+ if ( not space.isinstance_w(w_object, space.w_str) and
+ not space.isinstance_w(w_object, space.w_unicode) and
+ not isinstance(w_object, W_GenericBox)):
+ # use buffer interface
+ w_object = _array_from_buffer_3118(space, w_object, dtype)
+ else:
# continue with w_array, but do further operations in place
w_object = w_array
copy = False
dtype = w_object.get_dtype()
if not isinstance(w_object, W_NDimArray):
- w_array, _copy = try_interface_method(space, w_object)
+ w_array, _copy = try_interface_method(space, w_object, copy)
if w_array is not None:
w_object = w_array
copy = _copy
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -805,19 +805,19 @@
return w_result
def buffer_w(self, space, flags):
- return self.implementation.get_buffer(space, True)
+ return self.implementation.get_buffer(space, flags)
def readbuf_w(self, space):
- return self.implementation.get_buffer(space, True)
+ return self.implementation.get_buffer(space, space.BUF_FULL_RO)
def writebuf_w(self, space):
- return self.implementation.get_buffer(space, False)
+ return self.implementation.get_buffer(space, space.BUF_FULL)
def charbuf_w(self, space):
- return self.implementation.get_buffer(space, True).as_str()
+ return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
def descr_get_data(self, space):
- return space.newbuffer(self.implementation.get_buffer(space, False))
+ return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
@unwrap_spec(offset=int, axis1=int, axis2=int)
def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3215,7 +3215,9 @@
raises(TypeError, array, Dummy({'version': 3, 'typestr': 'f8', 'shape': ('a', 3)}))
a = array([1, 2, 3])
- b = array(Dummy(a.__array_interface__))
+ d = Dummy(a.__array_interface__)
+ b = array(d)
+ assert b.base is None
b[1] = 200
assert a[1] == 2 # upstream compatibility, is this a bug?
interface_a = a.__array_interface__
@@ -3226,6 +3228,8 @@
interface_b.pop('data')
interface_a.pop('data')
assert interface_a == interface_b
+ b = array(d, copy=False)
+ assert b.base is d
b = array(Dummy({'version':3, 'shape': (50,), 'typestr': 'u1',
'data': 'a'*100}))
@@ -3594,6 +3598,7 @@
cls.w_float32val = cls.space.wrap(struct.pack('f', 5.2))
cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
+ cls.w_one = cls.space.wrap(struct.pack('i', 1))
def test_frombuffer(self):
import numpy as np
@@ -3645,8 +3650,6 @@
else:
EMPTY = None
x = np.array([1, 2, 3, 4, 5], dtype='i')
- y = memoryview('abc')
- assert y.format == 'B'
y = memoryview(x)
assert y.format == 'i'
assert y.shape == (5,)
@@ -3654,6 +3657,16 @@
assert y.strides == (4,)
assert y.suboffsets == EMPTY
assert y.itemsize == 4
+ assert isinstance(y, memoryview)
+ assert y[0] == self.one
+ assert (np.array(y) == x).all()
+
+ x = np.array([0, 0, 0, 0], dtype='O')
+ y = memoryview(x)
+ # handles conversion of address to pinned object?
+ z = np.array(y)
+ assert z.dtype == 'O'
+ assert (z == x).all()
def test_fromstring(self):
import sys
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -702,3 +702,32 @@
ret = obj.sum()
print type(ret)
assert ret.info == 'spam'
+
+ def test_ndarray_subclass_assigns_base(self):
+ import numpy as np
+ init_called = []
+ class _DummyArray(object):
+ """ Dummy object that just exists to hang __array_interface__ dictionaries
+ and possibly keep alive a reference to a base array.
+ """
+ def __init__(self, interface, base=None):
+ self.__array_interface__ = interface
+ init_called.append(1)
+ self.base = base
+
+ x = np.zeros(10)
+ d = _DummyArray(x.__array_interface__, base=x)
+ y = np.array(d, copy=False)
+ assert sum(init_called) == 1
+ assert y.base is d
+
+ x = np.zeros((0,), dtype='float32')
+ intf = x.__array_interface__.copy()
+ intf["strides"] = x.strides
+ x.__array_interface__["strides"] = x.strides
+ d = _DummyArray(x.__array_interface__, base=x)
+ y = np.array(d, copy=False)
+ assert sum(init_called) == 2
+ assert y.base is d
+
+
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1851,7 +1851,7 @@
arr.gcstruct)
def read(self, arr, i, offset, dtype):
- if arr.gcstruct is V_OBJECTSTORE:
+ if arr.gcstruct is V_OBJECTSTORE and not arr.base():
raise oefmt(self.space.w_NotImplementedError,
"cannot read object from array with no gc hook")
return self.box(self._read(arr.storage, i, offset))
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -392,31 +392,39 @@
extobj_w = space.newlist([space.wrap(8192), space.wrap(0), space.w_None])
return extobj_w
+
+_reflected_ops = {
+ 'add': 'radd',
+ 'subtract': 'rsub',
+ 'multiply': 'rmul',
+ 'divide': 'rdiv',
+ 'true_divide': 'rtruediv',
+ 'floor_divide': 'rfloordiv',
+ 'remainder': 'rmod',
+ 'power': 'rpow',
+ 'left_shift': 'rlshift',
+ 'right_shift': 'rrshift',
+ 'bitwise_and': 'rand',
+ 'bitwise_xor': 'rxor',
+ 'bitwise_or': 'ror',
+ #/* Comparisons */
More information about the pypy-commit
mailing list