[pypy-commit] pypy py3.6: merge default into branch
mattip
pypy.commits at gmail.com
Sun Mar 10 06:42:51 EDT 2019
Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r96233:bc8fb0e12474
Date: 2019-03-10 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/bc8fb0e12474/
Log: merge default into branch
diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py
--- a/extra_tests/ctypes_tests/test_structures.py
+++ b/extra_tests/ctypes_tests/test_structures.py
@@ -119,12 +119,15 @@
ms.n = 0xff00
return repr(ba[:])
+ nstruct = dostruct(Native)
if sys.byteorder == 'little':
- assert dostruct(Native) == dostruct(Little)
- assert dostruct(Native) != dostruct(Big)
+ assert nstruct == dostruct(Little)
+ assert nstruct != dostruct(Big)
+ assert Big._fields_[0][1] is not i
else:
- assert dostruct(Native) == dostruct(Big)
- assert dostruct(Native) != dostruct(Little)
+ assert nstruct == dostruct(Big)
+ assert nstruct != dostruct(Little)
+ assert Little._fields_[0][1] is not i
def test_from_buffer_copy():
from array import array
@@ -185,3 +188,20 @@
assert sizeof(s) == 3 * sizeof(c_int)
assert s.a == 4 # 256 + 4
assert s.b == -123
+
+def test_memoryview():
+ class S(Structure):
+ _fields_ = [('a', c_int16),
+ ('b', c_int16),
+ ]
+
+ S3 = S * 3
+ c_array = (2 * S3)(
+ S3(S(a=0, b=1), S(a=2, b=3), S(a=4, b=5)),
+ S3(S(a=6, b=7), S(a=8, b=9), S(a=10, b=11)),
+ )
+
+ mv = memoryview(c_array)
+ assert mv.format == 'T{<h:a:<h:b:}'
+ assert mv.shape == (2, 3)
+ assert mv.itemsize == 4
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -4,6 +4,7 @@
from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof
from _ctypes.basics import keepalive_key, store_reference, ensure_objects
from _ctypes.basics import CArgObject, as_ffi_pointer
+import sys, __pypy__, struct
class ArrayMeta(_CDataMeta):
def __new__(self, name, cls, typedict):
@@ -247,6 +248,24 @@
def _as_ffi_pointer_(self, ffitype):
return as_ffi_pointer(self, ffitype)
+ def __buffer__(self, flags):
+ shape = []
+ obj = self
+ while 1:
+ shape.append(obj._length_)
+ try:
+ obj[0]._length_
+ except (AttributeError, IndexError):
+ break
+ obj = obj[0]
+
+ fmt = get_format_str(obj._type_)
+ try:
+ itemsize = struct.calcsize(fmt[1:])
+ except:
+ itemsize = len(buffer(obj[0]))
+ return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt, shape)
+
ARRAY_CACHE = {}
def create_array_type(base, length):
@@ -266,3 +285,31 @@
cls = ArrayMeta(name, (Array,), tpdict)
ARRAY_CACHE[key] = cls
return cls
+
+byteorder = {'little': '<', 'big': '>'}
+swappedorder = {'little': '>', 'big': '<'}
+
+def get_format_str(typ):
+ if hasattr(typ, '_fields_'):
+ if hasattr(typ, '_swappedbytes_'):
+ bo = swappedorder[sys.byteorder]
+ else:
+ bo = byteorder[sys.byteorder]
+ flds = []
+ for name, obj in typ._fields_:
+ # Trim off the leading '<' or '>'
+ ch = get_format_str(obj)[1:]
+ if (ch) == 'B':
+ flds.append(byteorder[sys.byteorder])
+ else:
+ flds.append(bo)
+ flds.append(ch)
+ flds.append(':')
+ flds.append(name)
+ flds.append(':')
+ return 'T{' + ''.join(flds) + '}'
+ elif hasattr(typ, '_type_'):
+ ch = typ._type_
+ return byteorder[sys.byteorder] + ch
+ else:
+ raise ValueError('cannot get format string for %r' % typ)
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -2,8 +2,15 @@
from _rawffi import alt as _ffi
import sys
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+try:
+ from __pypy__ import builtinify
+except ImportError:
+ builtinify = lambda f: f
+
+try:
+ from __pypy__.bufferable import bufferable
+except ImportError:
+ bufferable = object
keepalive_key = str # XXX fix this when provided with test
@@ -64,7 +71,7 @@
'resbuffer' is a _rawffi array of length 1 containing the value,
and this returns a general Python object that corresponds.
"""
- res = object.__new__(self)
+ res = bufferable.__new__(self)
res.__class__ = self
res.__dict__['_buffer'] = resbuffer
if base is not None:
@@ -148,7 +155,7 @@
def __ne__(self, other):
return self._obj != other
-class _CData(object, metaclass=_CDataMeta):
+class _CData(bufferable, metaclass=_CDataMeta):
""" The most basic object for all ctypes types
"""
_objects = None
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -7,8 +7,7 @@
from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\
array_slice_setitem
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+from __pypy__ import builtinify, newmemoryview
# This cache maps types to pointers to them.
_pointer_type_cache = {}
@@ -134,6 +133,9 @@
def _as_ffi_pointer_(self, ffitype):
return as_ffi_pointer(self, ffitype)
+ def __buffer__(self, flags):
+ mv = memoryview(self.getcontents())
+ return newmemoryview(mv, mv.itemsize, '&' + mv.format, mv.shape)
def _cast_addr(obj, _, tp):
if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()):
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -2,9 +2,9 @@
import _rawffi
from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\
store_reference, ensure_objects, CArgObject
-from _ctypes.array import Array
+from _ctypes.array import Array, get_format_str
from _ctypes.pointer import _Pointer
-import inspect
+import inspect, __pypy__
def names_and_fields(self, _fields_, superclass, anonymous_fields=None):
@@ -176,6 +176,11 @@
class StructOrUnionMeta(_CDataMeta):
def __new__(self, name, cls, typedict):
res = type.__new__(self, name, cls, typedict)
+ if hasattr(res, '_swappedbytes_') and '_fields_' in typedict:
+ # Activate the stdlib ctypes._swapped_meta.__setattr__ to convert fields
+ tmp = res._fields_
+ delattr(res, '_fields_')
+ setattr(res, '_fields_', tmp)
if "_abstract_" in typedict:
return res
cls = cls or (object,)
@@ -253,17 +258,7 @@
or cls is union.Union):
raise TypeError("abstract class")
if hasattr(cls, '_swappedbytes_'):
- fields = [None] * len(cls._fields_)
- for i in range(len(cls._fields_)):
- if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None):
- swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1])
- else:
- swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1])
- if len(cls._fields_[i]) < 3:
- fields[i] = (cls._fields_[i][0], swapped)
- else:
- fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2])
- names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None))
+ names_and_fields(cls, cls._fields_, _CData, cls.__dict__.get('_anonymous_', None))
self = super(_CData, cls).__new__(cls)
if hasattr(cls, '_ffistruct_'):
self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
@@ -303,6 +298,10 @@
def _to_ffi_param(self):
return self._buffer
+ def __buffer__(self, flags):
+ fmt = get_format_str(self)
+ itemsize = type(self)._sizeofinstances()
+ return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt)
class StructureMeta(StructOrUnionMeta):
_is_union = False
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -13,7 +13,6 @@
``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from
the CPython interpreter.
-
Generally available functionality
---------------------------------
@@ -25,13 +24,14 @@
- ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation).
- - ``identity_dict(object)``: A dictionary that considers keys by object identity.
- Distinct objects will have separate entries even if they compare equal.
- All objects can be used as keys, even non-hashable ones --- but avoid using
- immutable objects like integers: two int objects 42 may or may not be
- internally the same object.
+ - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``:
+ create a `memoryview` instance with the data from ``buffer`` and the
+ specified itemsize, format, and optional shape and strides.
- - ``set_debug``
+ - ``bufferable``: a base class that provides a ``__buffer__(self, flags)``
+ method for subclasses to override. This method should return a memoryview
+ instance of the class instance. It is called by the C-API's ``tp_as_buffer.
+ bf_getbuffer``.
- ``builtinify(func)``: To implement at app-level modules that are, in CPython,
implemented in C: this decorator protects a function from being ever bound
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -32,3 +32,10 @@
.. branch: unicode-utf8
Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode
+
+.. branch: newmemoryview-app-level
+
+Since _ctypes is implemented in pure python over libffi, add interfaces and
+methods to support the buffer interface from python. Specifically, add a
+``__pypy__.newmemoryview`` function to create a memoryview and extend the use
+of the PyPy-specific ``__buffer__`` class method.
\ No newline at end of file
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -62,11 +62,18 @@
class PyPyDateTime(MixedModule):
appleveldefs = {}
interpleveldefs = {
- 'dateinterop': 'interp_pypydatetime.W_DateTime_Date',
- 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time',
- 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta',
+ 'dateinterop' : 'interp_pypydatetime.W_DateTime_Date',
+ 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time',
+ 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta',
}
+class PyPyBufferable(MixedModule):
+ appleveldefs = {}
+ interpleveldefs = {
+ 'bufferable': 'interp_buffer.W_Bufferable',
+ }
+
+
class Module(MixedModule):
""" PyPy specific "magic" functions. A lot of them are experimental and
subject to change, many are internal. """
@@ -111,6 +118,7 @@
'fsencode' : 'interp_magic.fsencode',
'fsdecode' : 'interp_magic.fsdecode',
'pyos_inputhook' : 'interp_magic.pyos_inputhook',
+ 'newmemoryview' : 'interp_buffer.newmemoryview',
}
submodules = {
@@ -120,6 +128,7 @@
"intop": IntOpModule,
"os": OsModule,
'_pypydatetime': PyPyDateTime,
+ 'bufferable': PyPyBufferable,
}
def setup_after_space_initialization(self):
diff --git a/pypy/module/__pypy__/interp_buffer.py b/pypy/module/__pypy__/interp_buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/interp_buffer.py
@@ -0,0 +1,100 @@
+#
+# An app-level interface to tp_as_buffer->bf_getbuffer.
+#
+
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.objspace.std.memoryobject import BufferViewND
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef, generic_new_descr
+
+class W_Bufferable(W_Root):
+ def __init__(self, space):
+ pass
+
+ def descr_buffer(self, space, w_flags):
+ if type(self) is W_Bufferable:
+ raise oefmt(space.w_ValueError, "override __buffer__ in a subclass")
+ return space.call_method(self, '__buffer__', w_flags)
+
+ def readbuf_w(self, space):
+ mv = space.call_method(self, '__buffer__', space.newint(0))
+ return mv.buffer_w(space, 0).as_readbuf()
+
+W_Bufferable.typedef = TypeDef("Bufferable", None, None, 'read-write',
+ __doc__ = """a helper class for a app-level class (like _ctypes.Array)
+that want to support tp_as_buffer.bf_getbuffer via a __buffer__ method""",
+ __new__ = generic_new_descr(W_Bufferable),
+ __buffer__ = interp2app(W_Bufferable.descr_buffer),
+)
+
+ at unwrap_spec(itemsize=int, format='text')
+def newmemoryview(space, w_obj, itemsize, format, w_shape=None, w_strides=None):
+ '''
+ newmemoryview(buf, itemsize, format, shape=None, strides=None)
+ '''
+ if not space.isinstance_w(w_obj, space.w_memoryview):
+ raise oefmt(space.w_ValueError, "memoryview expected")
+ # minimal error checking
+ lgt = space.len_w(w_obj)
+ old_size = w_obj.getitemsize()
+ nbytes = lgt * old_size
+ if w_shape:
+ tot = 1
+ shape = []
+ for w_v in space.listview(w_shape):
+ v = space.int_w(w_v)
+ shape.append(v)
+ tot *= v
+ if tot * itemsize != nbytes:
+ raise oefmt(space.w_ValueError,
+ "shape/itemsize %s/%d does not match obj len/itemsize %d/%d",
+ str(shape), itemsize, lgt, old_size)
+ else:
+ if nbytes % itemsize != 0:
+ raise oefmt(space.w_ValueError,
+ "itemsize %d does not match obj len/itemsize %d/%d",
+ itemsize, lgt, old_size)
+ shape = [nbytes / itemsize,]
+ ndim = len(shape)
+ if w_strides:
+ strides = []
+ for w_v in space.listview(w_strides):
+ v = space.int_w(w_v)
+ strides.append(v)
+ if not w_shape and len(strides) != 1:
+ raise oefmt(space.w_ValueError,
+ "strides must have one value if shape not provided")
+ if len(strides) != ndim:
+ raise oefmt(space.w_ValueError,
+ "shape %s does not match strides %s",
+ str(shape), str(strides))
+ else:
+ # start from the right, c-order layout
+ strides = [itemsize] * ndim
+ for v in range(ndim - 2, -1, -1):
+ strides[v] = strides[v + 1] * shape[v + 1]
+ # check that the strides are not too big
+ for i in range(ndim):
+ if strides[i] * shape[i] > nbytes:
+ raise oefmt(space.w_ValueError,
+ "shape %s and strides %s exceed object size %d",
+ shape, strides, nbytes)
+ view = space.buffer_w(w_obj, 0)
+ return space.newmemoryview(FormatBufferViewND(view, itemsize, format, ndim,
+ shape, strides))
+
+class FormatBufferViewND(BufferViewND):
+ _immutable_ = True
+ _attrs_ = ['readonly', 'parent', 'ndim', 'shape', 'strides',
+ 'format', 'itemsize']
+ def __init__(self, parent, itemsize, format, ndim, shape, strides):
+ BufferViewND.__init__(self, parent, ndim, shape, strides)
+ self.format = format
+ self.itemsize = itemsize
+
+ def getformat(self):
+ return self.format
+
+ def getitemsize(self):
+ return self.itemsize
diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_newmemoryview.py
@@ -0,0 +1,32 @@
+
+
+class AppTestMinimal:
+ spaceconfig = dict(usemodules=['__pypy__'])
+
+ def test_newmemoryview(self):
+ from __pypy__ import newmemoryview
+ b = bytearray(12)
+ # The format can be anything, we only verify shape, strides, and itemsize
+ m = newmemoryview(memoryview(b), 2, 'T{<h:a}', shape=(2, 3))
+ assert m.strides == (6, 2)
+ m = newmemoryview(memoryview(b), 2, 'T{<h:a}', shape=(2, 3),
+ strides=(6, 2))
+ assert m.strides == (6, 2)
+ assert m.format == 'T{<h:a}'
+ assert m.itemsize == 2
+
+ def test_bufferable(self):
+ from __pypy__ import bufferable, newmemoryview
+ class B(bufferable.bufferable):
+ def __init__(self):
+ self.data = bytearray('abc')
+
+ def __buffer__(self, flags):
+ return newmemoryview(memoryview(self.data), 1, 'B')
+
+
+ obj = B()
+ buf = buffer(obj)
+ v = obj.data[2]
+ assert ord(buf[2]) == v
+
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -11,7 +11,7 @@
'binascii', 'struct', '_posixsubprocess'))
if sys.platform == 'win32':
- spaceconfig['usemodules'] += ('_rawffi',)
+ spaceconfig['usemodules'] += ('_rawffi', '_cffi_backend')
else:
spaceconfig['usemodules'] += ('fcntl',)
diff --git a/pypy/module/_multiprocessing/test/test_win32.py b/pypy/module/_multiprocessing/test/test_win32.py
--- a/pypy/module/_multiprocessing/test/test_win32.py
+++ b/pypy/module/_multiprocessing/test/test_win32.py
@@ -2,7 +2,7 @@
import sys
class AppTestWin32:
- spaceconfig = dict(usemodules=('_multiprocessing',
+ spaceconfig = dict(usemodules=('_multiprocessing', _cffi_backend',
'signal', '_rawffi', 'binascii'))
def setup_class(cls):
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -30,6 +30,7 @@
from pypy.objspace.std.unicodeobject import encode_object
from pypy.module.__builtin__.descriptor import W_Property
#from pypy.module.micronumpy.base import W_NDimArray
+from pypy.module.__pypy__.interp_buffer import W_Bufferable
from rpython.rlib.entrypoint import entrypoint_lowlevel
from rpython.rlib.rposix import FdValidator
from rpython.rlib.unroll import unrolling_iterable
@@ -731,6 +732,7 @@
'PyMethodDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)',
'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)',
'PyInstanceMethod_Type': 'space.gettypeobject(cpyext.classobject.InstanceMethod.typedef)',
+ 'PyBufferable_Type': 'space.gettypeobject(W_Bufferable.typedef)',
}.items():
register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -43,7 +43,9 @@
fill_Py_buffer(space, w_obj.view, view)
try:
view.c_buf = rffi.cast(rffi.VOIDP, w_obj.view.get_raw_address())
- view.c_obj = make_ref(space, w_userdata)
+ # not used in PyPy to keep something alive,
+ # but some c-extensions check the type without checking for NULL
+ view.c_obj = make_ref(space, space.w_None)
rffi.setintfield(view, 'c_readonly', w_obj.view.readonly)
except ValueError:
w_s = w_obj.descr_tobytes(space)
diff --git a/pypy/module/cpyext/parse/cpyext_memoryobject.h b/pypy/module/cpyext/parse/cpyext_memoryobject.h
--- a/pypy/module/cpyext/parse/cpyext_memoryobject.h
+++ b/pypy/module/cpyext/parse/cpyext_memoryobject.h
@@ -1,6 +1,12 @@
/* The struct is declared here but it shouldn't
be considered public. Don't access those fields directly,
use the functions instead! */
+
+
+/* this is wrong, PyMemoryViewObject should use PyObject_VAR_HEAD, and use
+ ob_data[1] to hold the shapes, strides, and offsets for the view. Then
+ we should use specialized allocators (that break the cpyext model) to
+ allocate ob_data = malloc(sizeof(Py_ssize_t) * view.ndims * 3) */
typedef struct {
PyObject_HEAD
Py_buffer view;
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -52,7 +52,8 @@
/* Py3k buffer interface, adapted for PyPy */
-#define Py_MAX_NDIMS 32
+/* XXX remove this constant, us a PyObject_VAR_HEAD instead */
+#define Py_MAX_NDIMS 36
#define Py_MAX_FMT 128
typedef struct bufferinfo {
void *buf;
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -36,6 +36,23 @@
decref(space, ref)
decref(space, c_memoryview)
+ def test_class_with___buffer__(self, space, api):
+ w_obj = space.appexec([], """():
+ from __pypy__.bufferable import bufferable
+ class B(bufferable):
+ def __init__(self):
+ self.buf = bytearray(10)
+
+ def __buffer__(self, flags):
+ return memoryview(self.buf)
+ return B()""")
+ py_obj = make_ref(space, w_obj)
+ assert py_obj.c_ob_type.c_tp_as_buffer
+ assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer
+ assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getreadbuffer
+ assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getwritebuffer
+
+
class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
def test_fillWithObject(self):
module = self.import_extension('foo', [
More information about the pypy-commit
mailing list