[pypy-commit] pypy default: Merged in cpyext-old-buffers (pull request #452)
arigo
pypy.commits at gmail.com
Sat May 28 03:40:40 EDT 2016
Author: Armin Rigo <armin.rigo at gmail.com>
Branch:
Changeset: r84779:6bc61af961f7
Date: 2016-05-28 09:39 +0200
http://bitbucket.org/pypy/pypy/changeset/6bc61af961f7/
Log: Merged in cpyext-old-buffers (pull request #452)
Generalize cpyext old-style buffers to more than just str/buffer,
add support for mmap.
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -12,7 +12,8 @@
class TypeDef(object):
- def __init__(self, __name, __base=None, __total_ordering__=None, **rawdict):
+ def __init__(self, __name, __base=None, __total_ordering__=None,
+ __buffer=None, **rawdict):
"NOT_RPYTHON: initialization-time only"
self.name = __name
if __base is None:
@@ -22,6 +23,8 @@
else:
bases = [__base]
self.bases = bases
+ assert __buffer in {None, 'read-write', 'read'}, "Unknown value for __buffer"
+ self.buffer = __buffer
self.heaptype = False
self.hasdict = '__dict__' in rawdict
# no __del__: use an RPython _finalize_() method and register_finalizer
diff --git a/pypy/module/cpyext/test/test_abstract.py b/pypy/module/cpyext/test/test_abstract.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_abstract.py
@@ -0,0 +1,106 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+import pytest
+
+class AppTestBufferProtocol(AppTestCpythonExtensionBase):
+ """Tests for the old buffer protocol."""
+
+ def w_get_buffer_support(self):
+ return self.import_extension('buffer_support', [
+ ("charbuffer_as_string", "METH_O",
+ """
+ char *ptr;
+ Py_ssize_t size;
+ if (PyObject_AsCharBuffer(args, &ptr, &size) < 0)
+ return NULL;
+ return PyString_FromStringAndSize(ptr, size);
+ """),
+ ("check_readbuffer", "METH_O",
+ """
+ return PyBool_FromLong(PyObject_CheckReadBuffer(args));
+ """),
+ ("readbuffer_as_string", "METH_O",
+ """
+ const void *ptr;
+ Py_ssize_t size;
+ if (PyObject_AsReadBuffer(args, &ptr, &size) < 0)
+ return NULL;
+ return PyString_FromStringAndSize((char*)ptr, size);
+ """),
+ ("writebuffer_as_string", "METH_O",
+ """
+ void *ptr;
+ Py_ssize_t size;
+ if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0)
+ return NULL;
+ return PyString_FromStringAndSize((char*)ptr, size);
+ """),
+ ("zero_out_writebuffer", "METH_O",
+ """
+ void *ptr;
+ Py_ssize_t size;
+ Py_ssize_t i;
+ if (PyObject_AsWriteBuffer(args, &ptr, &size) < 0)
+ return NULL;
+ for (i = 0; i < size; i++) {
+ ((char*)ptr)[i] = 0;
+ }
+ Py_RETURN_NONE;
+ """),
+ ])
+
+ def test_string(self):
+ buffer_support = self.get_buffer_support()
+
+ s = 'a\0x'
+
+ assert buffer_support.check_readbuffer(s)
+ assert s == buffer_support.readbuffer_as_string(s)
+ assert raises(TypeError, buffer_support.writebuffer_as_string, s)
+ assert s == buffer_support.charbuffer_as_string(s)
+
+ def test_buffer(self):
+ buffer_support = self.get_buffer_support()
+
+ s = 'a\0x'
+ buf = buffer(s)
+
+ assert buffer_support.check_readbuffer(buf)
+ assert s == buffer_support.readbuffer_as_string(buf)
+ assert raises(TypeError, buffer_support.writebuffer_as_string, buf)
+ assert s == buffer_support.charbuffer_as_string(buf)
+
+ def test_mmap(self):
+ import mmap
+ buffer_support = self.get_buffer_support()
+
+ s = 'a\0x'
+ mm = mmap.mmap(-1, 3)
+ mm[:] = s
+
+ assert buffer_support.check_readbuffer(mm)
+ assert s == buffer_support.readbuffer_as_string(mm)
+ assert s == buffer_support.writebuffer_as_string(mm)
+ assert s == buffer_support.charbuffer_as_string(mm)
+
+ s = '\0' * 3
+ buffer_support.zero_out_writebuffer(mm)
+ assert s == ''.join(mm)
+ assert s == buffer_support.readbuffer_as_string(mm)
+ assert s == buffer_support.writebuffer_as_string(mm)
+ assert s == buffer_support.charbuffer_as_string(mm)
+
+ s = '\0' * 3
+ ro_mm = mmap.mmap(-1, 3, access=mmap.ACCESS_READ)
+ assert buffer_support.check_readbuffer(ro_mm)
+ assert s == buffer_support.readbuffer_as_string(ro_mm)
+ assert raises(TypeError, buffer_support.writebuffer_as_string, ro_mm)
+ assert s == buffer_support.charbuffer_as_string(ro_mm)
+
+ def test_nonbuffer(self):
+ # e.g. int
+ buffer_support = self.get_buffer_support()
+
+ assert not buffer_support.check_readbuffer(42)
+ assert raises(TypeError, buffer_support.readbuffer_as_string, 42)
+ assert raises(TypeError, buffer_support.writebuffer_as_string, 42)
+ assert raises(TypeError, buffer_support.charbuffer_as_string, 42)
diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -415,12 +415,15 @@
lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
w_text = space.wrap("text")
- assert api.PyObject_AsCharBuffer(w_text, bufp, lenp) == 0
+ ref = make_ref(space, w_text)
+ prev_refcnt = ref.c_ob_refcnt
+ assert api.PyObject_AsCharBuffer(ref, bufp, lenp) == 0
+ assert ref.c_ob_refcnt == prev_refcnt
assert lenp[0] == 4
assert rffi.charp2str(bufp[0]) == 'text'
-
lltype.free(bufp, flavor='raw')
lltype.free(lenp, flavor='raw')
+ api.Py_DecRef(ref)
def test_intern(self, space, api):
buf = rffi.str2charp("test")
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -136,7 +136,7 @@
"""Base class for all cpyext tests."""
spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
'itertools', 'time', 'binascii',
- 'micronumpy',
+ 'micronumpy', 'mmap'
])
enable_leak_checking = True
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -486,18 +486,45 @@
@cpython_api([PyObject, Py_ssize_tP], lltype.Signed, header=None,
error=CANNOT_FAIL)
-def str_segcount(space, w_obj, ref):
+def bf_segcount(space, w_obj, ref):
if ref:
ref[0] = space.len_w(w_obj)
return 1
@cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
header=None, error=-1)
+def bf_getreadbuffer(space, w_buf, segment, ref):
+ if segment != 0:
+ raise oefmt(space.w_SystemError,
+ "accessing non-existent segment")
+ buf = space.readbuf_w(w_buf)
+ address = buf.get_raw_address()
+ ref[0] = address
+ return len(buf)
+
+ at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
+ header=None, error=-1)
+def bf_getcharbuffer(space, w_buf, segment, ref):
+ return bf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
+
+ at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
+ header=None, error=-1)
+def bf_getwritebuffer(space, w_buf, segment, ref):
+ if segment != 0:
+ raise oefmt(space.w_SystemError,
+ "accessing non-existent segment")
+
+ buf = space.writebuf_w(w_buf)
+ ref[0] = buf.get_raw_address()
+ return len(buf)
+
+ at cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
+ header=None, error=-1)
def str_getreadbuffer(space, w_str, segment, ref):
from pypy.module.cpyext.bytesobject import PyString_AsString
if segment != 0:
- raise oefmt(space.w_SystemError,
- "accessing non-existent string segment")
+ raise OperationError(space.w_SystemError, space.wrap
+ ("accessing non-existent string segment"))
pyref = make_ref(space, w_str)
ref[0] = PyString_AsString(space, pyref)
# Stolen reference: the object has better exist somewhere else
@@ -506,16 +533,8 @@
@cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
header=None, error=-1)
-def str_getcharbuffer(space, w_str, segment, ref):
- from pypy.module.cpyext.bytesobject import PyString_AsString
- if segment != 0:
- raise oefmt(space.w_SystemError,
- "accessing non-existent string segment")
- pyref = make_ref(space, w_str)
- ref[0] = PyString_AsString(space, pyref)
- # Stolen reference: the object has better exist somewhere else
- Py_DecRef(space, pyref)
- return space.len_w(w_str)
+def str_getcharbuffer(space, w_buf, segment, ref):
+ return str_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
@cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
header=None, error=-1)
@@ -523,33 +542,59 @@
from pypy.module.cpyext.bufferobject import PyBufferObject
if segment != 0:
raise oefmt(space.w_SystemError,
- "accessing non-existent string segment")
+ "accessing non-existent buffer segment")
py_buf = rffi.cast(PyBufferObject, pyref)
ref[0] = py_buf.c_b_ptr
- #Py_DecRef(space, pyref)
return py_buf.c_b_size
-def setup_string_buffer_procs(space, pto):
+ at cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
+ header=None, error=-1)
+def buf_getcharbuffer(space, w_buf, segment, ref):
+ return buf_getreadbuffer(space, w_buf, segment, rffi.cast(rffi.VOIDPP, ref))
+
+def setup_buffer_procs(space, w_type, pto):
+ bufspec = w_type.layout.typedef.buffer
+ if bufspec is None:
+ # not a buffer
+ return
c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True)
lltype.render_immortal(c_buf)
- c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype,
- str_segcount.api_func.get_wrapper(space))
- c_buf.c_bf_getreadbuffer = llhelper(str_getreadbuffer.api_func.functype,
- str_getreadbuffer.api_func.get_wrapper(space))
- c_buf.c_bf_getcharbuffer = llhelper(str_getcharbuffer.api_func.functype,
- str_getcharbuffer.api_func.get_wrapper(space))
+ c_buf.c_bf_getsegcount = llhelper(bf_segcount.api_func.functype,
+ bf_segcount.api_func.get_wrapper(space))
+ if space.is_w(w_type, space.w_str):
+ # Special case: str doesn't support get_raw_address(), so we have a
+ # custom get*buffer that instead gives the address of the char* in the
+ # PyStringObject*!
+ c_buf.c_bf_getreadbuffer = llhelper(
+ str_getreadbuffer.api_func.functype,
+ str_getreadbuffer.api_func.get_wrapper(space))
+ c_buf.c_bf_getcharbuffer = llhelper(
+ str_getcharbuffer.api_func.functype,
+ str_getcharbuffer.api_func.get_wrapper(space))
+ elif space.is_w(w_type, space.w_buffer):
+ # Special case: we store a permanent address on the cpyext wrapper,
+ # so we'll reuse that.
+ # Note: we could instead store a permanent address on the buffer object,
+ # and use get_raw_address()
+ c_buf.c_bf_getreadbuffer = llhelper(
+ buf_getreadbuffer.api_func.functype,
+ buf_getreadbuffer.api_func.get_wrapper(space))
+ c_buf.c_bf_getcharbuffer = llhelper(
+ buf_getcharbuffer.api_func.functype,
+ buf_getcharbuffer.api_func.get_wrapper(space))
+ else:
+ # use get_raw_address()
+ c_buf.c_bf_getreadbuffer = llhelper(bf_getreadbuffer.api_func.functype,
+ bf_getreadbuffer.api_func.get_wrapper(space))
+ c_buf.c_bf_getcharbuffer = llhelper(bf_getcharbuffer.api_func.functype,
+ bf_getcharbuffer.api_func.get_wrapper(space))
+ if bufspec == 'read-write':
+ c_buf.c_bf_getwritebuffer = llhelper(
+ bf_getwritebuffer.api_func.functype,
+ bf_getwritebuffer.api_func.get_wrapper(space))
pto.c_tp_as_buffer = c_buf
pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER
-def setup_buffer_buffer_procs(space, pto):
- c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True)
- lltype.render_immortal(c_buf)
- c_buf.c_bf_getsegcount = llhelper(str_segcount.api_func.functype,
- str_segcount.api_func.get_wrapper(space))
- c_buf.c_bf_getreadbuffer = llhelper(buf_getreadbuffer.api_func.functype,
- buf_getreadbuffer.api_func.get_wrapper(space))
- pto.c_tp_as_buffer = c_buf
-
@cpython_api([PyObject], lltype.Void, header=None)
def type_dealloc(space, obj):
from pypy.module.cpyext.object import PyObject_dealloc
@@ -613,10 +658,7 @@
subtype_dealloc.api_func.functype,
subtype_dealloc.api_func.get_wrapper(space))
# buffer protocol
- if space.is_w(w_type, space.w_str):
- setup_string_buffer_procs(space, pto)
- if space.is_w(w_type, space.w_buffer):
- setup_buffer_buffer_procs(space, pto)
+ setup_buffer_procs(space, w_type, pto)
pto.c_tp_free = llhelper(PyObject_Free.api_func.functype,
PyObject_Free.api_func.get_wrapper(space))
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -280,7 +280,7 @@
raise mmap_error(space, e)
return space.wrap(self)
-W_MMap.typedef = TypeDef("mmap.mmap",
+W_MMap.typedef = TypeDef("mmap.mmap", None, None, "read-write",
__new__ = interp2app(mmap),
close = interp2app(W_MMap.close),
read_byte = interp2app(W_MMap.read_byte),
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -135,7 +135,7 @@
return space.wrap(rffi.cast(lltype.Signed, ptr))
W_Buffer.typedef = TypeDef(
- "buffer",
+ "buffer", None, None, "read-write",
__doc__ = """\
buffer(object [, offset[, size]])
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -848,7 +848,7 @@
W_BytesObject.typedef = TypeDef(
- "str", basestring_typedef,
+ "str", basestring_typedef, None, "read",
__new__ = interp2app(W_BytesObject.descr_new),
__doc__ = """str(object='') -> string
More information about the pypy-commit
mailing list