[pypy-commit] cffi default: Port the new_allocator() work from pypy
arigo
noreply at buildbot.pypy.org
Mon Jul 6 17:15:29 CEST 2015
Author: Armin Rigo <arigo at tunes.org>
Branch:
Changeset: r2213:307f969f209f
Date: 2015-07-06 17:16 +0200
http://bitbucket.org/cffi/cffi/changeset/307f969f209f/
Log: Port the new_allocator() work from pypy
diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -238,7 +238,9 @@
typedef struct {
CDataObject head;
- PyObject *origobj, *destructor;
+ Py_ssize_t length; /* same as CDataObject_own_length up to here */
+ PyObject *origobj;
+ PyObject *destructor;
} CDataObject_gcp;
typedef struct {
@@ -281,6 +283,8 @@
# include "wchar_helper.h"
#endif
+typedef PyObject *const cffi_allocator_t[3];
+static cffi_allocator_t default_allocator = { NULL, NULL, NULL };
static PyObject *FFIError;
/************************************************************/
@@ -1643,14 +1647,17 @@
PyObject *origobj = cd->origobj;
cdata_dealloc((CDataObject *)cd);
- result = PyObject_CallFunctionObjArgs(destructor, origobj, NULL);
- if (result != NULL) {
- Py_DECREF(result);
- }
- else {
- _my_PyErr_WriteUnraisable("From callback for ffi.gc ", origobj, NULL);
- }
- Py_DECREF(destructor);
+ if (destructor != NULL) {
+ result = PyObject_CallFunctionObjArgs(destructor, origobj, NULL);
+ if (result != NULL) {
+ Py_DECREF(result);
+ }
+ else {
+ _my_PyErr_WriteUnraisable("From callback for ffi.gc ",
+ origobj, NULL);
+ }
+ Py_DECREF(destructor);
+ }
Py_DECREF(origobj);
}
@@ -2918,7 +2925,79 @@
return (PyObject *)cd;
}
-static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init)
+static CDataObject *allocate_gcp_object(CDataObject *origobj,
+ CTypeDescrObject *ct,
+ PyObject *destructor)
+{
+ CDataObject_gcp *cd = PyObject_GC_New(CDataObject_gcp, &CDataGCP_Type);
+ if (cd == NULL)
+ return NULL;
+
+ Py_XINCREF(destructor);
+ Py_INCREF(origobj);
+ Py_INCREF(ct);
+ cd->head.c_data = origobj->c_data;
+ cd->head.c_type = ct;
+ cd->head.c_weakreflist = NULL;
+ cd->origobj = (PyObject *)origobj;
+ cd->destructor = destructor;
+
+ PyObject_GC_Track(cd);
+ return (CDataObject *)cd;
+}
+
+static CDataObject *allocate_with_allocator(Py_ssize_t basesize,
+ Py_ssize_t datasize,
+ CTypeDescrObject *ct,
+ cffi_allocator_t allocator)
+{
+ CDataObject *cd;
+ PyObject *my_alloc = allocator[0];
+ PyObject *my_free = allocator[1];
+ PyObject *dont_clear_after_alloc = allocator[2];
+
+ if (my_alloc == NULL) { /* alloc */
+ cd = allocate_owning_object(basesize + datasize, ct);
+ if (cd == NULL)
+ return NULL;
+ cd->c_data = ((char *)cd) + basesize;
+ }
+ else {
+ PyObject *res = PyObject_CallFunction(my_alloc, "n", datasize);
+ if (res == NULL)
+ return NULL;
+
+ if (!CData_Check(res)) {
+ PyErr_Format(PyExc_TypeError,
+ "alloc() must return a cdata object (got %.200s)",
+ Py_TYPE(res)->tp_name);
+ Py_DECREF(res);
+ return NULL;
+ }
+ cd = (CDataObject *)res;
+ if (!(cd->c_type->ct_flags & (CT_POINTER|CT_ARRAY))) {
+ PyErr_Format(PyExc_TypeError,
+ "alloc() must return a cdata pointer, not '%s'",
+ cd->c_type->ct_name);
+ Py_DECREF(res);
+ return NULL;
+ }
+ if (!cd->c_data) {
+ PyErr_SetString(PyExc_MemoryError, "alloc() returned NULL");
+ Py_DECREF(res);
+ return NULL;
+ }
+
+ cd = allocate_gcp_object(cd, ct, my_free);
+ Py_DECREF(res);
+ }
+ if (dont_clear_after_alloc == NULL)
+ memset(cd->c_data, 0, datasize);
+ return cd;
+}
+
+static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init,
+ cffi_allocator_t allocator)
{
CTypeDescrObject *ctitem;
CDataObject *cd;
@@ -2982,7 +3061,8 @@
having a strong reference to it */
CDataObject *cds;
- cds = allocate_owning_object(dataoffset + datasize, ct->ct_itemdescr);
+ cds = allocate_with_allocator(dataoffset, datasize, ct->ct_itemdescr,
+ allocator);
if (cds == NULL)
return NULL;
@@ -2995,19 +3075,17 @@
((CDataObject_own_structptr *)cd)->structobj = (PyObject *)cds;
assert(explicitlength < 0);
- cds->c_data = cd->c_data = ((char *)cds) + dataoffset;
+ cd->c_data = cds->c_data;
}
else {
- cd = allocate_owning_object(dataoffset + datasize, ct);
+ cd = allocate_with_allocator(dataoffset, datasize, ct, allocator);
if (cd == NULL)
return NULL;
- cd->c_data = ((char *)cd) + dataoffset;
if (explicitlength >= 0)
((CDataObject_own_length*)cd)->length = explicitlength;
}
- memset(cd->c_data, 0, datasize);
if (init != Py_None) {
if (convert_from_object(cd->c_data,
(ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct, init) < 0) {
@@ -3024,7 +3102,7 @@
PyObject *init = Py_None;
if (!PyArg_ParseTuple(args, "O!|O:newp", &CTypeDescr_Type, &ct, &init))
return NULL;
- return direct_newp(ct, init);
+ return direct_newp(ct, init, default_allocator);
}
static int
@@ -5630,7 +5708,7 @@
static PyObject *b_gcp(PyObject *self, PyObject *args, PyObject *kwds)
{
- CDataObject_gcp *cd;
+ CDataObject *cd;
CDataObject *origobj;
PyObject *destructor;
static char *keywords[] = {"cdata", "destructor", NULL};
@@ -5639,20 +5717,7 @@
&CData_Type, &origobj, &destructor))
return NULL;
- cd = PyObject_GC_New(CDataObject_gcp, &CDataGCP_Type);
- if (cd == NULL)
- return NULL;
-
- Py_INCREF(destructor);
- Py_INCREF(origobj);
- Py_INCREF(origobj->c_type);
- cd->head.c_data = origobj->c_data;
- cd->head.c_type = origobj->c_type;
- cd->head.c_weakreflist = NULL;
- cd->origobj = (PyObject *)origobj;
- cd->destructor = destructor;
-
- PyObject_GC_Track(cd);
+ cd = allocate_gcp_object(origobj, origobj->c_type, destructor);
return (PyObject *)cd;
}
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -331,7 +331,8 @@
"used for a longer time. Be careful about that when copying the\n"
"pointer to the memory somewhere else, e.g. into another structure.");
-static PyObject *ffi_new(FFIObject *self, PyObject *args, PyObject *kwds)
+static PyObject *_ffi_new(FFIObject *self, PyObject *args, PyObject *kwds,
+ cffi_allocator_t allocator)
{
CTypeDescrObject *ct;
PyObject *arg, *init = Py_None;
@@ -344,7 +345,70 @@
if (ct == NULL)
return NULL;
- return direct_newp(ct, init);
+ return direct_newp(ct, init, allocator);
+}
+
+static PyObject *ffi_new(FFIObject *self, PyObject *args, PyObject *kwds)
+{
+ return _ffi_new(self, args, kwds, default_allocator);
+}
+
+static PyObject *_ffi_new_with_allocator(PyObject *allocator, PyObject *args,
+ PyObject *kwds)
+{
+ return _ffi_new((FFIObject *)PyTuple_GET_ITEM(allocator, 0),
+ args, kwds,
+ &PyTuple_GET_ITEM(allocator, 1));
+}
+
+PyDoc_STRVAR(ffi_new_allocator_doc, "XXX");
+
+static PyObject *ffi_new_allocator(FFIObject *self, PyObject *args,
+ PyObject *kwds)
+{
+ PyObject *allocator, *result;
+ PyObject *my_alloc = Py_None, *my_free = Py_None;
+ int should_clear_after_alloc = 1;
+ static char *keywords[] = {"alloc", "free", "should_clear_after_alloc",
+ NULL};
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi:new_allocator", keywords,
+ &my_alloc, &my_free,
+ &should_clear_after_alloc))
+ return NULL;
+
+ if (my_alloc == Py_None && my_free != Py_None) {
+ PyErr_SetString(PyExc_TypeError, "cannot pass 'free' without 'alloc'");
+ return NULL;
+ }
+
+ allocator = PyTuple_New(4);
+ if (allocator == NULL)
+ return NULL;
+
+ Py_INCREF(self);
+ PyTuple_SET_ITEM(allocator, 0, (PyObject *)self);
+
+ if (my_alloc != Py_None) {
+ Py_INCREF(my_alloc);
+ PyTuple_SET_ITEM(allocator, 1, my_alloc);
+ }
+ if (my_free != Py_None) {
+ Py_INCREF(my_free);
+ PyTuple_SET_ITEM(allocator, 2, my_free);
+ }
+ if (!should_clear_after_alloc) {
+ Py_INCREF(Py_True);
+ PyTuple_SET_ITEM(allocator, 3, Py_True); /* dont_clear_after_alloc */
+ }
+
+ {
+ static PyMethodDef md = {"allocator",
+ (PyCFunction)_ffi_new_with_allocator,
+ METH_VARARGS | METH_KEYWORDS};
+ result = PyCFunction_New(&md, allocator);
+ }
+ Py_DECREF(allocator);
+ return result;
}
PyDoc_STRVAR(ffi_cast_doc,
@@ -805,6 +869,7 @@
#endif
{"integer_const",(PyCFunction)ffi_int_const,METH_VKW, ffi_int_const_doc},
{"new", (PyCFunction)ffi_new, METH_VKW, ffi_new_doc},
+{"new_allocator",(PyCFunction)ffi_new_allocator,METH_VKW,ffi_new_allocator_doc},
{"new_handle", (PyCFunction)ffi_new_handle, METH_O, ffi_new_handle_doc},
{"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc},
{"sizeof", (PyCFunction)ffi_sizeof, METH_O, ffi_sizeof_doc},
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -236,6 +236,30 @@
cdecl = self._typeof(cdecl)
return self._backend.newp(cdecl, init)
+ def new_allocator(self, alloc=None, free=None,
+ should_clear_after_alloc=True):
+ """Return a new allocator, i.e. a function that behaves like ffi.new()
+ but uses the provided low-level 'alloc' and 'free' functions.
+
+ 'alloc' is called with the size as argument. If it returns NULL, a
+ MemoryError is raised. 'free' is called with the result of 'alloc'
+ as argument. Both can be either Python function or directly C
+ functions. If 'free' is None, then no free function is called.
+ If both 'alloc' and 'free' are None, the default is used.
+
+ If 'should_clear_after_alloc' is set to False, then the memory
+ returned by 'alloc' is assumed to be already cleared (or you are
+ fine with garbage); otherwise CFFI will clear it.
+ """
+ compiled_ffi = self._backend.FFI()
+ allocator = compiled_ffi.new_allocator(alloc, free,
+ should_clear_after_alloc)
+ def allocate(cdecl, init=None):
+ if isinstance(cdecl, basestring):
+ cdecl = self._typeof(cdecl)
+ return allocator(cdecl, init)
+ return allocate
+
def cast(self, cdecl, source):
"""Similar to a C cast: returns an instance of the named C
type initialized with the given 'source'. The source is
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -57,6 +57,77 @@
assert tb.tb_frame.f_code.co_name == 'cb'
assert tb.tb_frame.f_locals['n'] == 234
+ def test_ffi_new_allocator_2(self):
+ ffi = FFI(backend=self.Backend())
+ seen = []
+ def myalloc(size):
+ seen.append(size)
+ return ffi.new("char[]", "X" * size)
+ def myfree(raw):
+ seen.append(raw)
+ alloc1 = ffi.new_allocator(myalloc, myfree)
+ alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+ should_clear_after_alloc=False)
+ p1 = alloc1("int[10]")
+ p2 = alloc2("int[]", 10)
+ assert seen == [40, 40]
+ assert ffi.typeof(p1) == ffi.typeof("int[10]")
+ assert ffi.sizeof(p1) == 40
+ assert ffi.typeof(p2) == ffi.typeof("int[]")
+ assert ffi.sizeof(p2) == 40
+ assert p1[5] == 0
+ assert p2[6] == ord('X') * 0x01010101
+ raw1 = ffi.cast("char *", p1)
+ raw2 = ffi.cast("char *", p2)
+ del p1, p2
+ retries = 0
+ while len(seen) != 4:
+ retries += 1
+ assert retries <= 5
+ import gc; gc.collect()
+ assert seen == [40, 40, raw1, raw2]
+ assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+ assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+ def test_ffi_new_allocator_3(self):
+ ffi = FFI(backend=self.Backend())
+ seen = []
+ def myalloc(size):
+ seen.append(size)
+ return ffi.new("char[]", "X" * size)
+ alloc1 = ffi.new_allocator(myalloc) # no 'free'
+ p1 = alloc1("int[10]")
+ assert seen == [40]
+ assert ffi.typeof(p1) == ffi.typeof("int[10]")
+ assert ffi.sizeof(p1) == 40
+ assert p1[5] == 0
+
+ def test_ffi_new_allocator_4(self):
+ ffi = FFI(backend=self.Backend())
+ py.test.raises(TypeError, ffi.new_allocator, free=lambda x: None)
+ #
+ def myalloc2(size):
+ raise LookupError
+ alloc2 = ffi.new_allocator(myalloc2)
+ py.test.raises(LookupError, alloc2, "int[5]")
+ #
+ def myalloc3(size):
+ return 42
+ alloc3 = ffi.new_allocator(myalloc3)
+ e = py.test.raises(TypeError, alloc3, "int[5]")
+ assert str(e.value) == "alloc() must return a cdata object (got int)"
+ #
+ def myalloc4(size):
+ return ffi.cast("int", 42)
+ alloc4 = ffi.new_allocator(myalloc4)
+ e = py.test.raises(TypeError, alloc4, "int[5]")
+ assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+ #
+ def myalloc5(size):
+ return ffi.NULL
+ alloc5 = ffi.new_allocator(myalloc5)
+ py.test.raises(MemoryError, alloc5, "int[5]")
+
class TestBitfield:
def check(self, source, expected_ofs_y, expected_align, expected_size):
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -224,3 +224,95 @@
n = (1 << 29) + 42
code, message = ffi.getwinerror(code=n)
assert code == n
+
+def test_ffi_new_allocator_1():
+ ffi = _cffi1_backend.FFI()
+ alloc1 = ffi.new_allocator()
+ alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+ for retry in range(100):
+ p1 = alloc1("int[10]")
+ p2 = alloc2("int[10]")
+ combination = 0
+ for i in range(10):
+ assert p1[i] == 0
+ combination |= p2[i]
+ p1[i] = -42
+ p2[i] = -43
+ if combination != 0:
+ break
+ del p1, p2
+ import gc; gc.collect()
+ else:
+ raise AssertionError("cannot seem to get an int[10] not "
+ "completely cleared")
+
+def test_ffi_new_allocator_2():
+ ffi = _cffi1_backend.FFI()
+ seen = []
+ def myalloc(size):
+ seen.append(size)
+ return ffi.new("char[]", "X" * size)
+ def myfree(raw):
+ seen.append(raw)
+ alloc1 = ffi.new_allocator(myalloc, myfree)
+ alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+ should_clear_after_alloc=False)
+ p1 = alloc1("int[10]")
+ p2 = alloc2("int[]", 10)
+ assert seen == [40, 40]
+ assert ffi.typeof(p1) == ffi.typeof("int[10]")
+ assert ffi.sizeof(p1) == 40
+ assert ffi.typeof(p2) == ffi.typeof("int[]")
+ assert ffi.sizeof(p2) == 40
+ assert p1[5] == 0
+ assert p2[6] == ord('X') * 0x01010101
+ raw1 = ffi.cast("char *", p1)
+ raw2 = ffi.cast("char *", p2)
+ del p1, p2
+ retries = 0
+ while len(seen) != 4:
+ retries += 1
+ assert retries <= 5
+ import gc; gc.collect()
+ assert seen == [40, 40, raw1, raw2]
+ assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+ assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+def test_ffi_new_allocator_3():
+ ffi = _cffi1_backend.FFI()
+ seen = []
+ def myalloc(size):
+ seen.append(size)
+ return ffi.new("char[]", "X" * size)
+ alloc1 = ffi.new_allocator(myalloc) # no 'free'
+ p1 = alloc1("int[10]")
+ assert seen == [40]
+ assert ffi.typeof(p1) == ffi.typeof("int[10]")
+ assert ffi.sizeof(p1) == 40
+ assert p1[5] == 0
+
+def test_ffi_new_allocator_4():
+ ffi = _cffi1_backend.FFI()
+ py.test.raises(TypeError, ffi.new_allocator, free=lambda x: None)
+ #
+ def myalloc2(size):
+ raise LookupError
+ alloc2 = ffi.new_allocator(myalloc2)
+ py.test.raises(LookupError, alloc2, "int[5]")
+ #
+ def myalloc3(size):
+ return 42
+ alloc3 = ffi.new_allocator(myalloc3)
+ e = py.test.raises(TypeError, alloc3, "int[5]")
+ assert str(e.value) == "alloc() must return a cdata object (got int)"
+ #
+ def myalloc4(size):
+ return ffi.cast("int", 42)
+ alloc4 = ffi.new_allocator(myalloc4)
+ e = py.test.raises(TypeError, alloc4, "int[5]")
+ assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+ #
+ def myalloc5(size):
+ return ffi.NULL
+ alloc5 = ffi.new_allocator(myalloc5)
+ py.test.raises(MemoryError, alloc5, "int[5]")
More information about the pypy-commit
mailing list