[Python-checkins] bpo-38140: Make dict and weakref offsets opaque for C heap types (#16076)
Dino Viehland
webhook-mailer at python.org
Thu Sep 19 12:29:09 EDT 2019
https://github.com/python/cpython/commit/3368f3c6ae4140a0883e19350e672fd09c9db616
commit: 3368f3c6ae4140a0883e19350e672fd09c9db616
branch: master
author: Eddie Elizondo <eelizondo at fb.com>
committer: Dino Viehland <dinoviehland at gmail.com>
date: 2019-09-19T17:29:05+01:00
summary:
bpo-38140: Make dict and weakref offsets opaque for C heap types (#16076)
* Make dict and weakref offsets opaque for C heap types
* Add news
files:
A Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst
M Doc/c-api/type.rst
M Lib/test/test_capi.py
M Modules/_struct.c
M Modules/_testcapimodule.c
M Objects/typeobject.c
M Parser/asdl_c.py
M Python/Python-ast.c
diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 6416951ac5a9..ad3e022d42ca 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -187,9 +187,8 @@ The following functions and structs are used to create
* :c:member:`~PyTypeObject.tp_cache`
* :c:member:`~PyTypeObject.tp_subclasses`
* :c:member:`~PyTypeObject.tp_weaklist`
+ * :c:member:`~PyTypeObject.tp_vectorcall`
* :c:member:`~PyTypeObject.tp_print`
- * :c:member:`~PyTypeObject.tp_weaklistoffset`
- * :c:member:`~PyTypeObject.tp_dictoffset`
* :c:member:`~PyBufferProcs.bf_getbuffer`
* :c:member:`~PyBufferProcs.bf_releasebuffer`
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index b1d045c53080..79059bc7c964 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -12,6 +12,7 @@
import threading
import time
import unittest
+import weakref
from test import support
from test.support import MISSING_C_DOCSTRINGS
from test.support.script_helper import assert_python_failure, assert_python_ok
@@ -437,6 +438,32 @@ def __del__(self):
# Test that subtype_dealloc decref the newly assigned __class__ only once
self.assertEqual(new_type_refcnt, sys.getrefcount(A))
+ def test_heaptype_with_dict(self):
+ inst = _testcapi.HeapCTypeWithDict()
+ inst.foo = 42
+ self.assertEqual(inst.foo, 42)
+ self.assertEqual(inst.dictobj, inst.__dict__)
+ self.assertEqual(inst.dictobj, {"foo": 42})
+
+ inst = _testcapi.HeapCTypeWithDict()
+ self.assertEqual({}, inst.__dict__)
+
+ def test_heaptype_with_negative_dict(self):
+ inst = _testcapi.HeapCTypeWithNegativeDict()
+ inst.foo = 42
+ self.assertEqual(inst.foo, 42)
+ self.assertEqual(inst.dictobj, inst.__dict__)
+ self.assertEqual(inst.dictobj, {"foo": 42})
+
+ inst = _testcapi.HeapCTypeWithNegativeDict()
+ self.assertEqual({}, inst.__dict__)
+
+ def test_heaptype_with_weakref(self):
+ inst = _testcapi.HeapCTypeWithWeakref()
+ ref = weakref.ref(inst)
+ self.assertEqual(ref(), inst)
+ self.assertEqual(inst.weakreflist, ref)
+
def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self):
subclass_instance = _testcapi.HeapCTypeSubclass()
type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass)
diff --git a/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst b/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst
new file mode 100644
index 000000000000..7df1d25632b3
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst
@@ -0,0 +1,2 @@
+Make dict and weakref offsets opaque for C heap types by passing the offsets
+through PyMemberDef
diff --git a/Modules/_struct.c b/Modules/_struct.c
index c387ae2ffc19..cc536b46a623 100644
--- a/Modules/_struct.c
+++ b/Modules/_struct.c
@@ -2024,7 +2024,7 @@ static struct PyMethodDef s_methods[] = {
};
static PyMemberDef s_members[] = {
- {"__weaklistoffset__", T_NONE, offsetof(PyStructObject, weakreflist), READONLY},
+ {"__weaklistoffset__", T_PYSSIZET, offsetof(PyStructObject, weakreflist), READONLY},
{NULL} /* sentinel */
};
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 4cca784d0148..30a786cd3455 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -6363,6 +6363,106 @@ static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = {
HeapCTypeSubclassWithFinalizer_slots
};
+typedef struct {
+ PyObject_HEAD
+ PyObject *dict;
+} HeapCTypeWithDictObject;
+
+static void
+heapctypewithdict_dealloc(HeapCTypeWithDictObject* self)
+{
+
+ PyTypeObject *tp = Py_TYPE(self);
+ Py_XDECREF(self->dict);
+ PyObject_DEL(self);
+ Py_DECREF(tp);
+}
+
+static PyGetSetDef heapctypewithdict_getsetlist[] = {
+ {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},
+ {NULL} /* Sentinel */
+};
+
+static struct PyMemberDef heapctypewithdict_members[] = {
+ {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
+ {"__dictoffset__", T_PYSSIZET, offsetof(HeapCTypeWithDictObject, dict), READONLY},
+ {NULL} /* Sentinel */
+};
+
+static PyType_Slot HeapCTypeWithDict_slots[] = {
+ {Py_tp_members, heapctypewithdict_members},
+ {Py_tp_getset, heapctypewithdict_getsetlist},
+ {Py_tp_dealloc, heapctypewithdict_dealloc},
+ {0, 0},
+};
+
+static PyType_Spec HeapCTypeWithDict_spec = {
+ "_testcapi.HeapCTypeWithDict",
+ sizeof(HeapCTypeWithDictObject),
+ 0,
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ HeapCTypeWithDict_slots
+};
+
+static struct PyMemberDef heapctypewithnegativedict_members[] = {
+ {"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
+ {"__dictoffset__", T_PYSSIZET, -sizeof(void*), READONLY},
+ {NULL} /* Sentinel */
+};
+
+static PyType_Slot HeapCTypeWithNegativeDict_slots[] = {
+ {Py_tp_members, heapctypewithnegativedict_members},
+ {Py_tp_getset, heapctypewithdict_getsetlist},
+ {Py_tp_dealloc, heapctypewithdict_dealloc},
+ {0, 0},
+};
+
+static PyType_Spec HeapCTypeWithNegativeDict_spec = {
+ "_testcapi.HeapCTypeWithNegativeDict",
+ sizeof(HeapCTypeWithDictObject),
+ 0,
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ HeapCTypeWithNegativeDict_slots
+};
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *weakreflist;
+} HeapCTypeWithWeakrefObject;
+
+static struct PyMemberDef heapctypewithweakref_members[] = {
+ {"weakreflist", T_OBJECT, offsetof(HeapCTypeWithWeakrefObject, weakreflist)},
+ {"__weaklistoffset__", T_PYSSIZET,
+ offsetof(HeapCTypeWithWeakrefObject, weakreflist), READONLY},
+ {NULL} /* Sentinel */
+};
+
+static void
+heapctypewithweakref_dealloc(HeapCTypeWithWeakrefObject* self)
+{
+
+ PyTypeObject *tp = Py_TYPE(self);
+ if (self->weakreflist != NULL)
+ PyObject_ClearWeakRefs((PyObject *) self);
+ Py_XDECREF(self->weakreflist);
+ PyObject_DEL(self);
+ Py_DECREF(tp);
+}
+
+static PyType_Slot HeapCTypeWithWeakref_slots[] = {
+ {Py_tp_members, heapctypewithweakref_members},
+ {Py_tp_dealloc, heapctypewithweakref_dealloc},
+ {0, 0},
+};
+
+static PyType_Spec HeapCTypeWithWeakref_spec = {
+ "_testcapi.HeapCTypeWithWeakref",
+ sizeof(HeapCTypeWithWeakrefObject),
+ 0,
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ HeapCTypeWithWeakref_slots
+};
+
static PyMethodDef meth_instance_methods[] = {
{"meth_varargs", meth_varargs, METH_VARARGS},
{"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS},
@@ -6596,6 +6696,24 @@ PyInit__testcapi(void)
Py_DECREF(subclass_bases);
PyModule_AddObject(m, "HeapCTypeSubclass", HeapCTypeSubclass);
+ PyObject *HeapCTypeWithDict = PyType_FromSpec(&HeapCTypeWithDict_spec);
+ if (HeapCTypeWithDict == NULL) {
+ return NULL;
+ }
+ PyModule_AddObject(m, "HeapCTypeWithDict", HeapCTypeWithDict);
+
+ PyObject *HeapCTypeWithNegativeDict = PyType_FromSpec(&HeapCTypeWithNegativeDict_spec);
+ if (HeapCTypeWithNegativeDict == NULL) {
+ return NULL;
+ }
+ PyModule_AddObject(m, "HeapCTypeWithNegativeDict", HeapCTypeWithNegativeDict);
+
+ PyObject *HeapCTypeWithWeakref = PyType_FromSpec(&HeapCTypeWithWeakref_spec);
+ if (HeapCTypeWithWeakref == NULL) {
+ return NULL;
+ }
+ PyModule_AddObject(m, "HeapCTypeWithWeakref", HeapCTypeWithWeakref);
+
PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass);
if (subclass_with_finalizer_bases == NULL) {
return NULL;
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index dfdac9e2e4f5..94a4da22d679 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -2853,15 +2853,27 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
PyTypeObject *type, *base;
PyType_Slot *slot;
- Py_ssize_t nmembers;
+ Py_ssize_t nmembers, weaklistoffset, dictoffset;
char *s, *res_start;
- nmembers = 0;
+ nmembers = weaklistoffset = dictoffset = 0;
for (slot = spec->slots; slot->slot; slot++) {
if (slot->slot == Py_tp_members) {
nmembers = 0;
for (memb = slot->pfunc; memb->name != NULL; memb++) {
nmembers++;
+ if (strcmp(memb->name, "__weaklistoffset__") == 0) {
+ // The PyMemberDef must be a Py_ssize_t and readonly
+ assert(memb->type == T_PYSSIZET);
+ assert(memb->flags == READONLY);
+ weaklistoffset = memb->offset;
+ }
+ if (strcmp(memb->name, "__dictoffset__") == 0) {
+ // The PyMemberDef must be a Py_ssize_t and readonly
+ assert(memb->type == T_PYSSIZET);
+ assert(memb->flags == READONLY);
+ dictoffset = memb->offset;
+ }
}
}
}
@@ -2990,6 +3002,17 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
res->ht_cached_keys = _PyDict_NewKeysForClass();
}
+ if (weaklistoffset) {
+ type->tp_weaklistoffset = weaklistoffset;
+ if (PyDict_DelItemString((PyObject *)type->tp_dict, "__weaklistoffset__") < 0)
+ goto fail;
+ }
+ if (dictoffset) {
+ type->tp_dictoffset = dictoffset;
+ if (PyDict_DelItemString((PyObject *)type->tp_dict, "__dictoffset__") < 0)
+ goto fail;
+ }
+
/* Set type.__module__ */
s = strrchr(spec->name, '.');
if (s != NULL) {
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 0b72f9ee236c..3a9d4e6b4b92 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -724,6 +724,11 @@ def visitModule(self, mod):
return Py_BuildValue("O()", Py_TYPE(self));
}
+static PyMemberDef ast_type_members[] = {
+ {"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY},
+ {NULL} /* Sentinel */
+};
+
static PyMethodDef ast_type_methods[] = {
{"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
{NULL}
@@ -740,6 +745,7 @@ def visitModule(self, mod):
{Py_tp_setattro, PyObject_GenericSetAttr},
{Py_tp_traverse, ast_traverse},
{Py_tp_clear, ast_clear},
+ {Py_tp_members, ast_type_members},
{Py_tp_methods, ast_type_methods},
{Py_tp_getset, ast_type_getsets},
{Py_tp_init, ast_type_init},
@@ -930,7 +936,6 @@ def visitModule(self, mod):
self.emit("if (init_identifiers() < 0) return 0;", 1)
self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
self.emit("if (!state->AST_type) return 0;", 1)
- self.emit("((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object, dict);", 1)
self.emit("if (add_ast_fields() < 0) return 0;", 1)
for dfn in mod.dfns:
self.visit(dfn)
@@ -1372,6 +1377,7 @@ def main(srcfile, dump_module=False):
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
+ f.write('#include "structmember.h"\n')
f.write('\n')
generate_module_def(f, mod)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 7a38e98c435a..558d9ebea5c7 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -4,6 +4,7 @@
#include "Python.h"
#include "Python-ast.h"
+#include "structmember.h"
typedef struct {
int initialized;
@@ -1216,6 +1217,11 @@ ast_type_reduce(PyObject *self, PyObject *unused)
return Py_BuildValue("O()", Py_TYPE(self));
}
+static PyMemberDef ast_type_members[] = {
+ {"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY},
+ {NULL} /* Sentinel */
+};
+
static PyMethodDef ast_type_methods[] = {
{"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
{NULL}
@@ -1232,6 +1238,7 @@ static PyType_Slot AST_type_slots[] = {
{Py_tp_setattro, PyObject_GenericSetAttr},
{Py_tp_traverse, ast_traverse},
{Py_tp_clear, ast_clear},
+ {Py_tp_members, ast_type_members},
{Py_tp_methods, ast_type_methods},
{Py_tp_getset, ast_type_getsets},
{Py_tp_init, ast_type_init},
@@ -1421,8 +1428,6 @@ static int init_types(void)
if (init_identifiers() < 0) return 0;
state->AST_type = PyType_FromSpec(&AST_type_spec);
if (!state->AST_type) return 0;
- ((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object,
- dict);
if (add_ast_fields() < 0) return 0;
state->mod_type = make_type("mod", state->AST_type, NULL, 0);
if (!state->mod_type) return 0;
More information about the Python-checkins
mailing list