[Python-checkins] gh-99741: Implement Multi-Phase Init for the _xxsubinterpreters Module (gh-99742)
ericsnowcurrently
webhook-mailer at python.org
Mon Dec 5 15:40:30 EST 2022
https://github.com/python/cpython/commit/530cc9dbb61df55b83f0219d2282980c9cb1cbd8
commit: 530cc9dbb61df55b83f0219d2282980c9cb1cbd8
branch: main
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: ericsnowcurrently <ericsnowcurrently at gmail.com>
date: 2022-12-05T13:40:20-07:00
summary:
gh-99741: Implement Multi-Phase Init for the _xxsubinterpreters Module (gh-99742)
_xxsubinterpreters is an internal module used for testing.
https://github.com/python/cpython/issues/99741
files:
A Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst
M Include/cpython/pystate.h
M Lib/test/test__xxsubinterpreters.py
M Modules/_xxsubinterpretersmodule.c
M Python/pystate.c
M Tools/c-analyzer/cpython/globals-to-fix.tsv
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 0f56b1f21905..0117c23f518c 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -353,6 +353,9 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
// is necessary to pass safely between interpreters in the same process.
typedef struct _xid _PyCrossInterpreterData;
+typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *);
+typedef void (*xid_freefunc)(void *);
+
struct _xid {
// data is the cross-interpreter-safe derivation of a Python object
// (see _PyObject_GetCrossInterpreterData). It will be NULL if the
@@ -379,7 +382,7 @@ struct _xid {
// interpreter given the data. The resulting object (a new
// reference) will be equivalent to the original object. This field
// is required.
- PyObject *(*new_object)(_PyCrossInterpreterData *);
+ xid_newobjectfunc new_object;
// free is called when the data is released. If it is NULL then
// nothing will be done to free the data. For some types this is
// okay (e.g. bytes) and for those types this field should be set
@@ -389,9 +392,20 @@ struct _xid {
// leak. In that case, at the very least this field should be set
// to PyMem_RawFree (the default if not explicitly set to NULL).
// The call will happen with the original interpreter activated.
- void (*free)(void *);
+ xid_freefunc free;
};
+PyAPI_FUNC(void) _PyCrossInterpreterData_Init(
+ _PyCrossInterpreterData *data,
+ PyInterpreterState *interp, void *shared, PyObject *obj,
+ xid_newobjectfunc new_object);
+PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize(
+ _PyCrossInterpreterData *,
+ PyInterpreterState *interp, const size_t, PyObject *,
+ xid_newobjectfunc);
+PyAPI_FUNC(void) _PyCrossInterpreterData_Clear(
+ PyInterpreterState *, _PyCrossInterpreterData *);
+
PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *);
PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *);
PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *);
@@ -400,7 +414,8 @@ PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
/* cross-interpreter data registry */
-typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *);
+typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *,
+ _PyCrossInterpreterData *);
PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc);
PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *);
diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py
index f274b637d947..18900bb9f716 100644
--- a/Lib/test/test__xxsubinterpreters.py
+++ b/Lib/test/test__xxsubinterpreters.py
@@ -295,8 +295,8 @@ def clean_up_channels():
class TestBase(unittest.TestCase):
def tearDown(self):
- clean_up_interpreters()
clean_up_channels()
+ clean_up_interpreters()
##################################
@@ -411,6 +411,15 @@ def test_non_shareable_int(self):
interpreters.channel_send(self.cid, i)
+class ModuleTests(TestBase):
+
+ def test_import_in_interpreter(self):
+ _run_output(
+ interpreters.create(),
+ 'import _xxsubinterpreters as _interpreters',
+ )
+
+
##################################
# interpreter tests
diff --git a/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst b/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst
new file mode 100644
index 000000000000..791726b52bfb
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst
@@ -0,0 +1,2 @@
+We've implemented multi-phase init (PEP 489/630/687)
+for the internal (for testing) _xxsubinterpreters module.
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index 3e064ca8c0b3..d7d7fcaf00a8 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -96,18 +96,20 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base)
add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE)
static PyTypeObject *
-add_new_type(PyObject *mod, PyTypeObject *cls, crossinterpdatafunc shared)
+add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared)
{
- if (PyType_Ready(cls) != 0) {
+ PyTypeObject *cls = (PyTypeObject *)PyType_FromMetaclass(
+ NULL, mod, spec, NULL);
+ if (cls == NULL) {
return NULL;
}
- if (PyModule_AddType(mod, cls) != 0) {
- // XXX When this becomes a heap type, we need to decref here.
+ if (PyModule_AddType(mod, cls) < 0) {
+ Py_DECREF(cls);
return NULL;
}
if (shared != NULL) {
if (_PyCrossInterpreterData_RegisterClass(cls, shared)) {
- // XXX When this becomes a heap type, we need to decref here.
+ Py_DECREF(cls);
return NULL;
}
}
@@ -135,12 +137,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc)
* shareable types are all very basic, with no GC.
* That said, it becomes much messier once interpreters
* no longer share a GIL, so this needs to be fixed before then. */
- // We do what _release_xidata() does in pystate.c.
- if (data->free != NULL) {
- data->free(data->data);
- data->data = NULL;
- }
- Py_CLEAR(data->obj);
+ _PyCrossInterpreterData_Clear(NULL, data);
if (ignoreexc) {
// XXX Emit a warning?
PyErr_Clear();
@@ -153,6 +150,69 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc)
}
+/* module state *************************************************************/
+
+typedef struct {
+ PyTypeObject *ChannelIDType;
+
+ /* interpreter exceptions */
+ PyObject *RunFailedError;
+
+ /* channel exceptions */
+ PyObject *ChannelError;
+ PyObject *ChannelNotFoundError;
+ PyObject *ChannelClosedError;
+ PyObject *ChannelEmptyError;
+ PyObject *ChannelNotEmptyError;
+} module_state;
+
+static inline module_state *
+get_module_state(PyObject *mod)
+{
+ assert(mod != NULL);
+ module_state *state = PyModule_GetState(mod);
+ assert(state != NULL);
+ return state;
+}
+
+static int
+traverse_module_state(module_state *state, visitproc visit, void *arg)
+{
+ /* heap types */
+ Py_VISIT(state->ChannelIDType);
+
+ /* interpreter exceptions */
+ Py_VISIT(state->RunFailedError);
+
+ /* channel exceptions */
+ Py_VISIT(state->ChannelError);
+ Py_VISIT(state->ChannelNotFoundError);
+ Py_VISIT(state->ChannelClosedError);
+ Py_VISIT(state->ChannelEmptyError);
+ Py_VISIT(state->ChannelNotEmptyError);
+ return 0;
+}
+
+static int
+clear_module_state(module_state *state)
+{
+ /* heap types */
+ (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType);
+ Py_CLEAR(state->ChannelIDType);
+
+ /* interpreter exceptions */
+ Py_CLEAR(state->RunFailedError);
+
+ /* channel exceptions */
+ Py_CLEAR(state->ChannelError);
+ Py_CLEAR(state->ChannelNotFoundError);
+ Py_CLEAR(state->ChannelClosedError);
+ Py_CLEAR(state->ChannelEmptyError);
+ Py_CLEAR(state->ChannelNotEmptyError);
+ return 0;
+}
+
+
/* data-sharing-specific code ***********************************************/
struct _sharednsitem {
@@ -420,82 +480,80 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass)
#define ERR_CHANNELS_MUTEX_INIT -8
#define ERR_NO_NEXT_CHANNEL_ID -9
-static PyObject *ChannelError;
-static PyObject *ChannelNotFoundError;
-static PyObject *ChannelClosedError;
-static PyObject *ChannelEmptyError;
-static PyObject *ChannelNotEmptyError;
-
static int
channel_exceptions_init(PyObject *mod)
{
- // XXX Move the exceptions into per-module memory?
+ module_state *state = get_module_state(mod);
+ if (state == NULL) {
+ return -1;
+ }
#define ADD(NAME, BASE) \
do { \
- if (NAME == NULL) { \
- NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \
- if (NAME == NULL) { \
- return -1; \
- } \
+ assert(state->NAME == NULL); \
+ state->NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \
+ if (state->NAME == NULL) { \
+ return -1; \
} \
} while (0)
// A channel-related operation failed.
ADD(ChannelError, PyExc_RuntimeError);
// An operation tried to use a channel that doesn't exist.
- ADD(ChannelNotFoundError, ChannelError);
+ ADD(ChannelNotFoundError, state->ChannelError);
// An operation tried to use a closed channel.
- ADD(ChannelClosedError, ChannelError);
+ ADD(ChannelClosedError, state->ChannelError);
// An operation tried to pop from an empty channel.
- ADD(ChannelEmptyError, ChannelError);
+ ADD(ChannelEmptyError, state->ChannelError);
// An operation tried to close a non-empty channel.
- ADD(ChannelNotEmptyError, ChannelError);
+ ADD(ChannelNotEmptyError, state->ChannelError);
#undef ADD
return 0;
}
static int
-handle_channel_error(int err, PyObject *Py_UNUSED(mod), int64_t cid)
+handle_channel_error(int err, PyObject *mod, int64_t cid)
{
if (err == 0) {
assert(!PyErr_Occurred());
return 0;
}
assert(err < 0);
+ module_state *state = get_module_state(mod);
+ assert(state != NULL);
if (err == ERR_CHANNEL_NOT_FOUND) {
- PyErr_Format(ChannelNotFoundError,
+ PyErr_Format(state->ChannelNotFoundError,
"channel %" PRId64 " not found", cid);
}
else if (err == ERR_CHANNEL_CLOSED) {
- PyErr_Format(ChannelClosedError,
+ PyErr_Format(state->ChannelClosedError,
"channel %" PRId64 " is closed", cid);
}
else if (err == ERR_CHANNEL_INTERP_CLOSED) {
- PyErr_Format(ChannelClosedError,
+ PyErr_Format(state->ChannelClosedError,
"channel %" PRId64 " is already closed", cid);
}
else if (err == ERR_CHANNEL_EMPTY) {
- PyErr_Format(ChannelEmptyError,
+ PyErr_Format(state->ChannelEmptyError,
"channel %" PRId64 " is empty", cid);
}
else if (err == ERR_CHANNEL_NOT_EMPTY) {
- PyErr_Format(ChannelNotEmptyError,
+ PyErr_Format(state->ChannelNotEmptyError,
"channel %" PRId64 " may not be closed "
"if not empty (try force=True)",
cid);
}
else if (err == ERR_CHANNEL_MUTEX_INIT) {
- PyErr_SetString(ChannelError,
+ PyErr_SetString(state->ChannelError,
"can't initialize mutex for new channel");
}
else if (err == ERR_CHANNELS_MUTEX_INIT) {
- PyErr_SetString(ChannelError,
+ PyErr_SetString(state->ChannelError,
"can't initialize mutex for channel management");
}
else if (err == ERR_NO_NEXT_CHANNEL_ID) {
- PyErr_SetString(ChannelError,
+ PyErr_SetString(state->ChannelError,
"failed to get a channel ID");
}
else {
@@ -1604,8 +1662,6 @@ _channel_is_associated(_channels *channels, int64_t cid, int64_t interp,
/* ChannelID class */
-static PyTypeObject ChannelIDType;
-
typedef struct channelid {
PyObject_HEAD
int64_t id;
@@ -1624,7 +1680,9 @@ channel_id_converter(PyObject *arg, void *ptr)
{
int64_t cid;
struct channel_id_converter_data *data = ptr;
- if (PyObject_TypeCheck(arg, &ChannelIDType)) {
+ module_state *state = get_module_state(data->module);
+ assert(state != NULL);
+ if (PyObject_TypeCheck(arg, state->ChannelIDType)) {
cid = ((channelid *)arg)->id;
}
else if (PyIndex_Check(arg)) {
@@ -1731,11 +1789,20 @@ _channelid_new(PyObject *mod, PyTypeObject *cls,
}
static void
-channelid_dealloc(PyObject *v)
+channelid_dealloc(PyObject *self)
{
- int64_t cid = ((channelid *)v)->id;
- _channels *channels = ((channelid *)v)->channels;
- Py_TYPE(v)->tp_free(v);
+ int64_t cid = ((channelid *)self)->id;
+ _channels *channels = ((channelid *)self)->channels;
+
+ PyTypeObject *tp = Py_TYPE(self);
+ tp->tp_free(self);
+ /* "Instances of heap-allocated types hold a reference to their type."
+ * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol
+ * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse
+ */
+ // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse,
+ // like we do for _abc._abc_data?
+ Py_DECREF(tp);
_channels_drop_id_object(channels, cid);
}
@@ -1774,11 +1841,6 @@ channelid_int(PyObject *self)
return PyLong_FromLongLong(cid->id);
}
-static PyNumberMethods channelid_as_number = {
- .nb_int = (unaryfunc)channelid_int, /* nb_int */
- .nb_index = (unaryfunc)channelid_int, /* nb_index */
-};
-
static Py_hash_t
channelid_hash(PyObject *self)
{
@@ -1804,15 +1866,19 @@ channelid_richcompare(PyObject *self, PyObject *other, int op)
if (mod == NULL) {
return NULL;
}
+ module_state *state = get_module_state(mod);
+ if (state == NULL) {
+ goto done;
+ }
- if (!PyObject_TypeCheck(self, &ChannelIDType)) {
+ if (!PyObject_TypeCheck(self, state->ChannelIDType)) {
res = Py_NewRef(Py_NotImplemented);
goto done;
}
channelid *cid = (channelid *)self;
int equal;
- if (PyObject_TypeCheck(other, &ChannelIDType)) {
+ if (PyObject_TypeCheck(other, state->ChannelIDType)) {
channelid *othercid = (channelid *)other;
equal = (cid->end == othercid->end) && (cid->id == othercid->id);
}
@@ -1892,10 +1958,14 @@ _channelid_from_xid(_PyCrossInterpreterData *data)
if (mod == NULL) {
return NULL;
}
+ module_state *state = get_module_state(mod);
+ if (state == NULL) {
+ return NULL;
+ }
// Note that we do not preserve the "resolve" flag.
PyObject *cid = NULL;
- int err = newchannelid(&ChannelIDType, xid->id, xid->end,
+ int err = newchannelid(state->ChannelIDType, xid->id, xid->end,
_global_channels(), 0, 0,
(channelid **)&cid);
if (err != 0) {
@@ -1926,20 +1996,20 @@ _channelid_from_xid(_PyCrossInterpreterData *data)
}
static int
-_channelid_shared(PyObject *obj, _PyCrossInterpreterData *data)
-{
- struct _channelid_xid *xid = PyMem_NEW(struct _channelid_xid, 1);
- if (xid == NULL) {
+_channelid_shared(PyThreadState *tstate, PyObject *obj,
+ _PyCrossInterpreterData *data)
+{
+ if (_PyCrossInterpreterData_InitWithSize(
+ data, tstate->interp, sizeof(struct _channelid_xid), obj,
+ _channelid_from_xid
+ ) < 0)
+ {
return -1;
}
+ struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
xid->id = ((channelid *)obj)->id;
xid->end = ((channelid *)obj)->end;
xid->resolve = ((channelid *)obj)->resolve;
-
- data->data = xid;
- data->obj = Py_NewRef(obj);
- data->new_object = _channelid_from_xid;
- data->free = PyMem_Free;
return 0;
}
@@ -1992,61 +2062,45 @@ static PyGetSetDef channelid_getsets[] = {
PyDoc_STRVAR(channelid_doc,
"A channel ID identifies a channel and may be used as an int.");
-static PyTypeObject ChannelIDType = {
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
- "_xxsubinterpreters.ChannelID", /* tp_name */
- sizeof(channelid), /* tp_basicsize */
- 0, /* tp_itemsize */
- (destructor)channelid_dealloc, /* tp_dealloc */
- 0, /* tp_vectorcall_offset */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_as_async */
- (reprfunc)channelid_repr, /* tp_repr */
- &channelid_as_number, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- channelid_hash, /* tp_hash */
- 0, /* tp_call */
- (reprfunc)channelid_str, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- // Use Py_TPFLAGS_DISALLOW_INSTANTIATION so the type cannot be instantiated
- // from Python code. We do this because there is a strong relationship
- // between channel IDs and the channel lifecycle, so this limitation avoids
- // related complications. Use the _channel_id() function instead.
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
- | Py_TPFLAGS_DISALLOW_INSTANTIATION, /* tp_flags */
- channelid_doc, /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- channelid_richcompare, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- 0, /* tp_methods */
- 0, /* tp_members */
- channelid_getsets, /* tp_getset */
+static PyType_Slot ChannelIDType_slots[] = {
+ {Py_tp_dealloc, (destructor)channelid_dealloc},
+ {Py_tp_doc, (void *)channelid_doc},
+ {Py_tp_repr, (reprfunc)channelid_repr},
+ {Py_tp_str, (reprfunc)channelid_str},
+ {Py_tp_hash, channelid_hash},
+ {Py_tp_richcompare, channelid_richcompare},
+ {Py_tp_getset, channelid_getsets},
+ // number slots
+ {Py_nb_int, (unaryfunc)channelid_int},
+ {Py_nb_index, (unaryfunc)channelid_int},
+ {0, NULL},
};
+static PyType_Spec ChannelIDType_spec = {
+ .name = "_xxsubinterpreters.ChannelID",
+ .basicsize = sizeof(channelid),
+ .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
+ Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
+ .slots = ChannelIDType_slots,
+};
-/* interpreter-specific code ************************************************/
-static PyObject * RunFailedError = NULL;
+/* interpreter-specific code ************************************************/
static int
interp_exceptions_init(PyObject *mod)
{
- // XXX Move the exceptions into per-module memory?
+ module_state *state = get_module_state(mod);
+ if (state == NULL) {
+ return -1;
+ }
#define ADD(NAME, BASE) \
do { \
- if (NAME == NULL) { \
- NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \
- if (NAME == NULL) { \
- return -1; \
- } \
+ assert(state->NAME == NULL); \
+ state->NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \
+ if (state->NAME == NULL) { \
+ return -1; \
} \
} while (0)
@@ -2167,9 +2221,10 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp,
if (_ensure_not_running(interp) < 0) {
return -1;
}
+ module_state *state = get_module_state(mod);
int needs_import = 0;
- _sharedns *shared = _get_shared_ns(shareables, &ChannelIDType,
+ _sharedns *shared = _get_shared_ns(shareables, state->ChannelIDType,
&needs_import);
if (shared == NULL && PyErr_Occurred()) {
return -1;
@@ -2195,7 +2250,8 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp,
// Propagate any exception out to the caller.
if (exc != NULL) {
- _sharedexception_apply(exc, RunFailedError);
+ assert(state != NULL);
+ _sharedexception_apply(exc, state->RunFailedError);
_sharedexception_free(exc);
}
else if (result != 0) {
@@ -2530,8 +2586,12 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored))
(void)handle_channel_error(cid, self, -1);
return NULL;
}
+ module_state *state = get_module_state(self);
+ if (state == NULL) {
+ return NULL;
+ }
PyObject *id = NULL;
- int err = newchannelid(&ChannelIDType, cid, 0,
+ int err = newchannelid(state->ChannelIDType, cid, 0,
&_globals.channels, 0, 0,
(channelid **)&id);
if (handle_channel_error(err, self, cid)) {
@@ -2594,10 +2654,16 @@ channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
if (ids == NULL) {
goto finally;
}
+ module_state *state = get_module_state(self);
+ if (state == NULL) {
+ Py_DECREF(ids);
+ ids = NULL;
+ goto finally;
+ }
int64_t *cur = cids;
for (int64_t i=0; i < count; cur++, i++) {
PyObject *id = NULL;
- int err = newchannelid(&ChannelIDType, *cur, 0,
+ int err = newchannelid(state->ChannelIDType, *cur, 0,
&_globals.channels, 0, 0,
(channelid **)&id);
if (handle_channel_error(err, self, *cur)) {
@@ -2850,7 +2916,11 @@ ends are closed. Closing an already closed end is a noop.");
static PyObject *
channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds)
{
- PyTypeObject *cls = &ChannelIDType;
+ module_state *state = get_module_state(self);
+ if (state == NULL) {
+ return NULL;
+ }
+ PyTypeObject *cls = state->ChannelIDType;
PyObject *mod = get_module_from_owned_type(cls);
if (mod == NULL) {
return NULL;
@@ -2924,9 +2994,16 @@ module_exec(PyObject *mod)
}
/* Add other types */
- if (add_new_type(mod, &ChannelIDType, _channelid_shared) == NULL) {
+ module_state *state = get_module_state(mod);
+
+ // ChannelID
+ state->ChannelIDType = add_new_type(
+ mod, &ChannelIDType_spec, _channelid_shared);
+ if (state->ChannelIDType == NULL) {
goto error;
}
+
+ // PyInterpreterID
if (PyModule_AddType(mod, &_PyInterpreterID_Type) < 0) {
goto error;
}
@@ -2934,31 +3011,57 @@ module_exec(PyObject *mod)
return 0;
error:
- (void)_PyCrossInterpreterData_UnregisterClass(&ChannelIDType);
+ (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType);
_globals_fini();
return -1;
}
+static struct PyModuleDef_Slot module_slots[] = {
+ {Py_mod_exec, module_exec},
+ {0, NULL},
+};
+
+static int
+module_traverse(PyObject *mod, visitproc visit, void *arg)
+{
+ module_state *state = get_module_state(mod);
+ assert(state != NULL);
+ traverse_module_state(state, visit, arg);
+ return 0;
+}
+
+static int
+module_clear(PyObject *mod)
+{
+ module_state *state = get_module_state(mod);
+ assert(state != NULL);
+ clear_module_state(state);
+ return 0;
+}
+
+static void
+module_free(void *mod)
+{
+ module_state *state = get_module_state(mod);
+ assert(state != NULL);
+ clear_module_state(state);
+ _globals_fini();
+}
+
static struct PyModuleDef moduledef = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = MODULE_NAME,
.m_doc = module_doc,
- .m_size = -1,
+ .m_size = sizeof(module_state),
.m_methods = module_functions,
+ .m_slots = module_slots,
+ .m_traverse = module_traverse,
+ .m_clear = module_clear,
+ .m_free = (freefunc)module_free,
};
-
PyMODINIT_FUNC
PyInit__xxsubinterpreters(void)
{
- /* Create the module */
- PyObject *mod = PyModule_Create(&moduledef);
- if (mod == NULL) {
- return NULL;
- }
- if (module_exec(mod) < 0) {
- Py_DECREF(mod);
- return NULL;
- }
- return mod;
+ return PyModuleDef_Init(&moduledef);
}
diff --git a/Python/pystate.c b/Python/pystate.c
index 0fdcdf1e3569..ea3c22c5d71a 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -1789,30 +1789,78 @@ PyGILState_Release(PyGILState_STATE oldstate)
/* cross-interpreter data */
-crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *);
+static inline void
+_xidata_init(_PyCrossInterpreterData *data)
+{
+ // If the value is being reused
+ // then _xidata_clear() should have been called already.
+ assert(data->data == NULL);
+ assert(data->obj == NULL);
+ *data = (_PyCrossInterpreterData){0};
+ data->interp = -1;
+}
-/* This is a separate func from _PyCrossInterpreterData_Lookup in order
- to keep the registry code separate. */
-static crossinterpdatafunc
-_lookup_getdata(PyObject *obj)
+static inline void
+_xidata_clear(_PyCrossInterpreterData *data)
{
- crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj);
- if (getdata == NULL && PyErr_Occurred() == 0)
- PyErr_Format(PyExc_ValueError,
- "%S does not support cross-interpreter data", obj);
- return getdata;
+ if (data->free != NULL) {
+ data->free(data->data);
+ }
+ data->data = NULL;
+ Py_CLEAR(data->obj);
+}
+
+void
+_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data,
+ PyInterpreterState *interp,
+ void *shared, PyObject *obj,
+ xid_newobjectfunc new_object)
+{
+ assert(data != NULL);
+ assert(new_object != NULL);
+ _xidata_init(data);
+ data->data = shared;
+ if (obj != NULL) {
+ assert(interp != NULL);
+ // released in _PyCrossInterpreterData_Clear()
+ data->obj = Py_NewRef(obj);
+ }
+ // Ideally every object would know its owning interpreter.
+ // Until then, we have to rely on the caller to identify it
+ // (but we don't need it in all cases).
+ data->interp = (interp != NULL) ? interp->id : -1;
+ data->new_object = new_object;
}
int
-_PyObject_CheckCrossInterpreterData(PyObject *obj)
-{
- crossinterpdatafunc getdata = _lookup_getdata(obj);
- if (getdata == NULL) {
+_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data,
+ PyInterpreterState *interp,
+ const size_t size, PyObject *obj,
+ xid_newobjectfunc new_object)
+{
+ assert(size > 0);
+ // For now we always free the shared data in the same interpreter
+ // where it was allocated, so the interpreter is required.
+ assert(interp != NULL);
+ _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object);
+ data->data = PyMem_Malloc(size);
+ if (data->data == NULL) {
return -1;
}
+ data->free = PyMem_Free;
return 0;
}
+void
+_PyCrossInterpreterData_Clear(PyInterpreterState *interp,
+ _PyCrossInterpreterData *data)
+{
+ assert(data != NULL);
+ // This must be called in the owning interpreter.
+ assert(interp == NULL || data->interp == interp->id);
+ _xidata_clear(data);
+}
+
static int
_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data)
{
@@ -1835,6 +1883,30 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data)
return 0;
}
+crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *);
+
+/* This is a separate func from _PyCrossInterpreterData_Lookup in order
+ to keep the registry code separate. */
+static crossinterpdatafunc
+_lookup_getdata(PyObject *obj)
+{
+ crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj);
+ if (getdata == NULL && PyErr_Occurred() == 0)
+ PyErr_Format(PyExc_ValueError,
+ "%S does not support cross-interpreter data", obj);
+ return getdata;
+}
+
+int
+_PyObject_CheckCrossInterpreterData(PyObject *obj)
+{
+ crossinterpdatafunc getdata = _lookup_getdata(obj);
+ if (getdata == NULL) {
+ return -1;
+ }
+ return 0;
+}
+
int
_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
{
@@ -1847,7 +1919,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
// Reset data before re-populating.
*data = (_PyCrossInterpreterData){0};
- data->free = PyMem_RawFree; // Set a default that may be overridden.
+ data->interp = -1;
// Call the "getdata" func for the object.
Py_INCREF(obj);
@@ -1856,7 +1928,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
Py_DECREF(obj);
return -1;
}
- int res = getdata(obj, data);
+ int res = getdata(tstate, obj, data);
Py_DECREF(obj);
if (res != 0) {
return -1;
@@ -1872,21 +1944,17 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
return 0;
}
-static void
-_release_xidata(void *arg)
+PyObject *
+_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
{
- _PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg;
- if (data->free != NULL) {
- data->free(data->data);
- }
- data->data = NULL;
- Py_CLEAR(data->obj);
+ return data->new_object(data);
}
+typedef void (*releasefunc)(PyInterpreterState *, void *);
+
static void
_call_in_interpreter(struct _gilstate_runtime_state *gilstate,
- PyInterpreterState *interp,
- void (*func)(void *), void *arg)
+ PyInterpreterState *interp, releasefunc func, void *arg)
{
/* We would use Py_AddPendingCall() if it weren't specific to the
* main interpreter (see bpo-33608). In the meantime we take a
@@ -1902,7 +1970,7 @@ _call_in_interpreter(struct _gilstate_runtime_state *gilstate,
// XXX Once the GIL is per-interpreter, this should be called with the
// calling interpreter's GIL released and the target interpreter's held.
- func(arg);
+ func(interp, arg);
// Switch back.
if (save_tstate != NULL) {
@@ -1931,16 +1999,11 @@ _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
// "Release" the data and/or the object.
struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate;
- _call_in_interpreter(gilstate, interp, _release_xidata, data);
+ _call_in_interpreter(gilstate, interp,
+ (releasefunc)_PyCrossInterpreterData_Clear, data);
return 0;
}
-PyObject *
-_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
-{
- return data->new_object(data);
-}
-
/* registry of {type -> crossinterpdatafunc} */
/* For now we use a global registry of shareable classes. An
@@ -2091,16 +2154,21 @@ _new_bytes_object(_PyCrossInterpreterData *data)
}
static int
-_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data)
+_bytes_shared(PyThreadState *tstate, PyObject *obj,
+ _PyCrossInterpreterData *data)
{
- struct _shared_bytes_data *shared = PyMem_NEW(struct _shared_bytes_data, 1);
+ if (_PyCrossInterpreterData_InitWithSize(
+ data, tstate->interp, sizeof(struct _shared_bytes_data), obj,
+ _new_bytes_object
+ ) < 0)
+ {
+ return -1;
+ }
+ struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data;
if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) {
+ _PyCrossInterpreterData_Clear(tstate->interp, data);
return -1;
}
- data->data = (void *)shared;
- data->obj = Py_NewRef(obj); // Will be "released" (decref'ed) when data released.
- data->new_object = _new_bytes_object;
- data->free = PyMem_Free;
return 0;
}
@@ -2118,16 +2186,20 @@ _new_str_object(_PyCrossInterpreterData *data)
}
static int
-_str_shared(PyObject *obj, _PyCrossInterpreterData *data)
+_str_shared(PyThreadState *tstate, PyObject *obj,
+ _PyCrossInterpreterData *data)
{
- struct _shared_str_data *shared = PyMem_NEW(struct _shared_str_data, 1);
+ if (_PyCrossInterpreterData_InitWithSize(
+ data, tstate->interp, sizeof(struct _shared_str_data), obj,
+ _new_str_object
+ ) < 0)
+ {
+ return -1;
+ }
+ struct _shared_str_data *shared = (struct _shared_str_data *)data->data;
shared->kind = PyUnicode_KIND(obj);
shared->buffer = PyUnicode_DATA(obj);
shared->len = PyUnicode_GET_LENGTH(obj);
- data->data = (void *)shared;
- data->obj = Py_NewRef(obj); // Will be "released" (decref'ed) when data released.
- data->new_object = _new_str_object;
- data->free = PyMem_Free;
return 0;
}
@@ -2138,7 +2210,8 @@ _new_long_object(_PyCrossInterpreterData *data)
}
static int
-_long_shared(PyObject *obj, _PyCrossInterpreterData *data)
+_long_shared(PyThreadState *tstate, PyObject *obj,
+ _PyCrossInterpreterData *data)
{
/* Note that this means the size of shareable ints is bounded by
* sys.maxsize. Hence on 32-bit architectures that is half the
@@ -2151,10 +2224,9 @@ _long_shared(PyObject *obj, _PyCrossInterpreterData *data)
}
return -1;
}
- data->data = (void *)value;
- data->obj = NULL;
- data->new_object = _new_long_object;
- data->free = NULL;
+ _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL,
+ _new_long_object);
+ // data->obj and data->free remain NULL
return 0;
}
@@ -2166,12 +2238,12 @@ _new_none_object(_PyCrossInterpreterData *data)
}
static int
-_none_shared(PyObject *obj, _PyCrossInterpreterData *data)
+_none_shared(PyThreadState *tstate, PyObject *obj,
+ _PyCrossInterpreterData *data)
{
- data->data = NULL;
- // data->obj remains NULL
- data->new_object = _new_none_object;
- data->free = NULL; // There is nothing to free.
+ _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL,
+ _new_none_object);
+ // data->data, data->obj and data->free remain NULL
return 0;
}
diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index cc465134a9e0..adbb319a1b6f 100644
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -496,7 +496,6 @@ Modules/_pickle.c - PicklerMemoProxyType -
Modules/_pickle.c - Pickler_Type -
Modules/_pickle.c - UnpicklerMemoProxyType -
Modules/_pickle.c - Unpickler_Type -
-Modules/_xxsubinterpretersmodule.c - ChannelIDtype -
Modules/_zoneinfo.c - PyZoneInfo_ZoneInfoType -
Modules/ossaudiodev.c - OSSAudioType -
Modules/ossaudiodev.c - OSSMixerType -
@@ -523,12 +522,6 @@ Modules/_ctypes/_ctypes.c - PyExc_ArgError -
Modules/_cursesmodule.c - PyCursesError -
Modules/_decimal/_decimal.c - DecimalException -
Modules/_tkinter.c - Tkinter_TclError -
-Modules/_xxsubinterpretersmodule.c - ChannelError -
-Modules/_xxsubinterpretersmodule.c - ChannelNotFoundError -
-Modules/_xxsubinterpretersmodule.c - ChannelClosedError -
-Modules/_xxsubinterpretersmodule.c - ChannelEmptyError -
-Modules/_xxsubinterpretersmodule.c - ChannelNotEmptyError -
-Modules/_xxsubinterpretersmodule.c - RunFailedError -
Modules/ossaudiodev.c - OSSAudioError -
Modules/socketmodule.c - socket_herror -
Modules/socketmodule.c - socket_gaierror -
More information about the Python-checkins
mailing list