[Python-checkins] bpo-46417: Finalize structseq types at exit (GH-30645)
vstinner
webhook-mailer at python.org
Thu Jan 20 19:42:36 EST 2022
https://github.com/python/cpython/commit/e9e3eab0b868c7d0b48e472705024240d5c39d5c
commit: e9e3eab0b868c7d0b48e472705024240d5c39d5c
branch: main
author: Victor Stinner <vstinner at python.org>
committer: vstinner <vstinner at python.org>
date: 2022-01-21T01:42:25+01:00
summary:
bpo-46417: Finalize structseq types at exit (GH-30645)
Add _PyStructSequence_FiniType() and _PyStaticType_Dealloc()
functions to finalize a structseq static type in Py_Finalize().
Currrently, these functions do nothing if Python is built in release
mode.
Clear static types:
* AsyncGenHooksType: sys.set_asyncgen_hooks()
* FlagsType: sys.flags
* FloatInfoType: sys.float_info
* Hash_InfoType: sys.hash_info
* Int_InfoType: sys.int_info
* ThreadInfoType: sys.thread_info
* UnraisableHookArgsType: sys.unraisablehook
* VersionInfoType: sys.version
* WindowsVersionType: sys.getwindowsversion()
files:
A Lib/test/_test_embed_structseq.py
M Include/internal/pycore_floatobject.h
M Include/internal/pycore_long.h
M Include/internal/pycore_pyerrors.h
M Include/internal/pycore_pylifecycle.h
M Include/internal/pycore_typeobject.h
M Include/structseq.h
M Lib/test/test_embed.py
M Objects/floatobject.c
M Objects/longobject.c
M Objects/structseq.c
M Objects/typeobject.c
M Programs/_testembed.c
M Python/errors.c
M Python/pylifecycle.c
M Python/sysmodule.c
M Python/thread.c
diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h
index be6045587de1c..891e422f59472 100644
--- a/Include/internal/pycore_floatobject.h
+++ b/Include/internal/pycore_floatobject.h
@@ -14,6 +14,7 @@ extern "C" {
extern void _PyFloat_InitState(PyInterpreterState *);
extern PyStatus _PyFloat_InitTypes(PyInterpreterState *);
extern void _PyFloat_Fini(PyInterpreterState *);
+extern void _PyFloat_FiniType(PyInterpreterState *);
/* other API */
diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h
index 4d1a0d0424969..436bf08468599 100644
--- a/Include/internal/pycore_long.h
+++ b/Include/internal/pycore_long.h
@@ -15,6 +15,7 @@ extern "C" {
/* runtime lifecycle */
extern PyStatus _PyLong_InitTypes(PyInterpreterState *);
+extern void _PyLong_FiniTypes(PyInterpreterState *interp);
/* other API */
diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h
index f375337a405bb..e3c445ba5d926 100644
--- a/Include/internal/pycore_pyerrors.h
+++ b/Include/internal/pycore_pyerrors.h
@@ -12,6 +12,7 @@ extern "C" {
/* runtime lifecycle */
extern PyStatus _PyErr_InitTypes(PyInterpreterState *);
+extern void _PyErr_FiniTypes(PyInterpreterState *);
/* other API */
diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index 766e889f237b9..dfa8fd6bd0d28 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -58,6 +58,7 @@ extern PyStatus _PySys_Create(
extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options);
extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config);
extern int _PySys_UpdateConfig(PyThreadState *tstate);
+extern void _PySys_Fini(PyInterpreterState *interp);
extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod);
extern PyStatus _Py_HashRandomization_Init(const PyConfig *);
@@ -81,6 +82,7 @@ extern void _PyTraceMalloc_Fini(void);
extern void _PyWarnings_Fini(PyInterpreterState *interp);
extern void _PyAST_Fini(PyInterpreterState *interp);
extern void _PyAtExit_Fini(PyInterpreterState *interp);
+extern void _PyThread_FiniType(PyInterpreterState *interp);
extern PyStatus _PyGILState_Init(_PyRuntimeState *runtime);
extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate);
diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h
index 7fd8a1f35092f..ba95bbc1c4820 100644
--- a/Include/internal/pycore_typeobject.h
+++ b/Include/internal/pycore_typeobject.h
@@ -40,6 +40,8 @@ struct type_cache {
extern PyStatus _PyTypes_InitSlotDefs(void);
+extern void _PyStaticType_Dealloc(PyTypeObject *type);
+
#ifdef __cplusplus
}
diff --git a/Include/structseq.h b/Include/structseq.h
index e89265a67c322..8abd2443468b7 100644
--- a/Include/structseq.h
+++ b/Include/structseq.h
@@ -27,6 +27,9 @@ PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type,
PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type,
PyStructSequence_Desc *desc);
#endif
+#ifdef Py_BUILD_CORE
+PyAPI_FUNC(void) _PyStructSequence_FiniType(PyTypeObject *type);
+#endif
PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc);
PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type);
diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py
new file mode 100644
index 0000000000000..868f9f83e8be7
--- /dev/null
+++ b/Lib/test/_test_embed_structseq.py
@@ -0,0 +1,55 @@
+import sys
+import types
+import unittest
+
+
+# bpo-46417: Test that structseq types used by the sys module are still
+# valid when Py_Finalize()/Py_Initialize() are called multiple times.
+class TestStructSeq(unittest.TestCase):
+ # test PyTypeObject members
+ def check_structseq(self, obj_type):
+ # ob_refcnt
+ self.assertGreaterEqual(sys.getrefcount(obj_type), 1)
+ # tp_base
+ self.assertTrue(issubclass(obj_type, tuple))
+ # tp_bases
+ self.assertEqual(obj_type.__bases__, (tuple,))
+ # tp_dict
+ self.assertIsInstance(obj_type.__dict__, types.MappingProxyType)
+ # tp_mro
+ self.assertEqual(obj_type.__mro__, (obj_type, tuple, object))
+ # tp_name
+ self.assertIsInstance(type.__name__, str)
+ # tp_subclasses
+ self.assertEqual(obj_type.__subclasses__(), [])
+
+ def test_sys_attrs(self):
+ for attr_name in (
+ 'flags', # FlagsType
+ 'float_info', # FloatInfoType
+ 'hash_info', # Hash_InfoType
+ 'int_info', # Int_InfoType
+ 'thread_info', # ThreadInfoType
+ 'version_info', # VersionInfoType
+ ):
+ with self.subTest(attr=attr_name):
+ attr = getattr(sys, attr_name)
+ self.check_structseq(type(attr))
+
+ def test_sys_funcs(self):
+ func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType
+ if hasattr(sys, 'getwindowsversion'):
+ func_names.append('getwindowsversion') # WindowsVersionType
+ for func_name in func_names:
+ with self.subTest(func=func_name):
+ func = getattr(sys, func_name)
+ obj = func()
+ self.check_structseq(type(obj))
+
+
+try:
+ unittest.main()
+except SystemExit as exc:
+ if exc.args[0] != 0:
+ raise
+print("Tests passed")
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 9fed0a5f14e65..204b194ed3bf3 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -329,6 +329,18 @@ def test_run_main_loop(self):
self.assertEqual(out, "Py_RunMain(): sys.argv=['-c', 'arg2']\n" * nloop)
self.assertEqual(err, '')
+ def test_finalize_structseq(self):
+ # bpo-46417: Py_Finalize() clears structseq static types. Check that
+ # sys attributes using struct types still work when
+ # Py_Finalize()/Py_Initialize() is called multiple times.
+ # print() calls type->tp_repr(instance) and so checks that the types
+ # are still working properly.
+ script = support.findfile('_test_embed_structseq.py')
+ with open(script, encoding="utf-8") as fp:
+ code = fp.read()
+ out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
+ self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS)
+
class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
maxDiff = 4096
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index f8620d6f8ef0b..88f25d6b8c886 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -2082,6 +2082,14 @@ _PyFloat_Fini(PyInterpreterState *interp)
#endif
}
+void
+_PyFloat_FiniType(PyInterpreterState *interp)
+{
+ if (_Py_IsMainInterpreter(interp)) {
+ _PyStructSequence_FiniType(&FloatInfoType);
+ }
+}
+
/* Print summary info about the state of the optimized allocator */
void
_PyFloat_DebugMallocStats(FILE *out)
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 1b2d1266c6bc5..5aa53dd91c299 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -5949,3 +5949,14 @@ _PyLong_InitTypes(PyInterpreterState *interp)
return _PyStatus_OK();
}
+
+
+void
+_PyLong_FiniTypes(PyInterpreterState *interp)
+{
+ if (!_Py_IsMainInterpreter(interp)) {
+ return;
+ }
+
+ _PyStructSequence_FiniType(&Int_InfoType);
+}
diff --git a/Objects/structseq.c b/Objects/structseq.c
index a2eefb0455a17..f8bf9477f2848 100644
--- a/Objects/structseq.c
+++ b/Objects/structseq.c
@@ -532,6 +532,36 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc)
(void)PyStructSequence_InitType2(type, desc);
}
+
+void
+_PyStructSequence_FiniType(PyTypeObject *type)
+{
+ // Ensure that the type is initialized
+ assert(type->tp_name != NULL);
+ assert(type->tp_base == &PyTuple_Type);
+
+ // Cannot delete a type if it still has subclasses
+ if (type->tp_subclasses != NULL) {
+ return;
+ }
+
+ // Undo PyStructSequence_NewType()
+ type->tp_name = NULL;
+ PyMem_Free(type->tp_members);
+
+ _PyStaticType_Dealloc(type);
+ assert(Py_REFCNT(type) == 1);
+ // Undo Py_INCREF(type) of _PyStructSequence_InitType().
+ // Don't use Py_DECREF(): static type must not be deallocated
+ Py_SET_REFCNT(type, 0);
+
+ // Make sure that _PyStructSequence_InitType() will initialize
+ // the type again
+ assert(Py_REFCNT(type) == 0);
+ assert(type->tp_name == NULL);
+}
+
+
PyTypeObject *
PyStructSequence_NewType(PyStructSequence_Desc *desc)
{
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index cbf806b074b9f..66a10a5bc57dd 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -4070,10 +4070,27 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value)
extern void
_PyDictKeys_DecRef(PyDictKeysObject *keys);
+
+void
+_PyStaticType_Dealloc(PyTypeObject *type)
+{
+ // _PyStaticType_Dealloc() must not be called if a type has subtypes.
+ // A subtype can inherit attributes and methods of its parent type,
+ // and a type must no longer be used once it's deallocated.
+ assert(type->tp_subclasses == NULL);
+
+ Py_CLEAR(type->tp_dict);
+ Py_CLEAR(type->tp_bases);
+ Py_CLEAR(type->tp_mro);
+ Py_CLEAR(type->tp_cache);
+ Py_CLEAR(type->tp_subclasses);
+ type->tp_flags &= ~Py_TPFLAGS_READY;
+}
+
+
static void
type_dealloc(PyTypeObject *type)
{
- PyHeapTypeObject *et;
PyObject *tp, *val, *tb;
/* Assert this is a heap-allocated type object */
@@ -4082,8 +4099,8 @@ type_dealloc(PyTypeObject *type)
PyErr_Fetch(&tp, &val, &tb);
remove_all_subclasses(type, type->tp_bases);
PyErr_Restore(tp, val, tb);
+
PyObject_ClearWeakRefs((PyObject *)type);
- et = (PyHeapTypeObject *)type;
Py_XDECREF(type->tp_base);
Py_XDECREF(type->tp_dict);
Py_XDECREF(type->tp_bases);
@@ -4094,6 +4111,8 @@ type_dealloc(PyTypeObject *type)
* of most other objects. It's okay to cast it to char *.
*/
PyObject_Free((char *)type->tp_doc);
+
+ PyHeapTypeObject *et = (PyHeapTypeObject *)type;
Py_XDECREF(et->ht_name);
Py_XDECREF(et->ht_qualname);
Py_XDECREF(et->ht_slots);
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index b31781938eb39..5bc0a127a8750 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -15,12 +15,18 @@
#include <stdlib.h> // putenv()
#include <wchar.h>
+int main_argc;
+char **main_argv;
+
/*********************************************************
* Embedded interpreter tests that need a custom exe
*
* Executed via 'EmbeddingTests' in Lib/test/test_capi.py
*********************************************************/
+// Use to display the usage
+#define PROGRAM "test_embed"
+
/* Use path starting with "./" avoids a search along the PATH */
#define PROGRAM_NAME L"./_testembed"
@@ -113,6 +119,36 @@ PyInit_embedded_ext(void)
return PyModule_Create(&embedded_ext);
}
+/****************************************************************************
+ * Call Py_Initialize()/Py_Finalize() multiple times and execute Python code
+ ***************************************************************************/
+
+// Used by bpo-46417 to test that structseq types used by the sys module are
+// cleared properly and initialized again properly when Python is finalized
+// multiple times.
+static int test_repeated_init_exec(void)
+{
+ if (main_argc < 3) {
+ fprintf(stderr, "usage: %s test_repeated_init_exec CODE\n", PROGRAM);
+ exit(1);
+ }
+ const char *code = main_argv[2];
+
+ for (int i=1; i <= INIT_LOOPS; i++) {
+ fprintf(stderr, "--- Loop #%d ---\n", i);
+ fflush(stderr);
+
+ _testembed_Py_Initialize();
+ int err = PyRun_SimpleString(code);
+ Py_Finalize();
+ if (err) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+
/*****************************************************
* Test forcing a particular IO encoding
*****************************************************/
@@ -1880,6 +1916,7 @@ struct TestCase
static struct TestCase TestCases[] = {
// Python initialization
+ {"test_repeated_init_exec", test_repeated_init_exec},
{"test_forced_io_encoding", test_forced_io_encoding},
{"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters},
{"test_repeated_init_and_inittab", test_repeated_init_and_inittab},
@@ -1946,6 +1983,9 @@ static struct TestCase TestCases[] = {
int main(int argc, char *argv[])
{
+ main_argc = argc;
+ main_argv = argv;
+
if (argc > 1) {
for (struct TestCase *tc = TestCases; tc && tc->name; tc++) {
if (strcmp(argv[1], tc->name) == 0)
diff --git a/Python/errors.c b/Python/errors.c
index 6c5fe41142304..211881ca5eb6c 100644
--- a/Python/errors.c
+++ b/Python/errors.c
@@ -1241,6 +1241,17 @@ _PyErr_InitTypes(PyInterpreterState *interp)
}
+void
+_PyErr_FiniTypes(PyInterpreterState *interp)
+{
+ if (!_Py_IsMainInterpreter(interp)) {
+ return;
+ }
+
+ _PyStructSequence_FiniType(&UnraisableHookArgsType);
+}
+
+
static PyObject *
make_unraisable_hook_args(PyThreadState *tstate, PyObject *exc_type,
PyObject *exc_value, PyObject *exc_tb,
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 8bcad67e80a0c..0b1f47147696d 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1666,11 +1666,17 @@ flush_std_files(void)
static void
finalize_interp_types(PyInterpreterState *interp)
{
+ _PySys_Fini(interp);
_PyExc_Fini(interp);
_PyFrame_Fini(interp);
_PyAsyncGen_Fini(interp);
_PyContext_Fini(interp);
+ _PyFloat_FiniType(interp);
+ _PyLong_FiniTypes(interp);
+ _PyThread_FiniType(interp);
+ _PyErr_FiniTypes(interp);
_PyTypes_Fini(interp);
+
// Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses
// a dict internally.
_PyUnicode_ClearInterned(interp);
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 0b7b61d8b1e28..515994f049086 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -3102,6 +3102,21 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p)
}
+void
+_PySys_Fini(PyInterpreterState *interp)
+{
+ if (_Py_IsMainInterpreter(interp)) {
+ _PyStructSequence_FiniType(&VersionInfoType);
+ _PyStructSequence_FiniType(&FlagsType);
+#if defined(MS_WINDOWS)
+ _PyStructSequence_FiniType(&WindowsVersionType);
+#endif
+ _PyStructSequence_FiniType(&Hash_InfoType);
+ _PyStructSequence_FiniType(&AsyncGenHooksType);
+ }
+}
+
+
static PyObject *
makepathobject(const wchar_t *path, wchar_t delim)
{
diff --git a/Python/thread.c b/Python/thread.c
index b1c0cfe84f28d..c2457c4f8fe83 100644
--- a/Python/thread.c
+++ b/Python/thread.c
@@ -243,3 +243,14 @@ PyThread_GetInfo(void)
PyStructSequence_SET_ITEM(threadinfo, pos++, value);
return threadinfo;
}
+
+
+void
+_PyThread_FiniType(PyInterpreterState *interp)
+{
+ if (!_Py_IsMainInterpreter(interp)) {
+ return;
+ }
+
+ _PyStructSequence_FiniType(&ThreadInfoType);
+}
More information about the Python-checkins
mailing list