[Python-checkins] gh-59956: Clarify Runtime State Status Expectations (gh-101308)
ericsnowcurrently
webhook-mailer at python.org
Mon Jan 30 14:08:14 EST 2023
https://github.com/python/cpython/commit/e11fc032a75d067d2167a21037722a770b9dfb51
commit: e11fc032a75d067d2167a21037722a770b9dfb51
branch: main
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: ericsnowcurrently <ericsnowcurrently at gmail.com>
date: 2023-01-30T12:07:48-07:00
summary:
gh-59956: Clarify Runtime State Status Expectations (gh-101308)
A PyThreadState can be in one of many states in its lifecycle, represented by some status value. Those statuses haven't been particularly clear, so we're addressing that here. Specifically:
* made the distinct lifecycle statuses clear on PyThreadState
* identified expectations of how various lifecycle-related functions relate to status
* noted the various places where those expectations don't match the actual behavior
At some point we'll need to address the mismatches.
(This change also includes some cleanup.)
https://github.com/python/cpython/issues/59956
files:
M Include/cpython/pystate.h
M Include/internal/pycore_pystate.h
M Modules/_threadmodule.c
M Python/ceval_gil.c
M Python/pylifecycle.c
M Python/pystate.c
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 3c1e70de3e48..f5db52f76e8f 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -119,7 +119,30 @@ struct _ts {
PyThreadState *next;
PyInterpreterState *interp;
- int _status;
+ struct {
+ /* Has been initialized to a safe state.
+
+ In order to be effective, this must be set to 0 during or right
+ after allocation. */
+ unsigned int initialized:1;
+
+ /* Has been bound to an OS thread. */
+ unsigned int bound:1;
+ /* Has been unbound from its OS thread. */
+ unsigned int unbound:1;
+ /* Has been bound aa current for the GILState API. */
+ unsigned int bound_gilstate:1;
+ /* Currently in use (maybe holds the GIL). */
+ unsigned int active:1;
+
+ /* various stages of finalization */
+ unsigned int finalizing:1;
+ unsigned int cleared:1;
+ unsigned int finalized:1;
+
+ /* padding to align to 4 bytes */
+ unsigned int :24;
+ } _status;
int py_recursion_remaining;
int py_recursion_limit;
@@ -245,6 +268,8 @@ struct _ts {
// Alias for backward compatibility with Python 3.8
#define _PyInterpreterState_Get PyInterpreterState_Get
+/* An alias for the internal _PyThreadState_New(),
+ kept for stable ABI compatibility. */
PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *);
/* Similar to PyThreadState_Get(), but don't issue a fatal error
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 0e46693c1f12..7046ec8d9ada 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -120,13 +120,12 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) {
// PyThreadState functions
+PyAPI_FUNC(PyThreadState *) _PyThreadState_New(PyInterpreterState *interp);
PyAPI_FUNC(void) _PyThreadState_Bind(PyThreadState *tstate);
// We keep this around exclusively for stable ABI compatibility.
PyAPI_FUNC(void) _PyThreadState_Init(
PyThreadState *tstate);
-PyAPI_FUNC(void) _PyThreadState_DeleteExcept(
- _PyRuntimeState *runtime,
- PyThreadState *tstate);
+PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate);
static inline void
@@ -139,18 +138,6 @@ _PyThreadState_UpdateTracingState(PyThreadState *tstate)
}
-/* PyThreadState status */
-
-#define PyThreadState_UNINITIALIZED 0
-/* Has been initialized to a safe state.
-
- In order to be effective, this must be set to 0 during or right
- after allocation. */
-#define PyThreadState_INITIALIZED 1
-#define PyThreadState_BOUND 2
-#define PyThreadState_UNBOUND 3
-
-
/* Other */
PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap(
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
index bf4b6ec00e3e..9c12c6967574 100644
--- a/Modules/_threadmodule.c
+++ b/Modules/_threadmodule.c
@@ -1161,7 +1161,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs)
return PyErr_NoMemory();
}
boot->interp = _PyInterpreterState_GET();
- boot->tstate = _PyThreadState_Prealloc(boot->interp);
+ boot->tstate = _PyThreadState_New(boot->interp);
if (boot->tstate == NULL) {
PyMem_Free(boot);
if (!PyErr_Occurred()) {
diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c
index 73d412ba4d71..1bf223348d28 100644
--- a/Python/ceval_gil.c
+++ b/Python/ceval_gil.c
@@ -624,7 +624,7 @@ _PyEval_ReInitThreads(PyThreadState *tstate)
}
/* Destroy all threads except the current one */
- _PyThreadState_DeleteExcept(runtime, tstate);
+ _PyThreadState_DeleteExcept(tstate);
return _PyStatus_OK();
}
#endif
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 5ef2d3f6aa72..a8a8e7f3d84f 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -696,10 +696,11 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
const _PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT;
init_interp_settings(interp, &config);
- PyThreadState *tstate = PyThreadState_New(interp);
+ PyThreadState *tstate = _PyThreadState_New(interp);
if (tstate == NULL) {
return _PyStatus_ERR("can't make first thread");
}
+ _PyThreadState_Bind(tstate);
(void) PyThreadState_Swap(tstate);
status = init_interp_create_gil(tstate);
@@ -1821,6 +1822,11 @@ Py_FinalizeEx(void)
/* Get current thread state and interpreter pointer */
PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+ // XXX assert(_Py_IsMainInterpreter(tstate->interp));
+ // XXX assert(_Py_IsMainThread());
+
+ // Block some operations.
+ tstate->interp->finalizing = 1;
// Wrap up existing "threading"-module-created, non-daemon threads.
wait_for_thread_shutdown(tstate);
@@ -1867,7 +1873,23 @@ Py_FinalizeEx(void)
_PyRuntimeState_SetFinalizing() has been called, no other Python thread
can take the GIL at this point: if they try, they will exit
immediately. */
- _PyThreadState_DeleteExcept(runtime, tstate);
+ _PyThreadState_DeleteExcept(tstate);
+
+ /* At this point no Python code should be running at all.
+ The only thread state left should be the main thread of the main
+ interpreter (AKA tstate), in which this code is running right now.
+ There may be other OS threads running but none of them will have
+ thread states associated with them, nor will be able to create
+ new thread states.
+
+ Thus tstate is the only possible thread state from here on out.
+ It may still be used during finalization to run Python code as
+ needed or provide runtime state (e.g. sys.modules) but that will
+ happen sparingly. Furthermore, the order of finalization aims
+ to not need a thread (or interpreter) state as soon as possible.
+ */
+ // XXX Make sure we are preventing the creating of any new thread states
+ // (or interpreters).
/* Flush sys.stdout and sys.stderr */
if (flush_std_files() < 0) {
@@ -1958,6 +1980,20 @@ Py_FinalizeEx(void)
}
#endif /* Py_TRACE_REFS */
+ /* At this point there's almost no other Python code that will run,
+ nor interpreter state needed. The only possibility is the
+ finalizers of the objects stored on tstate (and tstate->interp),
+ which are triggered via finalize_interp_clear().
+
+ For now we operate as though none of those finalizers actually
+ need an operational thread state or interpreter. In reality,
+ those finalizers may rely on some part of tstate or
+ tstate->interp, and/or may raise exceptions
+ or otherwise fail.
+ */
+ // XXX Do this sooner during finalization.
+ // XXX Ensure finalizer errors are handled properly.
+
finalize_interp_clear(tstate);
finalize_interp_delete(tstate->interp);
@@ -2039,12 +2075,13 @@ new_interpreter(PyThreadState **tstate_p, const _PyInterpreterConfig *config)
return _PyStatus_OK();
}
- PyThreadState *tstate = PyThreadState_New(interp);
+ PyThreadState *tstate = _PyThreadState_New(interp);
if (tstate == NULL) {
PyInterpreterState_Delete(interp);
*tstate_p = NULL;
return _PyStatus_OK();
}
+ _PyThreadState_Bind(tstate);
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
diff --git a/Python/pystate.c b/Python/pystate.c
index bf7688fd3213..8bb49d954a81 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -37,9 +37,6 @@ to avoid the expense of doing their own locking).
extern "C" {
#endif
-/* Forward declarations */
-static void _PyThreadState_Delete(PyThreadState *tstate, int check_current);
-
/****************************************/
/* helpers for the current thread state */
@@ -81,69 +78,56 @@ current_fast_clear(_PyRuntimeState *runtime)
_Py_atomic_store_relaxed(&runtime->tstate_current, (uintptr_t)NULL);
}
+#define tstate_verify_not_active(tstate) \
+ if (tstate == current_fast_get((tstate)->interp->runtime)) { \
+ _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); \
+ }
+
//------------------------------------------------
// the thread state bound to the current OS thread
//------------------------------------------------
-/*
- The stored thread state is set by bind_tstate() (AKA PyThreadState_Bind().
-
- The GIL does no need to be held for these.
- */
-
-static int
-current_tss_initialized(_PyRuntimeState *runtime)
+static inline int
+tstate_tss_initialized(Py_tss_t *key)
{
- return PyThread_tss_is_created(&runtime->autoTSSkey);
+ return PyThread_tss_is_created(key);
}
-static PyStatus
-current_tss_init(_PyRuntimeState *runtime)
+static inline int
+tstate_tss_init(Py_tss_t *key)
{
- assert(!current_tss_initialized(runtime));
- if (PyThread_tss_create(&runtime->autoTSSkey) != 0) {
- return _PyStatus_NO_MEMORY();
- }
- return _PyStatus_OK();
+ assert(!tstate_tss_initialized(key));
+ return PyThread_tss_create(key);
}
-static void
-current_tss_fini(_PyRuntimeState *runtime)
+static inline void
+tstate_tss_fini(Py_tss_t *key)
{
- assert(current_tss_initialized(runtime));
- PyThread_tss_delete(&runtime->autoTSSkey);
+ assert(tstate_tss_initialized(key));
+ PyThread_tss_delete(key);
}
static inline PyThreadState *
-current_tss_get(_PyRuntimeState *runtime)
+tstate_tss_get(Py_tss_t *key)
{
- assert(current_tss_initialized(runtime));
- return (PyThreadState *)PyThread_tss_get(&runtime->autoTSSkey);
+ assert(tstate_tss_initialized(key));
+ return (PyThreadState *)PyThread_tss_get(key);
}
static inline int
-_current_tss_set(_PyRuntimeState *runtime, PyThreadState *tstate)
+tstate_tss_set(Py_tss_t *key, PyThreadState *tstate)
{
assert(tstate != NULL);
- assert(current_tss_initialized(runtime));
- return PyThread_tss_set(&runtime->autoTSSkey, (void *)tstate);
-}
-static inline void
-current_tss_set(_PyRuntimeState *runtime, PyThreadState *tstate)
-{
- if (_current_tss_set(runtime, tstate) != 0) {
- Py_FatalError("failed to set current tstate (TSS)");
- }
+ assert(tstate_tss_initialized(key));
+ return PyThread_tss_set(key, (void *)tstate);
}
-static inline void
-current_tss_clear(_PyRuntimeState *runtime)
+static inline int
+tstate_tss_clear(Py_tss_t *key)
{
- assert(current_tss_initialized(runtime));
- if (PyThread_tss_set(&runtime->autoTSSkey, NULL) != 0) {
- Py_FatalError("failed to clear current tstate (TSS)");
- }
+ assert(tstate_tss_initialized(key));
+ return PyThread_tss_set(key, (void *)NULL);
}
#ifdef HAVE_FORK
@@ -152,92 +136,178 @@ current_tss_clear(_PyRuntimeState *runtime)
* don't reset TSS upon fork(), see issue #10517.
*/
static PyStatus
-current_tss_reinit(_PyRuntimeState *runtime)
+tstate_tss_reinit(Py_tss_t *key)
{
- if (!current_tss_initialized(runtime)) {
+ if (!tstate_tss_initialized(key)) {
return _PyStatus_OK();
}
- PyThreadState *tstate = current_tss_get(runtime);
+ PyThreadState *tstate = tstate_tss_get(key);
- current_tss_fini(runtime);
- PyStatus status = current_tss_init(runtime);
- if (_PyStatus_EXCEPTION(status)) {
- return status;
+ tstate_tss_fini(key);
+ if (tstate_tss_init(key) != 0) {
+ return _PyStatus_NO_MEMORY();
}
/* If the thread had an associated auto thread state, reassociate it with
* the new key. */
- if (tstate && _current_tss_set(runtime, tstate) != 0) {
- return _PyStatus_ERR("failed to set autoTSSkey");
+ if (tstate && tstate_tss_set(key, tstate) != 0) {
+ return _PyStatus_ERR("failed to re-set autoTSSkey");
}
return _PyStatus_OK();
}
#endif
+/*
+ The stored thread state is set by bind_tstate() (AKA PyThreadState_Bind().
+
+ The GIL does no need to be held for these.
+ */
+
+#define gilstate_tss_initialized(runtime) \
+ tstate_tss_initialized(&(runtime)->autoTSSkey)
+#define gilstate_tss_init(runtime) \
+ tstate_tss_init(&(runtime)->autoTSSkey)
+#define gilstate_tss_fini(runtime) \
+ tstate_tss_fini(&(runtime)->autoTSSkey)
+#define gilstate_tss_get(runtime) \
+ tstate_tss_get(&(runtime)->autoTSSkey)
+#define _gilstate_tss_set(runtime, tstate) \
+ tstate_tss_set(&(runtime)->autoTSSkey, tstate)
+#define _gilstate_tss_clear(runtime) \
+ tstate_tss_clear(&(runtime)->autoTSSkey)
+#define gilstate_tss_reinit(runtime) \
+ tstate_tss_reinit(&(runtime)->autoTSSkey)
+
+static inline void
+gilstate_tss_set(_PyRuntimeState *runtime, PyThreadState *tstate)
+{
+ assert(tstate != NULL && tstate->interp->runtime == runtime);
+ if (_gilstate_tss_set(runtime, tstate) != 0) {
+ Py_FatalError("failed to set current tstate (TSS)");
+ }
+}
+
+static inline void
+gilstate_tss_clear(_PyRuntimeState *runtime)
+{
+ if (_gilstate_tss_clear(runtime) != 0) {
+ Py_FatalError("failed to clear current tstate (TSS)");
+ }
+}
+
+
+static inline int tstate_is_alive(PyThreadState *tstate);
+
+static inline int
+tstate_is_bound(PyThreadState *tstate)
+{
+ return tstate->_status.bound && !tstate->_status.unbound;
+}
+
+static void bind_gilstate_tstate(PyThreadState *);
+static void unbind_gilstate_tstate(PyThreadState *);
+
static void
bind_tstate(PyThreadState *tstate)
{
assert(tstate != NULL);
- assert(tstate->_status == PyThreadState_INITIALIZED);
+ assert(tstate_is_alive(tstate) && !tstate->_status.bound);
+ assert(!tstate->_status.unbound); // just in case
+ assert(!tstate->_status.bound_gilstate);
+ assert(tstate != gilstate_tss_get(tstate->interp->runtime));
+ assert(!tstate->_status.active);
assert(tstate->thread_id == 0);
assert(tstate->native_thread_id == 0);
- _PyRuntimeState *runtime = tstate->interp->runtime;
-
- /* Stick the thread state for this thread in thread specific storage.
- The only situation where you can legitimately have more than one
- thread state for an OS level thread is when there are multiple
- interpreters.
-
- You shouldn't really be using the PyGILState_ APIs anyway (see issues
- #10915 and #15751).
-
- The first thread state created for that given OS level thread will
- "win", which seems reasonable behaviour.
- */
- /* When a thread state is created for a thread by some mechanism
- other than PyGILState_Ensure(), it's important that the GILState
- machinery knows about it so it doesn't try to create another
- thread state for the thread.
- (This is a better fix for SF bug #1010677 than the first one attempted.)
- */
- // XXX Skipping like this does not play nice with multiple interpreters.
- if (current_tss_get(runtime) == NULL) {
- current_tss_set(runtime, tstate);
- }
+ // Currently we don't necessarily store the thread state
+ // in thread-local storage (e.g. per-interpreter).
tstate->thread_id = PyThread_get_thread_ident();
#ifdef PY_HAVE_THREAD_NATIVE_ID
tstate->native_thread_id = PyThread_get_thread_native_id();
#endif
- tstate->_status = PyThreadState_BOUND;
+ tstate->_status.bound = 1;
}
static void
unbind_tstate(PyThreadState *tstate)
{
assert(tstate != NULL);
- assert(tstate->_status == PyThreadState_BOUND);
+ // XXX assert(tstate_is_alive(tstate));
+ assert(tstate_is_bound(tstate));
+ // XXX assert(!tstate->_status.active);
assert(tstate->thread_id > 0);
#ifdef PY_HAVE_THREAD_NATIVE_ID
assert(tstate->native_thread_id > 0);
#endif
- _PyRuntimeState *runtime = tstate->interp->runtime;
-
- if (current_tss_initialized(runtime) &&
- tstate == current_tss_get(runtime))
- {
- current_tss_clear(runtime);
- }
// We leave thread_id and native_thread_id alone
// since they can be useful for debugging.
// Check the `_status` field to know if these values
// are still valid.
- tstate->_status = PyThreadState_UNBOUND;
+ // We leave tstate->_status.bound set to 1
+ // to indicate it was previously bound.
+ tstate->_status.unbound = 1;
+}
+
+
+/* Stick the thread state for this thread in thread specific storage.
+
+ When a thread state is created for a thread by some mechanism
+ other than PyGILState_Ensure(), it's important that the GILState
+ machinery knows about it so it doesn't try to create another
+ thread state for the thread.
+ (This is a better fix for SF bug #1010677 than the first one attempted.)
+
+ The only situation where you can legitimately have more than one
+ thread state for an OS level thread is when there are multiple
+ interpreters.
+
+ The PyGILState_*() APIs don't work with multiple
+ interpreters (see bpo-10915 and bpo-15751), so this function
+ sets TSS only once. Thus, the first thread state created for that
+ given OS level thread will "win", which seems reasonable behaviour.
+*/
+
+static void
+bind_gilstate_tstate(PyThreadState *tstate)
+{
+ assert(tstate != NULL);
+ assert(tstate_is_alive(tstate));
+ assert(tstate_is_bound(tstate));
+ // XXX assert(!tstate->_status.active);
+ assert(!tstate->_status.bound_gilstate);
+
+ _PyRuntimeState *runtime = tstate->interp->runtime;
+ PyThreadState *tcur = gilstate_tss_get(runtime);
+ assert(tstate != tcur);
+
+ if (tcur != NULL) {
+ // The original gilstate implementation only respects the
+ // first thread state set.
+ // XXX Skipping like this does not play nice with multiple interpreters.
+ return;
+ tcur->_status.bound_gilstate = 0;
+ }
+ gilstate_tss_set(runtime, tstate);
+ tstate->_status.bound_gilstate = 1;
+}
+
+static void
+unbind_gilstate_tstate(PyThreadState *tstate)
+{
+ assert(tstate != NULL);
+ // XXX assert(tstate_is_alive(tstate));
+ assert(tstate_is_bound(tstate));
+ // XXX assert(!tstate->_status.active);
+ assert(tstate->_status.bound_gilstate);
+ assert(tstate == gilstate_tss_get(tstate->interp->runtime));
+
+ gilstate_tss_clear(tstate->interp->runtime);
+ tstate->_status.bound_gilstate = 0;
}
@@ -261,7 +331,7 @@ holds_gil(PyThreadState *tstate)
assert(tstate != NULL);
_PyRuntimeState *runtime = tstate->interp->runtime;
/* Must be the tstate for this thread */
- assert(tstate == current_tss_get(runtime));
+ assert(tstate == gilstate_tss_get(runtime));
return tstate == current_fast_get(runtime);
}
@@ -394,10 +464,9 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime)
memcpy(runtime, &initial, sizeof(*runtime));
}
- PyStatus status = current_tss_init(runtime);
- if (_PyStatus_EXCEPTION(status)) {
+ if (gilstate_tss_init(runtime) != 0) {
_PyRuntimeState_Fini(runtime);
- return status;
+ return _PyStatus_NO_MEMORY();
}
if (PyThread_tss_create(&runtime->trashTSSkey) != 0) {
@@ -414,8 +483,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime)
void
_PyRuntimeState_Fini(_PyRuntimeState *runtime)
{
- if (current_tss_initialized(runtime)) {
- current_tss_fini(runtime);
+ if (gilstate_tss_initialized(runtime)) {
+ gilstate_tss_fini(runtime);
}
if (PyThread_tss_is_created(&runtime->trashTSSkey)) {
@@ -475,7 +544,7 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime)
}
- PyStatus status = current_tss_reinit(runtime);
+ PyStatus status = gilstate_tss_reinit(runtime);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
@@ -669,18 +738,38 @@ PyInterpreterState_New(void)
static void
interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
{
+ assert(interp != NULL);
+ assert(tstate != NULL);
_PyRuntimeState *runtime = interp->runtime;
+ /* XXX Conditions we need to enforce:
+
+ * the GIL must be held by the current thread
+ * tstate must be the "current" thread state (current_fast_get())
+ * tstate->interp must be interp
+ * for the main interpreter, tstate must be the main thread
+ */
+ // XXX Ideally, we would not rely on any thread state in this function
+ // (and we would drop the "tstate" argument).
+
if (_PySys_Audit(tstate, "cpython.PyInterpreterState_Clear", NULL) < 0) {
_PyErr_Clear(tstate);
}
HEAD_LOCK(runtime);
+ // XXX Clear the current/main thread state last.
for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) {
PyThreadState_Clear(p);
}
HEAD_UNLOCK(runtime);
+ /* It is possible that any of the objects below have a finalizer
+ that runs Python code or otherwise relies on a thread state
+ or even the interpreter state. For now we trust that isn't
+ a problem.
+ */
+ // XXX Make sure we properly deal with problematic finalizers.
+
Py_CLEAR(interp->audit_hooks);
PyConfig_Clear(&interp->config);
@@ -751,7 +840,6 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
// garbage. It can be different than the current Python thread state
// of 'interp'.
PyThreadState *current_tstate = current_fast_get(interp->runtime);
-
interpreter_clear(interp, current_tstate);
}
@@ -763,29 +851,25 @@ _PyInterpreterState_Clear(PyThreadState *tstate)
}
-static void
-zapthreads(PyInterpreterState *interp, int check_current)
-{
- PyThreadState *tstate;
- /* No need to lock the mutex here because this should only happen
- when the threads are all really dead (XXX famous last words). */
- while ((tstate = interp->threads.head) != NULL) {
- _PyThreadState_Delete(tstate, check_current);
- }
-}
-
+static void zapthreads(PyInterpreterState *interp);
void
PyInterpreterState_Delete(PyInterpreterState *interp)
{
_PyRuntimeState *runtime = interp->runtime;
struct pyinterpreters *interpreters = &runtime->interpreters;
- zapthreads(interp, 0);
- _PyEval_FiniState(&interp->ceval);
+ // XXX Clearing the "current" thread state should happen before
+ // we start finalizing the interpreter (or the current thread state).
+ PyThreadState *tcur = current_fast_get(runtime);
+ if (tcur != NULL && interp == tcur->interp) {
+ /* Unset current thread. After this, many C API calls become crashy. */
+ _PyThreadState_Swap(runtime, NULL);
+ }
+
+ zapthreads(interp);
- /* Delete current thread. After this, many C API calls become crashy. */
- _PyThreadState_Swap(runtime, NULL);
+ _PyEval_FiniState(&interp->ceval);
HEAD_LOCK(runtime);
PyInterpreterState **p;
@@ -843,8 +927,10 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime)
continue;
}
+ // XXX Won't this fail since PyInterpreterState_Clear() requires
+ // the "current" tstate to be set?
PyInterpreterState_Clear(interp); // XXX must activate?
- zapthreads(interp, 1);
+ zapthreads(interp);
if (interp->id_mutex != NULL) {
PyThread_free_lock(interp->id_mutex);
}
@@ -1062,6 +1148,16 @@ _PyInterpreterState_LookUpID(int64_t requested_id)
/* the per-thread runtime state */
/********************************/
+static inline int
+tstate_is_alive(PyThreadState *tstate)
+{
+ return (tstate->_status.initialized &&
+ !tstate->_status.finalized &&
+ !tstate->_status.cleared &&
+ !tstate->_status.finalizing);
+}
+
+
//----------
// lifecycle
//----------
@@ -1112,7 +1208,7 @@ init_threadstate(PyThreadState *tstate,
PyInterpreterState *interp, uint64_t id,
PyThreadState *next)
{
- if (tstate->_status != PyThreadState_UNINITIALIZED) {
+ if (tstate->_status.initialized) {
Py_FatalError("thread state already initialized");
}
@@ -1148,7 +1244,7 @@ init_threadstate(PyThreadState *tstate,
tstate->datastack_top = NULL;
tstate->datastack_limit = NULL;
- tstate->_status = PyThreadState_INITIALIZED;
+ tstate->_status.initialized = 1;
}
static PyThreadState *
@@ -1208,17 +1304,29 @@ PyThreadState_New(PyInterpreterState *interp)
PyThreadState *tstate = new_threadstate(interp);
if (tstate) {
bind_tstate(tstate);
+ // This makes sure there's a gilstate tstate bound
+ // as soon as possible.
+ if (gilstate_tss_get(tstate->interp->runtime) == NULL) {
+ bind_gilstate_tstate(tstate);
+ }
}
return tstate;
}
// This must be followed by a call to _PyThreadState_Bind();
PyThreadState *
-_PyThreadState_Prealloc(PyInterpreterState *interp)
+_PyThreadState_New(PyInterpreterState *interp)
{
return new_threadstate(interp);
}
+// We keep this for stable ABI compabibility.
+PyThreadState *
+_PyThreadState_Prealloc(PyInterpreterState *interp)
+{
+ return _PyThreadState_New(interp);
+}
+
// We keep this around for (accidental) stable ABI compatibility.
// Realistically, no extensions are using it.
void
@@ -1230,6 +1338,17 @@ _PyThreadState_Init(PyThreadState *tstate)
void
PyThreadState_Clear(PyThreadState *tstate)
{
+ assert(tstate->_status.initialized && !tstate->_status.cleared);
+ // XXX assert(!tstate->_status.bound || tstate->_status.unbound);
+ tstate->_status.finalizing = 1; // just in case
+
+ /* XXX Conditions we need to enforce:
+
+ * the GIL must be held by the current thread
+ * current_fast_get()->interp must match tstate->interp
+ * for the main interpreter, current_fast_get() must be the main thread
+ */
+
int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose;
if (verbose && tstate->cframe->current_frame != NULL) {
@@ -1244,6 +1363,17 @@ PyThreadState_Clear(PyThreadState *tstate)
"PyThreadState_Clear: warning: thread still has a frame\n");
}
+ /* At this point tstate shouldn't be used any more,
+ neither to run Python code nor for other uses.
+
+ This is tricky when current_fast_get() == tstate, in the same way
+ as noted in interpreter_clear() above. The below finalizers
+ can possibly run Python code or otherwise use the partially
+ cleared thread state. For now we trust that isn't a problem
+ in practice.
+ */
+ // XXX Deal with the possibility of problematic finalizers.
+
/* Don't clear tstate->pyframe: it is a borrowed reference */
Py_CLEAR(tstate->dict);
@@ -1274,6 +1404,11 @@ PyThreadState_Clear(PyThreadState *tstate)
if (tstate->on_delete != NULL) {
tstate->on_delete(tstate->on_delete_data);
}
+
+ tstate->_status.cleared = 1;
+
+ // XXX Call _PyThreadStateSwap(runtime, NULL) here if "current".
+ // XXX Do it as early in the function as possible.
}
@@ -1281,7 +1416,8 @@ PyThreadState_Clear(PyThreadState *tstate)
static void
tstate_delete_common(PyThreadState *tstate)
{
- _Py_EnsureTstateNotNULL(tstate);
+ assert(tstate->_status.cleared && !tstate->_status.finalized);
+
PyInterpreterState *interp = tstate->interp;
if (interp == NULL) {
Py_FatalError("NULL interpreter");
@@ -1300,7 +1436,11 @@ tstate_delete_common(PyThreadState *tstate)
}
HEAD_UNLOCK(runtime);
- // XXX Do this in PyThreadState_Swap() (and assert not-equal here)?
+ // XXX Unbind in PyThreadState_Clear(), or earlier
+ // (and assert not-equal here)?
+ if (tstate->_status.bound_gilstate) {
+ unbind_gilstate_tstate(tstate);
+ }
unbind_tstate(tstate);
// XXX Move to PyThreadState_Clear()?
@@ -1311,25 +1451,32 @@ tstate_delete_common(PyThreadState *tstate)
_PyObject_VirtualFree(chunk, chunk->size);
chunk = prev;
}
+
+ tstate->_status.finalized = 1;
}
+
static void
-_PyThreadState_Delete(PyThreadState *tstate, int check_current)
+zapthreads(PyInterpreterState *interp)
{
- if (check_current) {
- if (tstate == current_fast_get(tstate->interp->runtime)) {
- _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate);
- }
+ PyThreadState *tstate;
+ /* No need to lock the mutex here because this should only happen
+ when the threads are all really dead (XXX famous last words). */
+ while ((tstate = interp->threads.head) != NULL) {
+ tstate_verify_not_active(tstate);
+ tstate_delete_common(tstate);
+ free_threadstate(tstate);
}
- tstate_delete_common(tstate);
- free_threadstate(tstate);
}
void
PyThreadState_Delete(PyThreadState *tstate)
{
- _PyThreadState_Delete(tstate, 1);
+ _Py_EnsureTstateNotNULL(tstate);
+ tstate_verify_not_active(tstate);
+ tstate_delete_common(tstate);
+ free_threadstate(tstate);
}
@@ -1359,9 +1506,11 @@ PyThreadState_DeleteCurrent(void)
* be kept in those other interpreters.
*/
void
-_PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate)
+_PyThreadState_DeleteExcept(PyThreadState *tstate)
{
+ assert(tstate != NULL);
PyInterpreterState *interp = tstate->interp;
+ _PyRuntimeState *runtime = interp->runtime;
HEAD_LOCK(runtime);
/* Remove all thread states, except tstate, from the linked list of
@@ -1460,6 +1609,38 @@ PyThreadState_GetID(PyThreadState *tstate)
}
+static inline void
+tstate_activate(PyThreadState *tstate)
+{
+ assert(tstate != NULL);
+ // XXX assert(tstate_is_alive(tstate));
+ assert(tstate_is_bound(tstate));
+ assert(!tstate->_status.active);
+
+ assert(!tstate->_status.bound_gilstate ||
+ tstate == gilstate_tss_get((tstate->interp->runtime)));
+ if (!tstate->_status.bound_gilstate) {
+ bind_gilstate_tstate(tstate);
+ }
+
+ tstate->_status.active = 1;
+}
+
+static inline void
+tstate_deactivate(PyThreadState *tstate)
+{
+ assert(tstate != NULL);
+ // XXX assert(tstate_is_alive(tstate));
+ assert(tstate_is_bound(tstate));
+ assert(tstate->_status.active);
+
+ tstate->_status.active = 0;
+
+ // We do not unbind the gilstate tstate here.
+ // It will still be used in PyGILState_Ensure().
+}
+
+
//----------
// other API
//----------
@@ -1535,31 +1716,43 @@ PyThreadState_Get(void)
PyThreadState *
_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts)
{
+#if defined(Py_DEBUG)
+ /* This can be called from PyEval_RestoreThread(). Similar
+ to it, we need to ensure errno doesn't change.
+ */
+ int err = errno;
+#endif
PyThreadState *oldts = current_fast_get(runtime);
- if (newts == NULL) {
- current_fast_clear(runtime);
+ current_fast_clear(runtime);
+
+ if (oldts != NULL) {
+ // XXX assert(tstate_is_alive(oldts) && tstate_is_bound(oldts));
+ tstate_deactivate(oldts);
}
- else {
+
+ if (newts != NULL) {
+ // XXX assert(tstate_is_alive(newts));
+ assert(tstate_is_bound(newts));
current_fast_set(runtime, newts);
+ tstate_activate(newts);
}
+
/* It should not be possible for more than one thread state
to be used for a thread. Check this the best we can in debug
builds.
*/
// XXX The above isn't true when multiple interpreters are involved.
#if defined(Py_DEBUG)
- if (newts && current_tss_initialized(runtime)) {
- /* This can be called from PyEval_RestoreThread(). Similar
- to it, we need to ensure errno doesn't change.
- */
- int err = errno;
- PyThreadState *check = current_tss_get(runtime);
- if (check && check->interp == newts->interp && check != newts) {
- Py_FatalError("Invalid thread state for this thread");
+ if (newts && gilstate_tss_initialized(runtime)) {
+ PyThreadState *check = gilstate_tss_get(runtime);
+ if (check != newts) {
+ if (check && check->interp == newts->interp) {
+ Py_FatalError("Invalid thread state for this thread");
+ }
}
- errno = err;
}
+ errno = err;
#endif
return oldts;
}
@@ -1575,6 +1768,11 @@ void
_PyThreadState_Bind(PyThreadState *tstate)
{
bind_tstate(tstate);
+ // This makes sure there's a gilstate tstate bound
+ // as soon as possible.
+ if (gilstate_tss_get(tstate->interp->runtime) == NULL) {
+ bind_gilstate_tstate(tstate);
+ }
}
@@ -1863,7 +2061,7 @@ _PyGILState_Init(PyInterpreterState *interp)
return _PyStatus_OK();
}
_PyRuntimeState *runtime = interp->runtime;
- assert(current_tss_get(runtime) == NULL);
+ assert(gilstate_tss_get(runtime) == NULL);
assert(runtime->gilstate.autoInterpreterState == NULL);
runtime->gilstate.autoInterpreterState = interp;
return _PyStatus_OK();
@@ -1899,7 +2097,7 @@ _PyGILState_SetTstate(PyThreadState *tstate)
_PyRuntimeState *runtime = tstate->interp->runtime;
assert(runtime->gilstate.autoInterpreterState == tstate->interp);
- assert(current_tss_get(runtime) == tstate);
+ assert(gilstate_tss_get(runtime) == tstate);
assert(tstate->gilstate_counter == 1);
#endif
@@ -1918,10 +2116,10 @@ PyThreadState *
PyGILState_GetThisThreadState(void)
{
_PyRuntimeState *runtime = &_PyRuntime;
- if (!current_tss_initialized(runtime)) {
+ if (!gilstate_tss_initialized(runtime)) {
return NULL;
}
- return current_tss_get(runtime);
+ return gilstate_tss_get(runtime);
}
int
@@ -1932,7 +2130,7 @@ PyGILState_Check(void)
return 1;
}
- if (!current_tss_initialized(runtime)) {
+ if (!gilstate_tss_initialized(runtime)) {
return 1;
}
@@ -1941,7 +2139,7 @@ PyGILState_Check(void)
return 0;
}
- return (tstate == current_tss_get(runtime));
+ return (tstate == gilstate_tss_get(runtime));
}
PyGILState_STATE
@@ -1957,17 +2155,19 @@ PyGILState_Ensure(void)
/* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been
called by Py_Initialize() */
assert(_PyEval_ThreadsInitialized(runtime));
- assert(current_tss_initialized(runtime));
+ assert(gilstate_tss_initialized(runtime));
assert(runtime->gilstate.autoInterpreterState != NULL);
- PyThreadState *tcur = current_tss_get(runtime);
+ PyThreadState *tcur = gilstate_tss_get(runtime);
int has_gil;
if (tcur == NULL) {
/* Create a new Python thread state for this thread */
- tcur = PyThreadState_New(runtime->gilstate.autoInterpreterState);
+ tcur = new_threadstate(runtime->gilstate.autoInterpreterState);
if (tcur == NULL) {
Py_FatalError("Couldn't create thread-state for new thread");
}
+ bind_tstate(tcur);
+ bind_gilstate_tstate(tcur);
/* This is our thread state! We'll need to delete it in the
matching call to PyGILState_Release(). */
@@ -1997,7 +2197,7 @@ void
PyGILState_Release(PyGILState_STATE oldstate)
{
_PyRuntimeState *runtime = &_PyRuntime;
- PyThreadState *tstate = current_tss_get(runtime);
+ PyThreadState *tstate = gilstate_tss_get(runtime);
if (tstate == NULL) {
Py_FatalError("auto-releasing thread-state, "
"but no thread-state for this thread");
@@ -2023,6 +2223,7 @@ PyGILState_Release(PyGILState_STATE oldstate)
if (tstate->gilstate_counter == 0) {
/* can't have been locked when we created it */
assert(oldstate == PyGILState_UNLOCKED);
+ // XXX Unbind tstate here.
PyThreadState_Clear(tstate);
/* Delete the thread-state. Note this releases the GIL too!
* It's vital that the GIL be held here, to avoid shutdown
More information about the Python-checkins
mailing list