From 43b07983632f9cea820f842be5709db77c668937 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 15:49:05 -0400 Subject: [PATCH 01/14] Copy the reference implementation. --- Include/cpython/pystate.h | 16 +- Include/internal/pycore_interp_structs.h | 6 + Include/internal/pycore_pystate.h | 14 + Include/pystate.h | 17 ++ Lib/test/test_embed.py | 10 +- Modules/_testcapimodule.c | 239 +++++++++++++++++ Modules/_testinternalcapi.c | 53 ++++ Programs/_testembed.c | 129 +++++++++ Python/pylifecycle.c | 47 +++- Python/pystate.c | 317 +++++++++++++++++++++-- Tools/c-analyzer/cpython/ignored.tsv | 3 + 11 files changed, 825 insertions(+), 26 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0cb57679df331d..a9d97e47e005df 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -105,7 +105,7 @@ struct _ts { # define _PyThreadState_WHENCE_INIT 1 # define _PyThreadState_WHENCE_FINI 2 # define _PyThreadState_WHENCE_THREADING 3 -# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_C_API 4 # define _PyThreadState_WHENCE_EXEC 5 # define _PyThreadState_WHENCE_THREADING_DAEMON 6 #endif @@ -239,6 +239,20 @@ struct _ts { // structure and all share the same per-interpreter structure). PyStats *pystats; #endif + + struct { + /* Number of nested PyThreadState_Ensure() calls on this thread state */ + Py_ssize_t counter; + + /* Should this thread state be deleted upon calling + PyThreadState_Release() (with the counter at 1)? + + This is only true for thread states created by PyThreadState_Ensure() */ + int delete_on_release; + + /* The interpreter guard owned by PyThreadState_EnsureFromView(), if any. */ + PyInterpreterGuard *owned_guard; + } ensure; }; /* other API */ diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 01adadd1485189..2eb4cf2fc77510 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -1053,6 +1053,12 @@ struct _is { #endif #endif + struct { + _PyRWMutex lock; + Py_ssize_t countdown; + PyEvent done; + } finalization_guards; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 189a8dde9f09ed..fbe49487d224d2 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -338,6 +338,20 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) #endif } +/* PEP 788 structures. */ + +struct _PyInterpreterGuard { + PyInterpreterState *interp; +}; + +struct _PyInterpreterView { + int64_t id; +}; + +// Exports for '_testinternalcapi' shared extension +PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GuardCountdown(PyInterpreterState *interp); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterGuard_GetInterpreter(PyInterpreterGuard *guard); + #ifdef __cplusplus } #endif diff --git a/Include/pystate.h b/Include/pystate.h index 727b8fbfffe0e6..254702d43e8728 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -120,6 +120,23 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); +/* PEP 788 -- Interpreter guards and views. */ + +typedef struct _PyInterpreterGuard PyInterpreterGuard; +typedef struct _PyInterpreterView PyInterpreterView; + +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterGuard_Close(PyInterpreterGuard *guard); +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromView(PyInterpreterView *view); + +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterView_Close(PyInterpreterView *view); +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromMain(void); + +PyAPI_FUNC(PyThreadState *) PyThreadState_Ensure(PyInterpreterGuard *guard); +PyAPI_FUNC(PyThreadState *) PyThreadState_EnsureFromView(PyInterpreterView *view); +PyAPI_FUNC(void) PyThreadState_Release(PyThreadState *tstate); + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1087cbd0836fd8..831aae9f264519 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1993,10 +1993,18 @@ def test_audit_run_stdin(self): def test_get_incomplete_frame(self): self.run_embedded_interpreter("test_get_incomplete_frame") - def test_gilstate_after_finalization(self): self.run_embedded_interpreter("test_gilstate_after_finalization") + def test_thread_state_ensure(self): + self.run_embedded_interpreter("test_thread_state_ensure") + + def test_main_interpreter_view(self): + self.run_embedded_interpreter("test_main_interpreter_view") + + def test_thread_state_ensure_from_view(self): + self.run_embedded_interpreter("test_thread_state_ensure_from_view") + class MiscTests(EmbeddingTestsMixin, unittest.TestCase): def test_unicode_id_init(self): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3ebe4ceea6a72e..feb651e269b68b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2606,6 +2606,240 @@ create_managed_weakref_nogc_type(PyObject *self, PyObject *Py_UNUSED(args)) return PyType_FromSpec(&ManagedWeakrefNoGC_spec); } +static void +test_interp_guards_common(void) +{ + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + assert(guard != NULL); + + PyInterpreterGuard *guard_2 = PyInterpreterGuard_FromCurrent(); + assert(guard_2 != NULL); + + // We can close the guards in any order + PyInterpreterGuard_Close(guard_2); + PyInterpreterGuard_Close(guard); +} + +static PyObject * +test_interpreter_guards(PyObject *self, PyObject *unused) +{ + // Test the main interpreter + test_interp_guards_common(); + + // Test a (legacy) subinterpreter + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + PyThreadState *interp_tstate = Py_NewInterpreter(); + test_interp_guards_common(); + Py_EndInterpreter(interp_tstate); + + // Test an isolated subinterpreter + PyInterpreterConfig config = { + .gil = PyInterpreterConfig_OWN_GIL, + .check_multi_interp_extensions = 1 + }; + + PyThreadState *isolated_interp_tstate; + PyStatus status = Py_NewInterpreterFromConfig(&isolated_interp_tstate, &config); + if (PyStatus_Exception(status)) { + PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + return NULL; + } + + test_interp_guards_common(); + Py_EndInterpreter(isolated_interp_tstate); + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_thread_state_ensure_nested(PyObject *self, PyObject *unused) +{ + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + if (guard == NULL) { + return NULL; + } + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + assert(PyGILState_GetThisThreadState() == save_tstate); + PyThreadState *thread_states[10]; + + for (int i = 0; i < 10; ++i) { + // Test reactivation of the detached tstate. + thread_states[i] = PyThreadState_Ensure(guard); + if (thread_states[i] == 0) { + PyInterpreterGuard_Close(guard); + return PyErr_NoMemory(); + } + + // No new thread state should've been created. + assert(PyThreadState_Get() == save_tstate); + PyThreadState_Release(thread_states[i]); + } + + assert(PyThreadState_GetUnchecked() == NULL); + + // Similarly, test ensuring with deep nesting and *then* releasing. + // If the (detached) gilstate matches the interpreter, then it shouldn't + // create a new thread state. + for (int i = 0; i < 10; ++i) { + thread_states[i] = PyThreadState_Ensure(guard); + if (thread_states[i] == 0) { + // This will technically leak other thread states, but it doesn't + // matter because this is a test. + PyInterpreterGuard_Close(guard); + return PyErr_NoMemory(); + } + + assert(PyThreadState_Get() == save_tstate); + } + + for (int i = 0; i < 10; ++i) { + assert(PyThreadState_Get() == save_tstate); + PyThreadState_Release(thread_states[i]); + } + + assert(PyThreadState_GetUnchecked() == NULL); + PyInterpreterGuard_Close(guard); + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_thread_state_ensure_crossinterp(PyObject *self, PyObject *unused) +{ + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + PyThreadState *interp_tstate = Py_NewInterpreter(); + assert(interp_tstate != NULL); + + /* This should create a new thread state for the calling interpreter, *not* + reactivate the old one. In a real-world scenario, this would arise in + something like this: + + def some_func(): + import something + # This re-enters the main interpreter, but we + # shouldn't have access to prior thread-locals. + something.call_something() + + interp = interpreters.create() + interp.exec(some_func) + */ + PyThreadState *thread_state = PyThreadState_Ensure(guard); + assert(thread_state != NULL); + + PyThreadState *ensured_tstate = PyThreadState_Get(); + assert(ensured_tstate != save_tstate); + assert(PyGILState_GetThisThreadState() == ensured_tstate); + + // Now though, we should reactivate the thread state + PyThreadState *other_thread_state = PyThreadState_Ensure(guard); + assert(other_thread_state != NULL); + assert(PyThreadState_Get() == ensured_tstate); + + PyThreadState_Release(other_thread_state); + + // Ensure that we're restoring the prior thread state + PyThreadState_Release(thread_state); + assert(PyThreadState_Get() == interp_tstate); + assert(PyGILState_GetThisThreadState() == interp_tstate); + + PyThreadState_Swap(interp_tstate); + Py_EndInterpreter(interp_tstate); + + PyInterpreterGuard_Close(guard); + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_interp_view_after_shutdown(PyObject *self, PyObject *unused) +{ + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + PyThreadState *interp_tstate = Py_NewInterpreter(); + if (interp_tstate == NULL) { + PyThreadState_Swap(save_tstate); + return PyErr_NoMemory(); + } + + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + if (view == NULL) { + Py_EndInterpreter(interp_tstate); + PyThreadState_Swap(save_tstate); + return PyErr_NoMemory(); + } + + // As a sanity check, ensure that the view actually works + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + PyInterpreterGuard_Close(guard); + + // Now, destroy the interpreter and try to acquire a lock from a view. + // It should fail. + Py_EndInterpreter(interp_tstate); + guard = PyInterpreterGuard_FromView(view); + assert(guard == NULL); + + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_thread_state_ensure_view(PyObject *self, PyObject *unused) +{ + // For simplicity's sake, we assume that functions won't fail due to being + // out of memory. + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + PyThreadState *interp_tstate = Py_NewInterpreter(); + assert(interp_tstate != NULL); + assert(PyInterpreterState_Get() == PyThreadState_GetInterpreter(interp_tstate)); + + PyInterpreterView *main_view = PyInterpreterView_FromMain(); + assert(main_view != NULL); + + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + assert(view != NULL); + + Py_BEGIN_ALLOW_THREADS; + PyThreadState *tstate = PyThreadState_EnsureFromView(view); + assert(tstate != NULL); + assert(PyThreadState_Get() == interp_tstate); + + // Test a nested call + PyThreadState *tstate2 = PyThreadState_EnsureFromView(view); + assert(PyThreadState_Get() == interp_tstate); + + // We're in a new interpreter now. PyThreadState_EnsureFromView() should + // now create a new thread state. + PyThreadState *main_tstate = PyThreadState_EnsureFromView(main_view); + assert(main_tstate == interp_tstate); // The old thread state + assert(PyInterpreterState_Get() == PyInterpreterState_Main()); + + // Going back to the old interpreter should create a new thread state again. + PyThreadState *tstate3 = PyThreadState_EnsureFromView(view); + assert(PyInterpreterState_Get() == PyThreadState_GetInterpreter(interp_tstate)); + assert(PyThreadState_Get() != interp_tstate); + PyThreadState_Release(tstate3); + PyThreadState_Release(main_tstate); + + // We're back in the original interpreter. PyThreadState_EnsureFromView() should + // no longer create a new thread state. + assert(PyThreadState_Get() == interp_tstate); + PyThreadState *tstate4 = PyThreadState_EnsureFromView(view); + assert(PyThreadState_Get() == interp_tstate); + PyThreadState_Release(tstate4); + PyThreadState_Release(tstate2); + PyThreadState_Release(tstate); + assert(PyThreadState_GetUnchecked() == NULL); + Py_END_ALLOW_THREADS; + + assert(PyThreadState_Get() == interp_tstate); + PyInterpreterView_Close(view); + PyInterpreterView_Close(main_view); + Py_EndInterpreter(interp_tstate); + PyThreadState_Swap(save_tstate); + + Py_RETURN_NONE; +} + static PyObject* test_soft_deprecated_macros(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) @@ -2740,6 +2974,11 @@ static PyMethodDef TestMethods[] = { {"create_managed_weakref_nogc_type", create_managed_weakref_nogc_type, METH_NOARGS}, {"test_soft_deprecated_macros", test_soft_deprecated_macros, METH_NOARGS}, + {"test_interpreter_lock", test_interpreter_guards, METH_NOARGS}, + {"test_thread_state_ensure_nested", test_thread_state_ensure_nested, METH_NOARGS}, + {"test_thread_state_ensure_crossinterp", test_thread_state_ensure_crossinterp, METH_NOARGS}, + {"test_interp_view_after_shutdown", test_interp_view_after_shutdown, METH_NOARGS}, + {"test_thread_state_ensure_view", test_thread_state_ensure_view, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 619f9f50574429..a403e07232df65 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2883,6 +2883,57 @@ test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +#define NUM_GUARDS 100 + +static PyObject * +test_interp_guard_countdown(PyObject *self, PyObject *unused) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + assert(_PyInterpreterState_GuardCountdown(interp) == 0); + PyInterpreterGuard *guards[NUM_GUARDS]; + for (int i = 0; i < NUM_GUARDS; ++i) { + guards[i] = PyInterpreterGuard_FromCurrent(); + assert(guards[i] != 0); + assert(_PyInterpreterState_GuardCountdown(interp) == i + 1); + } + + for (int i = 0; i < NUM_GUARDS; ++i) { + PyInterpreterGuard_Close(guards[i]); + assert(_PyInterpreterState_GuardCountdown(interp) == (NUM_GUARDS - i - 1)); + } + + Py_RETURN_NONE; +} + +static PyObject * +test_interp_view_countdown(PyObject *self, PyObject *unused) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + if (view == NULL) { + return NULL; + } + assert(_PyInterpreterState_GuardCountdown(interp) == 0); + + PyInterpreterGuard *guards[NUM_GUARDS]; + + for (int i = 0; i < NUM_GUARDS; ++i) { + guards[i] = PyInterpreterGuard_FromView(view); + assert(guards[i] != 0); + assert(_PyInterpreterGuard_GetInterpreter(guards[i]) == interp); + assert(_PyInterpreterState_GuardCountdown(interp) == i + 1); + } + + for (int i = 0; i < NUM_GUARDS; ++i) { + PyInterpreterGuard_Close(guards[i]); + assert(_PyInterpreterState_GuardCountdown(interp) == (NUM_GUARDS - i - 1)); + } + + PyInterpreterView_Close(view); + Py_RETURN_NONE; +} + +#undef NUM_LOCKS static PyObject * _pyerr_setkeyerror(PyObject *self, PyObject *arg) @@ -3021,6 +3072,8 @@ static PyMethodDef module_functions[] = { {"test_threadstate_set_stack_protection", test_threadstate_set_stack_protection, METH_NOARGS}, {"_pyerr_setkeyerror", _pyerr_setkeyerror, METH_O}, + {"test_interp_guard_countdown", test_interp_guard_countdown, METH_NOARGS}, + {"test_interp_view_countdown", test_interp_view_countdown, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 285f4f091b2f7a..8575fb30c95a8d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2670,6 +2670,132 @@ test_gilstate_after_finalization(void) return PyThread_detach_thread(handle); } + +const char *THREAD_CODE = \ + "import time\n" + "time.sleep(0.2)\n" + "def fib(n):\n" + " if n <= 1:\n" + " return n\n" + " else:\n" + " return fib(n - 1) + fib(n - 2)\n" + "fib(10)"; + +typedef struct { + void *argument; + int done; + PyEvent event; +} ThreadData; + +static void +do_tstate_ensure(void *arg) +{ + ThreadData *data = (ThreadData *)arg; + PyThreadState *tstates[4]; + PyInterpreterGuard *guard = data->argument; + tstates[0] = PyThreadState_Ensure(guard); + tstates[1] = PyThreadState_Ensure(guard); + tstates[2] = PyThreadState_Ensure(guard); + PyGILState_STATE gstate = PyGILState_Ensure(); + tstates[3] = PyThreadState_Ensure(guard); + assert(tstates[0] != NULL); + assert(tstates[1] != NULL); + assert(tstates[2] != NULL); + assert(tstates[3] != NULL); + int res = PyRun_SimpleString(THREAD_CODE); + assert(res == 0); + PyThreadState_Release(tstates[3]); + PyGILState_Release(gstate); + PyThreadState_Release(tstates[2]); + PyThreadState_Release(tstates[1]); + PyThreadState_Release(tstates[0]); + PyInterpreterGuard_Close(guard); + data->done = 1; +} + +static int +test_thread_state_ensure(void) +{ + _testembed_initialize(); + PyThread_handle_t handle; + PyThread_ident_t ident; + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + assert(guard != NULL); + ThreadData data = { guard }; + if (PyThread_start_joinable_thread(do_tstate_ensure, &data, + &ident, &handle) < 0) { + PyInterpreterGuard_Close(guard); + return -1; + } + // We hold an interpreter guard, so we don't + // have to worry about the interpreter shutting down before + // we finalize. + Py_Finalize(); + assert(data.done == 1); + return 0; +} + +static int +test_main_interpreter_view(void) +{ + _testembed_initialize(); + + // Main interpreter is initialized and ready. + PyInterpreterView *view = PyInterpreterView_FromMain(); + assert(view != NULL); + + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + assert(guard != NULL); + PyInterpreterGuard_Close(guard); + + Py_Finalize(); + + // We shouldn't be able to get locks for the interpreter now + guard = PyInterpreterGuard_FromView(view); + assert(guard == NULL); + + PyInterpreterView_Close(view); + + return 0; +} + +static void +do_tstate_ensure_from_view(void *arg) +{ + ThreadData *data = (ThreadData *)arg; + PyInterpreterView *view = data->argument; + assert(view != NULL); + PyThreadState *tstate = PyThreadState_EnsureFromView(view); + assert(tstate != NULL); + _PyEvent_Notify(&data->event); + int res = PyRun_SimpleString(THREAD_CODE); + assert(res == 0); + data->done = 1; + PyThreadState_Release(tstate); +} + +static int +test_thread_state_ensure_from_view(void) +{ + _testembed_initialize(); + PyThread_handle_t handle; + PyThread_ident_t ident; + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + assert(view != NULL); + + ThreadData data = { view }; + if (PyThread_start_joinable_thread(do_tstate_ensure_from_view, &data, + &ident, &handle) < 0) { + PyInterpreterView_Close(view); + return -1; + } + + PyEvent_Wait(&data.event); + Py_Finalize(); + assert(data.done == 1); + return 0; +} + /* ********************************************************* * List of test cases and the function that implements it. * @@ -2764,6 +2890,9 @@ static struct TestCase TestCases[] = { {"test_create_module_from_initfunc", test_create_module_from_initfunc}, {"test_inittab_submodule_multiphase", test_inittab_submodule_multiphase}, {"test_inittab_submodule_singlephase", test_inittab_submodule_singlephase}, + {"test_thread_state_ensure", test_thread_state_ensure}, + {"test_main_interpreter_view", test_main_interpreter_view}, + {"test_thread_state_ensure_from_view", test_thread_state_ensure_from_view}, {NULL, NULL} }; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0a88e32bb6b65e..57dd57fa586c79 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2300,16 +2300,37 @@ make_pre_finalization_calls(PyThreadState *tstate, int subinterpreters) if (subinterpreters) { /* Clean up any lingering subinterpreters. - - Two preconditions need to be met here: - - - This has to happen before _PyRuntimeState_SetFinalizing is - called, or else threads might get prematurely blocked. - - The world must not be stopped, as finalizers can run. - */ + * Two preconditions need to be met here: + * 1. This has to happen before _PyRuntimeState_SetFinalizing is + * called, or else threads might get prematurely blocked. + * 2. The world must not be stopped, as finalizers can run. + */ finalize_subinterpreters(); } + /* Wait on finalization guards. + * + * To avoid eating CPU cycles, we use an event to signal when we reach + * zero remaining guards. But, this isn't atomic! This event can be reset + * later if another thread creates a new finalization guard. The actual + * atomic check is made below, when we hold the finalization guard lock. + * Again, this is purely an optimization to avoid overloading the CPU. + */ + if (_Py_atomic_load_ssize_relaxed(&interp->finalization_guards.countdown) > 0) { + for (;;) { + PyTime_t wait_ns = 1000 * 1000; // 1ms + if (PyEvent_WaitTimed(&interp->finalization_guards.done, wait_ns, /*detach=*/1)) { + break; + } + + // For debugging purposes, we emit a fatal error if someone + // CTRL^C'ed the process. + if (PyErr_CheckSignals()) { + PyErr_FormatUnraisable("Exception ignored while waiting on finalization guards"); + Py_FatalError("Interrupted while waiting on finalization guard"); + } + } + } /* Stop the world to prevent other threads from creating threads or * atexit callbacks. On the default build, this is simply locked by @@ -2321,18 +2342,26 @@ make_pre_finalization_calls(PyThreadState *tstate, int subinterpreters) // XXX Why does _PyThreadState_DeleteList() rely on all interpreters // being stopped? _PyEval_StopTheWorldAll(interp->runtime); + _PyRWMutex_Lock(&interp->finalization_guards.lock); int has_subinterpreters = subinterpreters ? runtime_has_subinterpreters(interp->runtime) : 0; + // TODO: The interpreter guard countdown isn't very efficient. We should + // wait on an event or something like that. int should_continue = (interp_has_threads(interp) || interp_has_atexit_callbacks(interp) || interp_has_pending_calls(interp) - || has_subinterpreters); + || has_subinterpreters + || _Py_atomic_load_ssize_acquire(&interp->finalization_guards.countdown) > 0); if (!should_continue) { break; } + // Temporarily let other threads execute + _PyThreadState_Detach(tstate); + _PyRWMutex_Unlock(&interp->finalization_guards.lock); _PyEval_StartTheWorldAll(interp->runtime); PyMutex_Unlock(&interp->ceval.pending.mutex); + _PyThreadState_Attach(tstate); } assert(PyMutex_IsLocked(&interp->ceval.pending.mutex)); ASSERT_WORLD_STOPPED(interp); @@ -2393,6 +2422,7 @@ _Py_Finalize(_PyRuntimeState *runtime) for (PyThreadState *p = list; p != NULL; p = p->next) { _PyThreadState_SetShuttingDown(p); } + _PyRWMutex_Unlock(&tstate->interp->finalization_guards.lock); _PyEval_StartTheWorldAll(runtime); PyMutex_Unlock(&tstate->interp->ceval.pending.mutex); @@ -2775,6 +2805,7 @@ Py_EndInterpreter(PyThreadState *tstate) _PyThreadState_SetShuttingDown(p); } + _PyRWMutex_Unlock(&interp->finalization_guards.lock); _PyEval_StartTheWorldAll(interp->runtime); PyMutex_Unlock(&interp->ceval.pending.mutex); _PyThreadState_DeleteList(list, /*is_after_fork=*/0); diff --git a/Python/pystate.c b/Python/pystate.c index 2df24597e65785..fc3e5ea7db00c2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1610,6 +1610,7 @@ static void add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, PyThreadState *next) { + assert(interp != NULL); assert(interp->threads.head != tstate); if (next != NULL) { assert(next->prev == NULL || next->prev == tstate); @@ -2889,34 +2890,38 @@ PyGILState_Check(void) return (tstate == tcur); } +static PyInterpreterGuard * +get_main_interp_guard(void) +{ + PyInterpreterView *view = PyInterpreterView_FromMain(); + if (view == NULL) { + return NULL; + } + + return PyInterpreterGuard_FromView(view); +} + PyGILState_STATE PyGILState_Ensure(void) { - _PyRuntimeState *runtime = &_PyRuntime; - /* Note that we do not auto-init Python here - apart from potential races with 2 threads auto-initializing, pep-311 spells out other issues. Embedders are expected to have called Py_Initialize(). */ - /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been - called by Py_Initialize() - - TODO: This isn't thread-safe. There's no protection here against - concurrent finalization of the interpreter; it's simply a guard - for *after* the interpreter has finalized. - */ - if (!_PyEval_ThreadsInitialized() || runtime->gilstate.autoInterpreterState == NULL) { - PyThread_hang_thread(); - } - PyThreadState *tcur = gilstate_get(); int has_gil; if (tcur == NULL) { /* Create a new Python thread state for this thread */ - // XXX Use PyInterpreterState_EnsureThreadState()? - tcur = new_threadstate(runtime->gilstate.autoInterpreterState, - _PyThreadState_WHENCE_GILSTATE); + PyInterpreterGuard *guard = get_main_interp_guard(); + if (guard == NULL) { + // The main interpreter has finished, so we don't have + // any intepreter to make a thread state for. Hang the + // thread to act as failure. + PyThread_hang_thread(); + } + tcur = new_threadstate(guard->interp, + _PyThreadState_WHENCE_C_API); if (tcur == NULL) { Py_FatalError("Couldn't create thread-state for new thread"); } @@ -2928,6 +2933,7 @@ PyGILState_Ensure(void) assert(tcur->gilstate_counter == 1); tcur->gilstate_counter = 0; has_gil = 0; /* new thread state is never current */ + PyInterpreterGuard_Close(guard); } else { has_gil = holds_gil(tcur); @@ -3309,3 +3315,282 @@ _Py_GetMainConfig(void) } return _PyInterpreterState_GetConfig(interp); } + +Py_ssize_t +_PyInterpreterState_GuardCountdown(PyInterpreterState *interp) +{ + assert(interp != NULL); + Py_ssize_t count = _Py_atomic_load_ssize_relaxed(&interp->finalization_guards.countdown); + assert(count >= 0); + return count; +} + +PyInterpreterState * +_PyInterpreterGuard_GetInterpreter(PyInterpreterGuard *guard) +{ + assert(guard != NULL); + assert(guard->interp != NULL); + return guard->interp; +} + +static int +try_acquire_interp_guard(PyInterpreterState *interp, PyInterpreterGuard *guard) +{ + assert(interp != NULL); + _PyRWMutex_RLock(&interp->finalization_guards.lock); + + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { + _PyRWMutex_RUnlock(&interp->finalization_guards.lock); + assert(_Py_atomic_load_ssize_relaxed(&interp->finalization_guards.countdown) == 0); + return -1; + } + + Py_ssize_t old_value = _Py_atomic_add_ssize(&interp->finalization_guards.countdown, 1); + if (old_value == 0) { + // Reset the event. + // We first have to notify the finalization thread if it's waiting on us, but + // it will get trapped waiting on the RW lock. When it goes to check + // again after we release the lock, it will see that the countdown is + // non-zero and begin waiting again (hence why we need to reset the + // event). + _PyEvent_Notify(&interp->finalization_guards.done); + memset(&interp->finalization_guards.done, 0, sizeof(PyEvent)); + } + _PyRWMutex_RUnlock(&interp->finalization_guards.lock); + + guard->interp = interp; + return 0; +} + +PyInterpreterGuard * +PyInterpreterGuard_FromCurrent(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp != NULL); + + PyInterpreterGuard *guard = PyMem_RawMalloc(sizeof(PyInterpreterGuard)); + if (guard == NULL) { + PyErr_NoMemory(); + return NULL; + } + + if (try_acquire_interp_guard(interp, guard) < 0) { + PyMem_RawFree(guard); + PyErr_SetString(PyExc_PythonFinalizationError, + "cannot acquire finalization guard anymore"); + return NULL; + } + + return guard; +} + +void +PyInterpreterGuard_Close(PyInterpreterGuard *guard) +{ + PyInterpreterState *interp = guard->interp; + assert(interp != NULL); + + _PyRWMutex_RLock(&interp->finalization_guards.lock); + Py_ssize_t old = _Py_atomic_add_ssize(&interp->finalization_guards.countdown, -1); + if (old == 1) { + _PyEvent_Notify(&interp->finalization_guards.done); + } + _PyRWMutex_RUnlock(&interp->finalization_guards.lock); + + assert(old > 0); + PyMem_RawFree(guard); +} + +PyInterpreterView * +PyInterpreterView_FromCurrent(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp != NULL); + + // PyInterpreterView_Close() can be called without an attached thread + // state, so we have to use the raw allocator. + PyInterpreterView *view = PyMem_RawMalloc(sizeof(PyInterpreterView)); + if (view == NULL) { + PyErr_NoMemory(); + return NULL; + } + + view->id = interp->id; + return view; +} + +void +PyInterpreterView_Close(PyInterpreterView *view) +{ + assert(view != NULL); + PyMem_RawFree(view); +} + +PyInterpreterGuard * +PyInterpreterGuard_FromView(PyInterpreterView *view) +{ + assert(view != NULL); + int64_t interp_id = view->id; + assert(interp_id >= 0); + + // This allocation has to happen before we acquire the runtime lock, because + // PyMem_RawMalloc() might call some weird callback (such as tracemalloc) + // that tries to re-entrantly acquire the lock. + PyInterpreterGuard *guard = PyMem_RawMalloc(sizeof(PyInterpreterGuard)); + if (guard == NULL) { + return NULL; + } + + // Interpreters cannot be deleted while we hold the runtime lock. + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyInterpreterState *interp = interp_look_up_id(runtime, interp_id); + if (interp == NULL) { + HEAD_UNLOCK(runtime); + PyMem_RawFree(guard); + return NULL; + } + + int result = try_acquire_interp_guard(interp, guard); + HEAD_UNLOCK(runtime); + + if (result < 0) { + PyMem_RawFree(guard); + return NULL; + } + + assert(guard == NULL || guard->interp != NULL); + return guard; +} + +PyInterpreterView * +PyInterpreterView_FromMain(void) +{ + PyInterpreterView *view = PyMem_RawMalloc(sizeof(PyInterpreterView)); + if (view == NULL) { + return NULL; + } + + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + view->id = runtime->_main_interpreter.id; + HEAD_UNLOCK(runtime); + + return view; +} + +// This is a bit of a hack -- since NULL is reserved for failure, we need +// to have our own sentinel for when we want to indicate that no prior +// thread state was attached. +// To do this, we just use the memory address of a global variable and +// cast it to a PyThreadState *. +static int NO_TSTATE_SENTINEL = 0; + +PyThreadState * +PyThreadState_Ensure(PyInterpreterGuard *guard) +{ + assert(guard != NULL); + PyInterpreterState *interp = guard->interp; + assert(interp != NULL); + PyThreadState *attached_tstate = current_fast_get(); + if (attached_tstate != NULL && attached_tstate->interp == interp) { + /* Yay! We already have an attached thread state that matches. */ + ++attached_tstate->ensure.counter; + return (PyThreadState *)&NO_TSTATE_SENTINEL; + } + + PyThreadState *detached_gilstate = gilstate_get(); + if (detached_gilstate != NULL && detached_gilstate->interp == interp) { + /* There's a detached thread state that works. */ + assert(attached_tstate == NULL); + ++detached_gilstate->ensure.counter; + _PyThreadState_Attach(detached_gilstate); + return (PyThreadState *)&NO_TSTATE_SENTINEL; + } + + PyThreadState *fresh_tstate = _PyThreadState_NewBound(interp, + _PyThreadState_WHENCE_C_API); + if (fresh_tstate == NULL) { + return NULL; + } + fresh_tstate->ensure.counter = 1; + fresh_tstate->ensure.delete_on_release = 1; + + if (attached_tstate != NULL) { + return PyThreadState_Swap(fresh_tstate); + } else { + _PyThreadState_Attach(fresh_tstate); + } + + return (PyThreadState *)&NO_TSTATE_SENTINEL; +} + +PyThreadState * +PyThreadState_EnsureFromView(PyInterpreterView *view) +{ + assert(view != NULL); + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + if (guard == NULL) { + return NULL; + } + + PyThreadState *result_tstate = PyThreadState_Ensure(guard); + if (result_tstate == NULL) { + PyInterpreterGuard_Close(guard); + return NULL; + } + + PyThreadState *tstate = current_fast_get(); + assert(tstate != NULL); + + if (tstate->ensure.owned_guard != NULL) { + assert(tstate->ensure.owned_guard->interp == guard->interp); + PyInterpreterGuard_Close(guard); + } else { + assert(tstate->ensure.owned_guard == NULL); + tstate->ensure.owned_guard = guard; + } + + return result_tstate; +} + +void +PyThreadState_Release(PyThreadState *old_tstate) +{ + PyThreadState *tstate = current_fast_get(); + _Py_EnsureTstateNotNULL(tstate); + Py_ssize_t remaining = --tstate->ensure.counter; + if (remaining < 0) { + Py_FatalError("PyThreadState_Release() called more times than PyThreadState_Ensure()"); + } + + if (remaining != 0) { + return; + } + + PyThreadState *to_restore; + if (old_tstate == (PyThreadState *)&NO_TSTATE_SENTINEL) { + to_restore = NULL; + } + else { + to_restore = old_tstate; + } + + assert(tstate->ensure.delete_on_release == 1 || tstate->ensure.delete_on_release == 0); + if (tstate->ensure.delete_on_release) { + PyThreadState_Clear(tstate); + } else { + PyThreadState_Swap(to_restore); + } + + PyThreadState_Swap(to_restore); + + if (tstate->ensure.owned_guard != NULL) { + PyInterpreterGuard_Close(tstate->ensure.owned_guard); + tstate->ensure.owned_guard = NULL; + } + + if (tstate->ensure.delete_on_release) { + PyThreadState_Delete(tstate); + } +} diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index d2489387f46caa..2b9a5ee68de713 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -198,6 +198,9 @@ Python/pystate.c - _Py_tss_tstate - Python/pystate.c - _Py_tss_gilstate - Python/pystate.c - _Py_tss_interp - +# Global sentinel that is fine to share across interpreters +Python/pystate.c - NO_TSTATE_SENTINEL - + ##----------------------- ## should be const # XXX Make them const. From 73cccbb1503f4e2b2d0a7628bb0f25c6532d5106 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 16:58:15 -0400 Subject: [PATCH 02/14] Document the new APIs. --- Doc/c-api/interp-lifecycle.rst | 206 ++++++++++++-- Doc/c-api/threads.rst | 474 +++++++++++++++++++++++---------- Programs/_testembed.c | 1 + 3 files changed, 521 insertions(+), 160 deletions(-) diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index 186ab4370bcb9c..9095651554242b 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -578,31 +578,195 @@ Initializing and finalizing the interpreter .. _cautions-regarding-runtime-finalization: -Cautions regarding runtime finalization ---------------------------------------- +Cautions regarding interpreter finalization +------------------------------------------- In the late stage of :term:`interpreter shutdown`, after attempting to wait for non-daemon threads to exit (though this can be interrupted by :class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`Py_IsFinalizing` and -:func:`sys.is_finalizing` return true. At this point, only the *finalization -thread* that initiated finalization (typically the main thread) is allowed to -acquire the :term:`GIL`. - -If any thread, other than the finalization thread, attempts to attach a :term:`thread state` -during finalization, either explicitly or -implicitly, the thread enters **a permanently blocked state** -where it remains until the program exits. In most cases this is harmless, but this can result -in deadlock if a later stage of finalization attempts to acquire a lock owned by the -blocked thread, or otherwise waits on the blocked thread. - -Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ -finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs -have never had any error reporting or handling expectations at :term:`thread state` -attachment time that would've allowed for graceful exit from this situation. Changing that -would require new stable C APIs and rewriting the majority of C code in the -CPython ecosystem to use those with error handling. +is marked as finalizing, meaning that :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the finalization +thread (the thread that initiated finalization; this is typically the main thread) +is allowed to :term:`attach ` a thread state. + +Other threads that attempt to attach during finalization, either explicitly +(such as via :c:func:`PyThreadState_Ensure` or :c:macro:`Py_END_ALLOW_THREADS`) +or implicitly (such as in-between bytecode instructions), will enter a +**permanently blocked state**. Generally, this is harmless, but this can +result in deadlocks. For example, a thread may be permanently blocked while +holding a lock, meaning that the finalization thread can never acquire that +lock. + +Prior to CPython 3.13, the thread would exit instead of hanging, +which led to other issues (see the warning note at +:c:func:`PyThread_exit_thread`). + +Gross? Yes. Starting in Python 3.15, there are a number of C APIs that make +it possible to avoid these issues by temporarily preventing finalization: + +.. seealso:: + + :pep:`788` + +.. c:type:: PyInterpreterGuard + + An opaque interpreter guard structure. + + By holding an interpreter guard, the caller can ensure that the interpreter + will not finalize until the guard is closed (through + :c:func:`PyInterpreterGuard_Close`). + + When a guard is held, a thread attempting to finalize the interpreter will + have to wait until the guard is closed before threads can be blocked. + After finalization has started, threads are forever unable to acquire + guards for that interpreter. This means that if you forget to close an + interpreter guard, the process will **permanently hang** during + finalization! + + .. versionadded:: next + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromCurrent(void) + + Create a finalization guard for the current interpreter. This will prevent + finalization from occuring until the guard is closed. + + For example: + + .. code-block:: c + + // Temporarily prevent finalization. + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + if (guard == NULL) { + // Finalization has already started or we're out of memory. + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + // Do some critical processing here. For example, we can safely acquire + // locks that might be acquired by the finalization thread. + Py_END_ALLOW_THREADS; + + // Now that we're done with our critical processing, the interpreter is + // allowed to finalize again. + PyInterpreterGuard_Close(guard); + + On success, this function returns a guard for the current interpreter; + on failure, it returns ``NULL`` with an exception set. + + This function will fail only if the current interpreter has already started + finalizing, or if the process is out of memory. + + The guard pointer returned by this function must be eventually closed + with :c:func:`PyInterpreterGuard_Close`; failing to do so will result in + the Python process infinitely hanging. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: next + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromView(PyInterpreterView *view) + + Create a finalization guard for an interpreter through a view. + + On success, this function returns a guard to the interpreter + represented by *view*. The view is still valid after calling this + function. The guard must eventually be closed with + :c:func:`PyInterpreterGuard_Close`. + + If the interpreter no longer exists, is already finalizing, or out of memory, + then this function returns ``NULL`` without setting an exception. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: next + + +.. c:function:: void PyInterpreterGuard_Close(PyInterpreterGuard *guard) + + Close an interpreter guard, allowing the interpreter to start + finalization if no other guards remain. If an interpreter guard + is never closed, the interpreter will infinitely wait when trying + to enter finalization! + + After an interpreter guard is closed, it may not be used in + :c:func:`PyThreadState_Ensure`. Doing so will result in undefined + behavior. + + Currently, this function will deallocate *guard*, but this may change in + the future. + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: next + + +Interpreter views +----------------- + +In some cases, it may be necessary to access an interpreter that may have been +deleted. This can be done using interpreter views. + +.. c:type:: PyInterpreterView + + An opaque view of an interpreter. + + This is a thread-safe way to access an interpreter that may have be + finalizing or already destroyed. + + .. versionadded:: next + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromCurrent(void) + + Create a view to the current interpreter. + + This function is generally meant to be used alongside + :c:func:`PyInterpreterGuard_FromView` or :c:func:`PyThreadState_EnsureFromView`. + + On success, this function returns a view to the current interpreter; on + failure, it returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: next + + +.. c:function:: void PyInterpreterView_Close(PyInterpreterView *view) + + Close an interpreter view. + + If an interpreter view is never closed, the view's memory will never be + freed, but there are no other consequences. (In contrast, forgetting to + close a guard will infinitely hang the main thread during finalization.) + + Currently, this function will deallocate *view*, but this may change in + the future. + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromMain() + + Create a view for the main interpreter (the first and default + interpreter in a Python process; see + :c:func:`PyInterpreterState_Main`). + + On success, this function returns a view to the main + interpreter; on failure, it returns ``NULL`` without an exception set. + Failure indicates that the process is out of memory or that the main + interpreter has finalized (or never existed). + + Generally speaking, using this function is strongly discouraged, because + it typically compromises subinterpreter support for a program. It exists + for exceptional cases where there is no other option (such as when a native + threading library doesn't provide a ``void *arg`` parameter that could be + used to store a ``PyInterpreterGuard`` or ``PyInterpreterView`` pointer). + + The caller does not need to hold an :term:`attached thread state`. Process-wide parameters diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index 3b761d0c657cbd..ba0f711ea92c7d 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -178,8 +178,11 @@ example usage in the Python source distribution. declaration. -Non-Python created threads --------------------------- +.. _c-api-foreign-threads: + + +Using the C API from foreign threads +------------------------------------ When threads are created using the dedicated Python APIs (such as the :mod:`threading` module), a thread state is automatically associated with them, @@ -192,70 +195,358 @@ of a callback API provided by the aforementioned third-party library), you must first register these threads with the interpreter by creating a new thread state and attaching it. -The most robust way to do this is through :c:func:`PyThreadState_New` followed -by :c:func:`PyThreadState_Swap`. +The easiest way to do this is through :c:func:`PyThreadState_Ensure` +or :c:func:`PyThreadState_EnsureFromView`. .. note:: - ``PyThreadState_New`` requires an argument pointing to the desired + These functions require an argument pointing to the desired interpreter; such a pointer can be acquired via a call to - :c:func:`PyInterpreterState_Get` from the code where the thread was - created. + :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or + :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) + from the function that creates the thread. If no pointer is available (such + as when the given native thread library doesn't provide a data argument), + :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main + interpreter, but note that this will make the code incompatible with + subinterpreters. -For example:: - /* The return value of PyInterpreterState_Get() from the - function that created this thread. */ - PyInterpreterState *interp = thread_data->interp; +For example:: - /* Create a new thread state for the interpreter. It does not start out - attached. */ - PyThreadState *tstate = PyThreadState_New(interp); + // The return value of PyInterpreterGuard_FromCurrent() from the + // function that created this thread. + PyInterpreterGuard *guard = thread_data->guard; - /* Attach the thread state, which will acquire the GIL. */ - PyThreadState_Swap(tstate); + // Create a new thread state for the interpreter. + PyThreadState *tstate = PyThreadState_Ensure(guard); + if (tstate == NULL) { + PyInterpreterGuard_Close(guard); + return; + } - /* Perform Python actions here. */ + // We have a valid thread state -- perform Python actions here. result = CallSomeFunction(); - /* evaluate result or handle exception */ + // Evaluate result or handle exceptions. - /* Destroy the thread state. No Python API allowed beyond this point. */ - PyThreadState_Clear(tstate); - PyThreadState_DeleteCurrent(); + // Release the thread state. No calsl to the C API are allowed beyond this + // point. + PyThreadState_Release(tstate); + PyInterpreterGuard_Close(guard); -.. warning:: - If the interpreter finalized before ``PyThreadState_Swap`` was called, then - ``interp`` will be a dangling pointer! +Some notes about this: + +1. In the above code, ``tstate`` is the *previously* attached thread state, not + the one that was just created! In some cases, ``PyThreadState_Ensure`` might + return an internal pointer, so it is **not** safe to treat ``tstate`` as a + valid thread state (that is, do not pass ``tstate`` to a function other than + ``PyThreadState_Release``). +2. Calling ``PyThreadState_Ensure`` might not always create a new thread state, + and calling ``PyThreadState_Release`` might not always detach it. These + functions may reuse an existing attached thread state, or may re-attach a + thread state that was previously attached for the current thread. + +.. seealso:: + :pep:`788` + +.. c:function:: PyThreadState *PyThreadState_Ensure(PyInterpreterGuard *guard) + + Ensure that the thread has an attached thread state for the + interpreter protected by *guard*, and thus can safely invoke that + interpreter. + + It is OK to call this function if the thread already has an + attached thread state, as long as there is a subsequent call to + :c:func:`PyThreadState_Release` that matches this one. + + Nested calls to this function will only sometimes create a new + thread state. + + First, this function checks if an attached thread state is present. + If there is, this function then checks if the interpreter of that + thread state matches the interpreter guarded by *guard*. If that is + the case, this function simply marks the thread state as being used + by a ``PyThreadState_Ensure`` call and returns. + + If there is no attached thread state, then this function checks if any + thread state has been used by the current OS thread. (This is + returned by :c:func:`PyGILState_GetThisThreadState`.) + If there was, then this function checks if that thread state's interpreter + matches *guard*. If it does, it is re-attached and marked as used. + + Otherwise, if both of the above cases fail, a new thread state is created + for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. + + This function will return ``NULL`` to indicate a memory allocation failure, and + otherwise return a pointer to the thread state that was previously attached + (which might have been ``NULL``, in which case an non-``NULL`` sentinel value is + returned instead to differentiate between failure -- this means that this function + will sometimes return an invalid ``PyThreadState`` pointer). + + To visualize, this function is roughly equivalent to the following: + + .. code-block:: c + + PyThreadState * + PyThreadState_Ensure(PyInterpreterGuard *guard) + { + assert(guard != NULL); + PyInterpreterState *interp = PyInterpreterGuard_GetInterpreter(guard); + assert(interp != NULL); + + PyThreadState *current_tstate = PyThreadState_GetUnchecked(); + if (current_tstate == NULL) { + PyThreadState *last_used = PyGILState_GetThisThreadState(); + if (last_used != NULL) { + ++last_used->ensure_counter; + PyThreadState_Swap(last_used); + return NO_TSTATE_SENTINEL; + } + } else if (current_tstate->interp == interp) { + ++current_tstate->ensure_counter; + return current_tstate; + } + + PyThreadState *new_tstate = PyThreadState_New(interp); + if (new_tstate == NULL) { + return NULL; + } + + ++new_tstate->ensure_counter; + new_tstate->owned_by_pythreadstate_ensure = true; + PyThreadState_Swap(new_tstate); + return current_tstate == NULL ? NO_TSTATE_SENTINEL : current_tstate; + } + + .. versionadded:: next + + +.. c:function:: PyThreadState *PyThreadState_EnsureFromView(PyInterpreterView *view) + + Get an attached thread state for the interpreter referenced by *view*. + + *view* must not be ``NULL``. If the interpreter referenced by *view* has been + finalized or is currently finalizing, then this function returns ``NULL`` without + setting an exception. This function may also return ``NULL`` to indicate that the + process is out of memory. + + The interpreter referenced by *view* will be implicitly guarded. The + guard will be released upon the corresponding :c:func:`PyThreadState_Release` + call. + + On success, this function will return the thread state that was previously attached. + If no thread state was previously attached, this returns a non-``NULL`` sentinel + value. The behavior of whether this function creates a thread state is + equivalent to that of :c:func:`PyThreadState_Ensure`. + + To visualize, function is roughly equivalent to the following: + + .. code-block:: c + + PyThreadState * + PyThreadState_EnsureFromView(PyInterpreterView *view) + { + assert(view != NULL); + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + if (guard == NULL) { + return NULL; + } + + PyThreadState *tstate = PyThreadState_Ensure(guard); + if (tstate == NULL) { + PyInterpreterGuard_Close(guard); + return NULL; + } + + if (tstate->guard == NULL) { + tstate->guard = guard; + } else { + PyInterpreterGuard_Close(guard); + } + + return tstate; + } + + +.. c:function:: void PyThreadState_Release(PyThreadState *tstate) + + Undo a :c:func:`PyThreadState_Ensure` call. This must be called exactly once + for each call to ``PyThreadState_Ensure``. + + This function will decrement an internal counter on the attached thread state. If + this counter ever reaches below zero, this function emits a fatal error (via + :c:func:`Py_FatalError`). + + If the attached thread state is owned by ``PyThreadState_Ensure``, then the + attached thread state will be deallocated and deleted upon the internal counter + reaching zero. Otherwise, nothing happens when the counter reaches zero. + + If *tstate* is non-``NULL``, it will be attached upon returning. + If *tstate* indicates that no prior thread state was attached, there will be + no attached thread state upon returning. + + To visualize, this function is roughly equivalent to the following: + + .. code-block:: c + + void + PyThreadState_Release(PyThreadState *old_tstate) + { + PyThreadState *current_tstate = PyThreadState_Get(); + assert(old_tstate != NULL); + assert(current_tstate != NULL); + assert(current_tstate->ensure_counter > 0); + if (--current_tstate->ensure_counter > 0) { + // There are remaining PyThreadState_Ensure() calls + // for this thread state. + return; + } + + assert(current_tstate->ensure_counter == 0); + if (old_tstate == NO_TSTATE_SENTINEL) { + // No thread state was attached prior the PyThreadState_Ensure() + // call. So, we can just destroy the current thread state and return. + assert(current_tstate->owned_by_pythreadstate_ensure); + PyThreadState_Clear(current_tstate); + PyThreadState_DeleteCurrent(); + return; + } + + if (tstate->guard != NULL) { + PyInterpreterGuard_Close(tstate->guard); + return; + } + + if (tstate->owned_by_pythreadstate_ensure) { + // The attached thread state was created by the initial PyThreadState_Ensure() + // call. It's our job to destroy it. + PyThreadState_Clear(current_tstate); + PyThreadState_DeleteCurrent(); + } + + PyThreadState_Swap(old_tstate); + } + .. _gilstate: -Legacy API ----------- +GIL-state APIs +-------------- + +The following APIs are generally not compatible with subinterpreters and +will hang the process during interpreter finalization (see +:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were +:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs +`. -Another common pattern to call Python code from a non-Python thread is to use -:c:func:`PyGILState_Ensure` followed by a call to :c:func:`PyGILState_Release`. -These functions do not work well when multiple interpreters exist in the Python -process. If no Python interpreter has ever been used in the current thread (which -is common for threads created outside Python), ``PyGILState_Ensure`` will create -and attach a thread state for the "main" interpreter (the first interpreter in -the Python process). +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED -Additionally, these functions have thread-safety issues during interpreter -finalization. Using ``PyGILState_Ensure`` during finalization will likely -crash the process. + The GIL was already held when :c:func:`PyGILState_Ensure` was called. -Usage of these functions look like such:: + .. c:enumerator:: PyGILState_UNLOCKED - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); + The GIL was not held when :c:func:`PyGILState_Ensure` was called. - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. + + This function has no way to return an error. As such, errors are either fatal + (that is, they send ``SIGABRT`` and crash the process; see + :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as + during interpreter finalization). + + .. warning:: + Calling this function when the interpreter is finalizing will + infinitely hang the thread, which may cause deadlocks. + :ref:`cautions-regarding-runtime-finalization` for more details. + + In addition, this function generally does not work with subinterpreters + when used from foreign threads, because this function has no way of + knowing which interpreter created the thread (and as such, will implicitly + pick the main interpreter). + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` instead. + + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GIL-state API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Release` instead. + + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`thread state` that was most recently :term:`attached + ` for this thread. (If the most recent thread state + has been deleted, this returns ``NULL``.) + + If the caller has an attached thread state, it is returned. + + In other terms, this function returns the thread state that will be used by + :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then + ``PyGILState_Ensure`` will create a new thread state. + + This function cannot fail. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + instead. + + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread has an :term:`attached thread state` + that matches the thread state returned by + :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread + state or it otherwise doesn't match, then this returns ``0``. + + .. note:: + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. + + .. versionadded:: 3.4 + + .. soft-deprecated:: 3.15 + Use :c:expr:`PyThreadState_GetUnchecked() != NULL` instead. .. _fork-and-threads: @@ -398,101 +689,6 @@ C extensions. thread if the runtime is finalizing. -GIL-state APIs --------------- - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:type:: PyGILState_STATE - - The type of the value returned by :c:func:`PyGILState_Ensure` and passed to - :c:func:`PyGILState_Release`. - - .. c:enumerator:: PyGILState_LOCKED - - The GIL was already held when :c:func:`PyGILState_Ensure` was called. - - .. c:enumerator:: PyGILState_UNLOCKED - - The GIL was not held when :c:func:`PyGILState_Ensure` was called. - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. Failure is a fatal error. - - .. warning:: - Calling this function when the runtime is finalizing is unsafe. Doing - so will either hang the thread until the program ends, or fully crash - the interpreter in rare cases. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`attached thread state` for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - .. note:: - This function may return non-``NULL`` even when the :term:`thread state` - is detached. - Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. seealso:: :c:func:`PyThreadState_Get` - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its :term:`thread state ` initialized - via :c:func:`PyGILState_Ensure` will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the :term:`GIL` is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. note:: - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. versionadded:: 3.4 - - Low-level APIs -------------- diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 8575fb30c95a8d..38f45411a60863 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2738,6 +2738,7 @@ test_thread_state_ensure(void) static int test_main_interpreter_view(void) { + assert(PyInterpreterView_FromMain() == NULL); _testembed_initialize(); // Main interpreter is initialized and ready. From 25e1cf05c7e63d06c2b5741b43775889d7399bac Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 17:27:03 -0400 Subject: [PATCH 03/14] Add a whatsnew entry. --- Doc/c-api/interp-lifecycle.rst | 4 +++ Doc/c-api/threads.rst | 11 ++++++--- Doc/howto/free-threading-extensions.rst | 4 ++- Doc/whatsnew/3.15.rst | 33 +++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index 9095651554242b..9ecf5404788587 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -604,6 +604,8 @@ which led to other issues (see the warning note at Gross? Yes. Starting in Python 3.15, there are a number of C APIs that make it possible to avoid these issues by temporarily preventing finalization: +.. _interpreter-guards: + .. seealso:: :pep:`788` @@ -703,6 +705,8 @@ it possible to avoid these issues by temporarily preventing finalization: .. versionadded:: next +.. _interpreter-views: + Interpreter views ----------------- diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index ba0f711ea92c7d..8096f5b01b2f0b 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -61,9 +61,9 @@ as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. -If it turns out that you do need to create a thread state, call :c:func:`PyThreadState_New` -followed by :c:func:`PyThreadState_Swap`, or use the dangerous -:c:func:`PyGILState_Ensure` function. +If it turns out that you do need to create a thread state, it is recommended to +use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, +which will manage the thread state for you. .. _detaching-thread-state: @@ -248,6 +248,11 @@ Some notes about this: .. seealso:: :pep:`788` +.. _c-api-attach-detach: + +Attaching/detaching thread states +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + .. c:function:: PyThreadState *PyThreadState_Ensure(PyInterpreterGuard *guard) Ensure that the thread has an attached thread state for the diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index b21ed1c8f37be1..ad0578df0a2702 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -218,13 +218,15 @@ Thread State and GIL APIs Python provides a set of functions and macros to manage thread state and the GIL, such as: +* :c:func:`PyThreadState_Ensure`, :c:func:`PyThreadState_EnsureFromView`, + and :c:func:`PyThreadState_Release` * :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread` * :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` These functions should still be used in the free-threaded build to manage thread state even when the :term:`GIL` is disabled. For example, if you -create a thread outside of Python, you must call :c:func:`PyGILState_Ensure` +create a thread outside of Python, you must call :c:func:`PyThreadState_Ensure` before calling into the Python API to ensure that the thread has a valid Python thread state. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index eb08f8c4ed69e7..e7b2832775ac86 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -86,6 +86,7 @@ Summary -- Release highlights * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object ` * :pep:`803`: :ref:`Stable ABI for Free-Threaded Builds ` +* :pep:`788`: :ref:`Protection against finalization in the C API ` * :ref:`The JIT compiler has been significantly upgraded ` * :ref:`Improved error messages ` * :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter @@ -446,6 +447,38 @@ If not using a build tool -- or when writing such a tool -- you can select ``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed in :ref:`abi3-compiling`. +.. _whatsnew315-c-api-interpreter-finalization: + +:pep:`788`: Protecting the C API from interpreter finalization +-------------------------------------------------------------- + +In the C API, :term:`interpreter finalization ` can be +problematic for many extensions, because :term:`attaching ` a thread state will permanently hang the thread, resulting in deadlocks +and other spurious issues. Additionally, it has historically been impossible +to atomically check whether an interpreter is alive, leading to crashes +when a thread concurrently deletes an interpreter while another thread is +trying to attach to it. + +There are now several new suites of APIs to circumvent these problems: + +* :ref:`Interpreter guards `, which prevent an interpreter + from finalizing. +* :ref:`Interpreter views `, which allow thread-safe access + to an interpreter that may be concurrently finalizing or deleted. +* :ref:`New APIs ` to automatically attach and detach + thread states that come with built-in protection against finalization. + +In addition, APIs in the ``PyGILState`` family (most notably +:c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`) have been +:term:`soft deprecated`. There is **no** plan to remove them, and existing +code will continue to work, but there will be no new ``PyGILState`` APIs +in future versions of Python. + +.. seealso:: :pep:`788` for further details. + +(Contributed by Peter Bierma in :gh:`149101`.) + .. _whatsnew315-improved-error-messages: From af1022ac37ae562083bfe766df1f20e4659cdd71 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 17:36:27 -0400 Subject: [PATCH 04/14] Fix stable ABI things. --- Doc/data/stable_abi.dat | 11 +++++++++++ Include/pystate.h | 6 +++++- Lib/test/test_stable_abi_ctypes.py | 9 +++++++++ Misc/stable_abi.toml | 27 +++++++++++++++++++++++++++ PC/python3dll.c | 9 +++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 4ae5e999f0bf21..eaee1d523a6820 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -363,6 +363,10 @@ func,PyImport_ImportModuleLevel,3.2,, func,PyImport_ImportModuleLevelObject,3.7,, func,PyImport_ReloadModule,3.2,, func,PyIndex_Check,3.8,, +type,PyInterpreterGuard,3.15,,opaque +func,PyInterpreterGuard_Close,3.15,, +func,PyInterpreterGuard_FromCurrent,3.15,, +func,PyInterpreterGuard_FromView,3.15,, type,PyInterpreterState,3.2,,opaque func,PyInterpreterState_Clear,3.2,, func,PyInterpreterState_Delete,3.2,, @@ -370,6 +374,10 @@ func,PyInterpreterState_Get,3.9,, func,PyInterpreterState_GetDict,3.8,, func,PyInterpreterState_GetID,3.7,, func,PyInterpreterState_New,3.2,, +type,PyInterpreterView,3.15,,opaque +func,PyInterpreterView_Close,3.15,, +func,PyInterpreterView_FromCurrent,3.15,, +func,PyInterpreterView_FromMain,3.15,, func,PyIter_Check,3.8,, func,PyIter_Next,3.2,, func,PyIter_NextItem,3.14,, @@ -698,12 +706,15 @@ func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque func,PyThreadState_Clear,3.2,, func,PyThreadState_Delete,3.2,, +func,PyThreadState_Ensure,3.15,, +func,PyThreadState_EnsureFromView,3.15,, func,PyThreadState_Get,3.2,, func,PyThreadState_GetDict,3.2,, func,PyThreadState_GetFrame,3.10,, func,PyThreadState_GetID,3.10,, func,PyThreadState_GetInterpreter,3.10,, func,PyThreadState_New,3.2,, +func,PyThreadState_Release,3.15,, func,PyThreadState_SetAsyncExc,3.2,, func,PyThreadState_Swap,3.2,, func,PyThread_GetInfo,3.3,, diff --git a/Include/pystate.h b/Include/pystate.h index 254702d43e8728..3bb79959d6652e 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -120,7 +120,9 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); -/* PEP 788 -- Interpreter guards and views. */ +/* PEP 788 -- Protection against interpreter finalization */ + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) typedef struct _PyInterpreterGuard PyInterpreterGuard; typedef struct _PyInterpreterView PyInterpreterView; @@ -137,6 +139,8 @@ PyAPI_FUNC(PyThreadState *) PyThreadState_Ensure(PyInterpreterGuard *guard); PyAPI_FUNC(PyThreadState *) PyThreadState_EnsureFromView(PyInterpreterView *view); PyAPI_FUNC(void) PyThreadState_Release(PyThreadState *tstate); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index ed0868e0017fce..93759dfe8d31ae 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -365,12 +365,18 @@ def test_windows_feature_macros(self): "PyImport_ImportModuleNoBlock", "PyImport_ReloadModule", "PyIndex_Check", + "PyInterpreterGuard_Close", + "PyInterpreterGuard_FromCurrent", + "PyInterpreterGuard_FromView", "PyInterpreterState_Clear", "PyInterpreterState_Delete", "PyInterpreterState_Get", "PyInterpreterState_GetDict", "PyInterpreterState_GetID", "PyInterpreterState_New", + "PyInterpreterView_Close", + "PyInterpreterView_FromCurrent", + "PyInterpreterView_FromMain", "PyIter_Check", "PyIter_Next", "PyIter_NextItem", @@ -690,12 +696,15 @@ def test_windows_feature_macros(self): "PyThreadState_Clear", "PyThreadState_Delete", "PyThreadState_DeleteCurrent", + "PyThreadState_Ensure", + "PyThreadState_EnsureFromView", "PyThreadState_Get", "PyThreadState_GetDict", "PyThreadState_GetFrame", "PyThreadState_GetID", "PyThreadState_GetInterpreter", "PyThreadState_New", + "PyThreadState_Release", "PyThreadState_SetAsyncExc", "PyThreadState_Swap", "PyThread_GetInfo", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 101737a27829c9..285ff337490397 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2715,3 +2715,30 @@ # Note: The `_reserved` member of this struct is for interal use only. # (The definition of 'full-abi' was clarified when this entry was added.) struct_abi_kind = 'full-abi' + +# PEP 788 finalization protection + +[struct.PyInterpreterGuard] + added = '3.15' + struct_abi_kind = 'opaque' +[function.PyInterpreterGuard_FromCurrent] + added = '3.15' +[function.PyInterpreterGuard_FromView] + added = '3.15' +[function.PyInterpreterGuard_Close] + added = '3.15' +[struct.PyInterpreterView] + added = '3.15' + struct_abi_kind = 'opaque' +[function.PyInterpreterView_FromCurrent] + added = '3.15' +[function.PyInterpreterView_FromMain] + added = '3.15' +[function.PyInterpreterView_Close] + added = '3.15' +[function.PyThreadState_Ensure] + added = '3.15' +[function.PyThreadState_EnsureFromView] + added = '3.15' +[function.PyThreadState_Release] + added = '3.15' diff --git a/PC/python3dll.c b/PC/python3dll.c index abbe35c342c13e..501a6b7052c04e 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -326,12 +326,18 @@ EXPORT_FUNC(PyImport_ImportModuleLevelObject) EXPORT_FUNC(PyImport_ImportModuleNoBlock) EXPORT_FUNC(PyImport_ReloadModule) EXPORT_FUNC(PyIndex_Check) +EXPORT_FUNC(PyInterpreterGuard_Close) +EXPORT_FUNC(PyInterpreterGuard_FromCurrent) +EXPORT_FUNC(PyInterpreterGuard_FromView) EXPORT_FUNC(PyInterpreterState_Clear) EXPORT_FUNC(PyInterpreterState_Delete) EXPORT_FUNC(PyInterpreterState_Get) EXPORT_FUNC(PyInterpreterState_GetDict) EXPORT_FUNC(PyInterpreterState_GetID) EXPORT_FUNC(PyInterpreterState_New) +EXPORT_FUNC(PyInterpreterView_Close) +EXPORT_FUNC(PyInterpreterView_FromCurrent) +EXPORT_FUNC(PyInterpreterView_FromMain) EXPORT_FUNC(PyIter_Check) EXPORT_FUNC(PyIter_Next) EXPORT_FUNC(PyIter_NextItem) @@ -656,12 +662,15 @@ EXPORT_FUNC(PyThread_tss_set) EXPORT_FUNC(PyThreadState_Clear) EXPORT_FUNC(PyThreadState_Delete) EXPORT_FUNC(PyThreadState_DeleteCurrent) +EXPORT_FUNC(PyThreadState_Ensure) +EXPORT_FUNC(PyThreadState_EnsureFromView) EXPORT_FUNC(PyThreadState_Get) EXPORT_FUNC(PyThreadState_GetDict) EXPORT_FUNC(PyThreadState_GetFrame) EXPORT_FUNC(PyThreadState_GetID) EXPORT_FUNC(PyThreadState_GetInterpreter) EXPORT_FUNC(PyThreadState_New) +EXPORT_FUNC(PyThreadState_Release) EXPORT_FUNC(PyThreadState_SetAsyncExc) EXPORT_FUNC(PyThreadState_Swap) EXPORT_FUNC(PyTraceBack_Here) From 2fb8419dd37e67ae59b0cfaf1ecbe303aff452b9 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 17:38:01 -0400 Subject: [PATCH 05/14] Fix test_embed. --- Programs/_testembed.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 38f45411a60863..cf1e76ea071f9e 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2738,12 +2738,14 @@ test_thread_state_ensure(void) static int test_main_interpreter_view(void) { - assert(PyInterpreterView_FromMain() == NULL); - _testembed_initialize(); - - // Main interpreter is initialized and ready. PyInterpreterView *view = PyInterpreterView_FromMain(); assert(view != NULL); + // These should fail -- the main interpreter is not available yet. + assert(PyInterpreterGuard_FromView(view) == NULL); + assert(PyThreadState_EnsureFromView(view) == NULL); + + _testembed_initialize(); + // Main interpreter is initialized and ready at this point. PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); assert(guard != NULL); From 1096a3261fadcea1d0f8aa45709d28fd63dff9cb Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 17:43:14 -0400 Subject: [PATCH 06/14] SIlly news entry. --- .../next/C_API/2026-04-28-17-43-12.gh-issue-149101.HTuHTb.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/C_API/2026-04-28-17-43-12.gh-issue-149101.HTuHTb.rst diff --git a/Misc/NEWS.d/next/C_API/2026-04-28-17-43-12.gh-issue-149101.HTuHTb.rst b/Misc/NEWS.d/next/C_API/2026-04-28-17-43-12.gh-issue-149101.HTuHTb.rst new file mode 100644 index 00000000000000..9bcb835c19f09c --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-04-28-17-43-12.gh-issue-149101.HTuHTb.rst @@ -0,0 +1 @@ +Implement :pep:`788`. From 407062f0ed3b35fe2eff687941c1dbb0f4416f42 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 17:48:32 -0400 Subject: [PATCH 07/14] Documentation fixes. --- Doc/c-api/threads.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index 8096f5b01b2f0b..30d8df4724e77e 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -551,7 +551,7 @@ will hang the process during interpreter finalization (see .. versionadded:: 3.4 .. soft-deprecated:: 3.15 - Use :c:expr:`PyThreadState_GetUnchecked() != NULL` instead. + Use ``PyThreadState_GetUnchecked() != NULL`` instead. .. _fork-and-threads: @@ -905,7 +905,7 @@ pointer and a void pointer argument. possible. If the main thread is busy executing a system call, *func* won't be called before the system call returns. This function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. + arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. .. versionadded:: 3.1 From a33bdfceb925ce922b9620e7cb1e91f56e8c90f2 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 18:09:21 -0400 Subject: [PATCH 08/14] Make the sentinel const instead of changing the C analyzer. --- Python/pystate.c | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Python/pystate.c b/Python/pystate.c index fc3e5ea7db00c2..d13139c2cf35b3 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3484,7 +3484,7 @@ PyInterpreterView_FromMain(void) // thread state was attached. // To do this, we just use the memory address of a global variable and // cast it to a PyThreadState *. -static int NO_TSTATE_SENTINEL = 0; +static const int NO_TSTATE_SENTINEL = 0; PyThreadState * PyThreadState_Ensure(PyInterpreterGuard *guard) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 2b9a5ee68de713..d2489387f46caa 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -198,9 +198,6 @@ Python/pystate.c - _Py_tss_tstate - Python/pystate.c - _Py_tss_gilstate - Python/pystate.c - _Py_tss_interp - -# Global sentinel that is fine to share across interpreters -Python/pystate.c - NO_TSTATE_SENTINEL - - ##----------------------- ## should be const # XXX Make them const. From c504a9f3bea3b6a6e68261fca2ba9e3c27e1cbc3 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 28 Apr 2026 19:12:48 -0400 Subject: [PATCH 09/14] Fix the html IDs job. --- Doc/c-api/threads.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index 30d8df4724e77e..250c3ec1ccac57 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -178,6 +178,7 @@ example usage in the Python source distribution. declaration. +.. _non-python-created-threads: .. _c-api-foreign-threads: @@ -431,7 +432,7 @@ Attaching/detaching thread states PyThreadState_Swap(old_tstate); } - +.. _legacy-api: .. _gilstate: GIL-state APIs From a00bfbb074d6ba68785fcdc75bb8cc640bce7231 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 29 Apr 2026 08:24:51 -0400 Subject: [PATCH 10/14] Apply suggestions from code review Co-authored-by: Petr Viktorin --- Doc/c-api/interp-lifecycle.rst | 16 +++++++++++----- Doc/c-api/threads.rst | 7 ++++--- Doc/whatsnew/3.15.rst | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index 9ecf5404788587..7e3908ddde6f0c 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -608,7 +608,8 @@ it possible to avoid these issues by temporarily preventing finalization: .. seealso:: - :pep:`788` + :pep:`788` explains the design, motivation and rationale + for these APIs. .. c:type:: PyInterpreterGuard @@ -624,14 +625,19 @@ it possible to avoid these issues by temporarily preventing finalization: guards for that interpreter. This means that if you forget to close an interpreter guard, the process will **permanently hang** during finalization! - + + Holding a guard for an interpreter is similar to holding a + :term:`strong reference` to a Python object, except finalization does not happen + automatically after all guards are released: it requires an explicit + :c:func:`Py_EndInterpreter` call. + .. versionadded:: next .. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromCurrent(void) Create a finalization guard for the current interpreter. This will prevent - finalization from occuring until the guard is closed. + finalization until the guard is closed. For example: @@ -764,8 +770,8 @@ deleted. This can be done using interpreter views. Failure indicates that the process is out of memory or that the main interpreter has finalized (or never existed). - Generally speaking, using this function is strongly discouraged, because - it typically compromises subinterpreter support for a program. It exists + Using this function in extension libraries is strongly discouraged, because + it typically compromises the library's subinterpreter support. It exists for exceptional cases where there is no other option (such as when a native threading library doesn't provide a ``void *arg`` parameter that could be used to store a ``PyInterpreterGuard`` or ``PyInterpreterView`` pointer). diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index 250c3ec1ccac57..0bb9959aabf648 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -545,9 +545,10 @@ will hang the process during interpreter finalization (see :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread state or it otherwise doesn't match, then this returns ``0``. - .. note:: - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. + + This is mainly a helper/diagnostic function. .. versionadded:: 3.4 diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e7b2832775ac86..9914b030c06103 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -456,7 +456,7 @@ In the C API, :term:`interpreter finalization ` can be problematic for many extensions, because :term:`attaching ` a thread state will permanently hang the thread, resulting in deadlocks and other spurious issues. Additionally, it has historically been impossible -to atomically check whether an interpreter is alive, leading to crashes +to safely check whether an interpreter is alive before using it, leading to crashes when a thread concurrently deletes an interpreter while another thread is trying to attach to it. From 6ca24990546096bd873009999ce4f42a9dadf88a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 29 Apr 2026 08:29:25 -0400 Subject: [PATCH 11/14] Update Doc/c-api/threads.rst Co-authored-by: Petr Viktorin --- Doc/c-api/threads.rst | 2158 ++++++++++++++++++++--------------------- 1 file changed, 1079 insertions(+), 1079 deletions(-) diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index 0bb9959aabf648..f224d04589d739 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -1,1083 +1,1083 @@ -.. highlight:: c - -.. _threads: - -Thread states and the global interpreter lock -============================================= - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -Unless on a :term:`free-threaded build` of :term:`CPython`, -the Python interpreter is generally not thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by a thread before -accessing Python objects. Without the lock, even the simplest operations -could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -As such, only a thread that holds the GIL may operate on Python objects or -invoke Python's C API. - -.. index:: single: setswitchinterval (in module sys) - -In order to emulate concurrency, the interpreter regularly tries to switch -threads between bytecode instructions (see :func:`sys.setswitchinterval`). -This is why locks are also necessary for thread-safety in pure-Python code. - -Additionally, the global interpreter lock is released around blocking I/O -operations, such as reading or writing to a file. From the C API, this is done -by :ref:`detaching the thread state `. - - -.. index:: - single: PyThreadState (C type) - -The Python interpreter keeps some thread-local information inside -a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. -Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state -referenced by this pointer is considered to be :term:`attached `. - -A thread can only have one :term:`attached thread state` at a time. An attached -thread state is typically analogous with holding the GIL, except on -free-threaded builds. On builds with the GIL enabled, attaching a thread state -will block until the GIL can be acquired. However, even on builds with the GIL -disabled, it is still required to have an attached thread state, as the interpreter -needs to keep track of which threads may access Python objects. - -.. note:: - - Even on the free-threaded build, attaching a thread state may block, as the - GIL can be re-enabled or threads might be temporarily suspended (such as during - a garbage collection). - -Generally, there will always be an attached thread state when using Python's -C API, including during embedding and when implementing methods, so it's uncommon -to need to set up a thread state on your own. Only in some specific cases, such -as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the -thread not have an attached thread state. -If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. - -If it turns out that you do need to create a thread state, it is recommended to -use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, -which will manage the thread state for you. - - -.. _detaching-thread-state: - -Detaching the thread state from extension code ----------------------------------------------- - -Most extension code manipulating the :term:`thread state` has the following simple -structure:: - - Save the thread state in a local variable. - ... Do some blocking I/O operation ... - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS (C macro) - single: Py_END_ALLOW_THREADS (C macro) - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. - -The block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ... Do some blocking I/O operation ... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread (C function) - single: PyEval_SaveThread (C function) - -Here is how these functions work: - -The attached thread state implies that the GIL is held for the interpreter. -To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored -in a local variable. - -By detaching the thread state, the GIL is released, which allows other threads -to attach to the interpreter and execute while the current thread performs -blocking I/O. When the I/O operation is complete, the old thread state is -reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until -the GIL can be acquired. - -.. note:: - Performing blocking I/O is the most common use case for detaching - the thread state, but it is also useful to call it over long-running - native code that doesn't need access to Python objects or Python's C API. - For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the - :term:`thread state ` when compressing or hashing - data. - -On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, -but **detaching the thread state is still required**, because the interpreter -periodically needs to block all threads to get a consistent view of Python objects -without the risk of race conditions. -For example, CPython currently suspends all threads for a short period of time -while running the garbage collector. - -.. warning:: - - Detaching the thread state can lead to unexpected behavior during interpreter - finalization. See :ref:`cautions-regarding-runtime-finalization` for more - details. - - -APIs -^^^^ - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - -.. note:: - - These macros are still necessary on the :term:`free-threaded build` to prevent - deadlocks. - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. - - -.. _non-python-created-threads: -.. _c-api-foreign-threads: - - -Using the C API from foreign threads ------------------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated with them, -However, when a thread is created from native code (for example, by a -third-party library with its own thread management), it doesn't hold an -attached thread state. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating a new thread state and attaching it. - -The easiest way to do this is through :c:func:`PyThreadState_Ensure` -or :c:func:`PyThreadState_EnsureFromView`. - -.. note:: - These functions require an argument pointing to the desired - interpreter; such a pointer can be acquired via a call to - :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or - :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) - from the function that creates the thread. If no pointer is available (such - as when the given native thread library doesn't provide a data argument), - :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main - interpreter, but note that this will make the code incompatible with - subinterpreters. - - -For example:: - - // The return value of PyInterpreterGuard_FromCurrent() from the - // function that created this thread. - PyInterpreterGuard *guard = thread_data->guard; - - // Create a new thread state for the interpreter. - PyThreadState *tstate = PyThreadState_Ensure(guard); - if (tstate == NULL) { - PyInterpreterGuard_Close(guard); - return; - } - - // We have a valid thread state -- perform Python actions here. - result = CallSomeFunction(); - // Evaluate result or handle exceptions. - - // Release the thread state. No calsl to the C API are allowed beyond this - // point. - PyThreadState_Release(tstate); - PyInterpreterGuard_Close(guard); - - -Some notes about this: - -1. In the above code, ``tstate`` is the *previously* attached thread state, not - the one that was just created! In some cases, ``PyThreadState_Ensure`` might - return an internal pointer, so it is **not** safe to treat ``tstate`` as a - valid thread state (that is, do not pass ``tstate`` to a function other than - ``PyThreadState_Release``). -2. Calling ``PyThreadState_Ensure`` might not always create a new thread state, - and calling ``PyThreadState_Release`` might not always detach it. These - functions may reuse an existing attached thread state, or may re-attach a - thread state that was previously attached for the current thread. - -.. seealso:: - :pep:`788` - -.. _c-api-attach-detach: - -Attaching/detaching thread states -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. c:function:: PyThreadState *PyThreadState_Ensure(PyInterpreterGuard *guard) - - Ensure that the thread has an attached thread state for the - interpreter protected by *guard*, and thus can safely invoke that - interpreter. - - It is OK to call this function if the thread already has an - attached thread state, as long as there is a subsequent call to - :c:func:`PyThreadState_Release` that matches this one. - - Nested calls to this function will only sometimes create a new - thread state. - - First, this function checks if an attached thread state is present. - If there is, this function then checks if the interpreter of that - thread state matches the interpreter guarded by *guard*. If that is - the case, this function simply marks the thread state as being used - by a ``PyThreadState_Ensure`` call and returns. - - If there is no attached thread state, then this function checks if any - thread state has been used by the current OS thread. (This is - returned by :c:func:`PyGILState_GetThisThreadState`.) - If there was, then this function checks if that thread state's interpreter - matches *guard*. If it does, it is re-attached and marked as used. - - Otherwise, if both of the above cases fail, a new thread state is created - for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. - - This function will return ``NULL`` to indicate a memory allocation failure, and - otherwise return a pointer to the thread state that was previously attached - (which might have been ``NULL``, in which case an non-``NULL`` sentinel value is - returned instead to differentiate between failure -- this means that this function - will sometimes return an invalid ``PyThreadState`` pointer). - - To visualize, this function is roughly equivalent to the following: - - .. code-block:: c - - PyThreadState * - PyThreadState_Ensure(PyInterpreterGuard *guard) - { - assert(guard != NULL); - PyInterpreterState *interp = PyInterpreterGuard_GetInterpreter(guard); - assert(interp != NULL); - - PyThreadState *current_tstate = PyThreadState_GetUnchecked(); - if (current_tstate == NULL) { - PyThreadState *last_used = PyGILState_GetThisThreadState(); - if (last_used != NULL) { - ++last_used->ensure_counter; - PyThreadState_Swap(last_used); - return NO_TSTATE_SENTINEL; - } - } else if (current_tstate->interp == interp) { - ++current_tstate->ensure_counter; - return current_tstate; - } - - PyThreadState *new_tstate = PyThreadState_New(interp); - if (new_tstate == NULL) { - return NULL; - } - - ++new_tstate->ensure_counter; - new_tstate->owned_by_pythreadstate_ensure = true; - PyThreadState_Swap(new_tstate); - return current_tstate == NULL ? NO_TSTATE_SENTINEL : current_tstate; - } - - .. versionadded:: next - - -.. c:function:: PyThreadState *PyThreadState_EnsureFromView(PyInterpreterView *view) - - Get an attached thread state for the interpreter referenced by *view*. - - *view* must not be ``NULL``. If the interpreter referenced by *view* has been - finalized or is currently finalizing, then this function returns ``NULL`` without - setting an exception. This function may also return ``NULL`` to indicate that the - process is out of memory. - - The interpreter referenced by *view* will be implicitly guarded. The - guard will be released upon the corresponding :c:func:`PyThreadState_Release` - call. - - On success, this function will return the thread state that was previously attached. - If no thread state was previously attached, this returns a non-``NULL`` sentinel - value. The behavior of whether this function creates a thread state is - equivalent to that of :c:func:`PyThreadState_Ensure`. - - To visualize, function is roughly equivalent to the following: - - .. code-block:: c - - PyThreadState * - PyThreadState_EnsureFromView(PyInterpreterView *view) - { - assert(view != NULL); - PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); - if (guard == NULL) { - return NULL; - } - - PyThreadState *tstate = PyThreadState_Ensure(guard); - if (tstate == NULL) { - PyInterpreterGuard_Close(guard); - return NULL; - } - - if (tstate->guard == NULL) { - tstate->guard = guard; - } else { - PyInterpreterGuard_Close(guard); - } - - return tstate; - } - - -.. c:function:: void PyThreadState_Release(PyThreadState *tstate) - - Undo a :c:func:`PyThreadState_Ensure` call. This must be called exactly once - for each call to ``PyThreadState_Ensure``. - - This function will decrement an internal counter on the attached thread state. If - this counter ever reaches below zero, this function emits a fatal error (via - :c:func:`Py_FatalError`). - - If the attached thread state is owned by ``PyThreadState_Ensure``, then the - attached thread state will be deallocated and deleted upon the internal counter - reaching zero. Otherwise, nothing happens when the counter reaches zero. - - If *tstate* is non-``NULL``, it will be attached upon returning. - If *tstate* indicates that no prior thread state was attached, there will be - no attached thread state upon returning. - - To visualize, this function is roughly equivalent to the following: - - .. code-block:: c - - void - PyThreadState_Release(PyThreadState *old_tstate) - { - PyThreadState *current_tstate = PyThreadState_Get(); - assert(old_tstate != NULL); - assert(current_tstate != NULL); - assert(current_tstate->ensure_counter > 0); - if (--current_tstate->ensure_counter > 0) { - // There are remaining PyThreadState_Ensure() calls - // for this thread state. - return; - } - - assert(current_tstate->ensure_counter == 0); - if (old_tstate == NO_TSTATE_SENTINEL) { - // No thread state was attached prior the PyThreadState_Ensure() - // call. So, we can just destroy the current thread state and return. - assert(current_tstate->owned_by_pythreadstate_ensure); - PyThreadState_Clear(current_tstate); - PyThreadState_DeleteCurrent(); - return; - } - - if (tstate->guard != NULL) { - PyInterpreterGuard_Close(tstate->guard); - return; - } - - if (tstate->owned_by_pythreadstate_ensure) { - // The attached thread state was created by the initial PyThreadState_Ensure() - // call. It's our job to destroy it. - PyThreadState_Clear(current_tstate); - PyThreadState_DeleteCurrent(); - } - - PyThreadState_Swap(old_tstate); - } - -.. _legacy-api: -.. _gilstate: - -GIL-state APIs --------------- - -The following APIs are generally not compatible with subinterpreters and -will hang the process during interpreter finalization (see -:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were -:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs -`. - - -.. c:type:: PyGILState_STATE - - The type of the value returned by :c:func:`PyGILState_Ensure` and passed to - :c:func:`PyGILState_Release`. - - .. c:enumerator:: PyGILState_LOCKED - - The GIL was already held when :c:func:`PyGILState_Ensure` was called. - - .. c:enumerator:: PyGILState_UNLOCKED - - The GIL was not held when :c:func:`PyGILState_Ensure` was called. - - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. - - This function has no way to return an error. As such, errors are either fatal - (that is, they send ``SIGABRT`` and crash the process; see - :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as - during interpreter finalization). - - .. warning:: - Calling this function when the interpreter is finalizing will - infinitely hang the thread, which may cause deadlocks. - :ref:`cautions-regarding-runtime-finalization` for more details. - - In addition, this function generally does not work with subinterpreters - when used from foreign threads, because this function has no way of - knowing which interpreter created the thread (and as such, will implicitly - pick the main interpreter). - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - .. soft-deprecated:: 3.15 - Use :c:func:`PyThreadState_Ensure` or - :c:func:`PyThreadState_EnsureFromView` instead. - - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GIL-state API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - - .. soft-deprecated:: 3.15 - Use :c:func:`PyThreadState_Release` instead. - - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`thread state` that was most recently :term:`attached - ` for this thread. (If the most recent thread state - has been deleted, this returns ``NULL``.) - - If the caller has an attached thread state, it is returned. - - In other terms, this function returns the thread state that will be used by - :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then - ``PyGILState_Ensure`` will create a new thread state. - - This function cannot fail. - - .. soft-deprecated:: 3.15 - Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - instead. - - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread has an :term:`attached thread state` - that matches the thread state returned by - :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread - state or it otherwise doesn't match, then this returns ``0``. - +.. highlight:: c + +.. _threads: + +Thread states and the global interpreter lock +============================================= + +.. index:: + single: global interpreter lock + single: interpreter lock + single: lock, interpreter + +Unless on a :term:`free-threaded build` of :term:`CPython`, +the Python interpreter is generally not thread-safe. In order to support +multi-threaded Python programs, there's a global lock, called the :term:`global +interpreter lock` or :term:`GIL`, that must be held by a thread before +accessing Python objects. Without the lock, even the simplest operations +could cause problems in a multi-threaded program: for example, when +two threads simultaneously increment the reference count of the same object, the +reference count could end up being incremented only once instead of twice. + +As such, only a thread that holds the GIL may operate on Python objects or +invoke Python's C API. + +.. index:: single: setswitchinterval (in module sys) + +In order to emulate concurrency, the interpreter regularly tries to switch +threads between bytecode instructions (see :func:`sys.setswitchinterval`). +This is why locks are also necessary for thread-safety in pure-Python code. + +Additionally, the global interpreter lock is released around blocking I/O +operations, such as reading or writing to a file. From the C API, this is done +by :ref:`detaching the thread state `. + + +.. index:: + single: PyThreadState (C type) + +The Python interpreter keeps some thread-local information inside +a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached `. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the GIL, except on +free-threaded builds. On builds with the GIL enabled, attaching a thread state +will block until the GIL can be acquired. However, even on builds with the GIL +disabled, it is still required to have an attached thread state, as the interpreter +needs to keep track of which threads may access Python objects. + +.. note:: + + Even on the free-threaded build, attaching a thread state may block, as the + GIL can be re-enabled or threads might be temporarily suspended (such as during + a garbage collection). + +Generally, there will always be an attached thread state when using Python's +C API, including during embedding and when implementing methods, so it's uncommon +to need to set up a thread state on your own. Only in some specific cases, such +as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the +thread not have an attached thread state. +If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. + +If it turns out that you do need to create a thread state, it is recommended to +use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, +which will manage the thread state for you. + + +.. _detaching-thread-state: + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple +structure:: + + Save the thread state in a local variable. + ... Do some blocking I/O operation ... + Restore the thread state from the local variable. + +This is so common that a pair of macros exists to simplify it:: + + Py_BEGIN_ALLOW_THREADS + ... Do some blocking I/O operation ... + Py_END_ALLOW_THREADS + +.. index:: + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) + +The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a +hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the +block. + +The block above expands to the following code:: + + PyThreadState *_save; + + _save = PyEval_SaveThread(); + ... Do some blocking I/O operation ... + PyEval_RestoreThread(_save); + +.. index:: + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) + +Here is how these functions work: + +The attached thread state implies that the GIL is held for the interpreter. +To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored +in a local variable. + +By detaching the thread state, the GIL is released, which allows other threads +to attach to the interpreter and execute while the current thread performs +blocking I/O. When the I/O operation is complete, the old thread state is +reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until +the GIL can be acquired. + +.. note:: + Performing blocking I/O is the most common use case for detaching + the thread state, but it is also useful to call it over long-running + native code that doesn't need access to Python objects or Python's C API. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state ` when compressing or hashing + data. + +On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, +but **detaching the thread state is still required**, because the interpreter +periodically needs to block all threads to get a consistent view of Python objects +without the risk of race conditions. +For example, CPython currently suspends all threads for a short period of time +while running the garbage collector. + +.. warning:: + + Detaching the thread state can lead to unexpected behavior during interpreter + finalization. See :ref:`cautions-regarding-runtime-finalization` for more + details. + + +APIs +^^^^ + +The following macros are normally used without a trailing semicolon; look for +example usage in the Python source distribution. + +.. note:: + + These macros are still necessary on the :term:`free-threaded build` to prevent + deadlocks. + +.. c:macro:: Py_BEGIN_ALLOW_THREADS + + This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. + Note that it contains an opening brace; it must be matched with a following + :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this + macro. + + +.. c:macro:: Py_END_ALLOW_THREADS + + This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains + a closing brace; it must be matched with an earlier + :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of + this macro. + + +.. c:macro:: Py_BLOCK_THREADS + + This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. + + +.. c:macro:: Py_UNBLOCK_THREADS + + This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to + :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable + declaration. + + +.. _non-python-created-threads: +.. _c-api-foreign-threads: + + +Using the C API from foreign threads +------------------------------------ + +When threads are created using the dedicated Python APIs (such as the +:mod:`threading` module), a thread state is automatically associated with them, +However, when a thread is created from native code (for example, by a +third-party library with its own thread management), it doesn't hold an +attached thread state. + +If you need to call Python code from these threads (often this will be part +of a callback API provided by the aforementioned third-party library), +you must first register these threads with the interpreter by +creating a new thread state and attaching it. + +The easiest way to do this is through :c:func:`PyThreadState_Ensure` +or :c:func:`PyThreadState_EnsureFromView`. + +.. note:: + These functions require an argument pointing to the desired + interpreter; such a pointer can be acquired via a call to + :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or + :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) + from the function that creates the thread. If no pointer is available (such + as when the given native thread library doesn't provide a data argument), + :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main + interpreter, but note that this will make the code incompatible with + subinterpreters. + + +For example:: + + // The return value of PyInterpreterGuard_FromCurrent() from the + // function that created this thread. + PyInterpreterGuard *guard = thread_data->guard; + + // Create a new thread state for the interpreter. + PyThreadState *tstate = PyThreadState_Ensure(guard); + if (tstate == NULL) { + PyInterpreterGuard_Close(guard); + return; + } + + // We have a valid thread state -- perform Python actions here. + result = CallSomeFunction(); + // Evaluate result or handle exceptions. + + // Release the thread state. No calsl to the C API are allowed beyond this + // point. + PyThreadState_Release(tstate); + PyInterpreterGuard_Close(guard); + + +Some notes about this: + +1. In the above code, ``tstate`` is the *previously* attached thread state, not + the one that was just created! In some cases, ``PyThreadState_Ensure`` might + return an internal pointer, so it is **not** safe to treat ``tstate`` as a + valid thread state (that is, do not pass ``tstate`` to a function other than + ``PyThreadState_Release``). +2. Calling ``PyThreadState_Ensure`` might not always create a new thread state, + and calling ``PyThreadState_Release`` might not always detach it. These + functions may reuse an existing attached thread state, or may re-attach a + thread state that was previously attached for the current thread. + +.. seealso:: + :pep:`788` + +.. _c-api-attach-detach: + +Attaching/detaching thread states +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyThreadState *PyThreadState_Ensure(PyInterpreterGuard *guard) + + Ensure that the thread has an attached thread state for the + interpreter protected by *guard*, and thus can safely invoke that + interpreter. + + It is OK to call this function if the thread already has an + attached thread state, as long as there is a subsequent call to + :c:func:`PyThreadState_Release` that matches this one. + + Nested calls to this function will only sometimes create a new + thread state. + + First, this function checks if an attached thread state is present. + If there is, this function then checks if the interpreter of that + thread state matches the interpreter guarded by *guard*. If that is + the case, this function simply marks the thread state as being used + by a ``PyThreadState_Ensure`` call and returns. + + If there is no attached thread state, then this function checks if any + thread state has been used by the current OS thread. (This is + returned by :c:func:`PyGILState_GetThisThreadState`.) + If there was, then this function checks if that thread state's interpreter + matches *guard*. If it does, it is re-attached and marked as used. + + Otherwise, if both of the above cases fail, a new thread state is created + for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. + + This function will return ``NULL`` to indicate a memory allocation failure, and + otherwise return a pointer to the thread state that was previously attached + (which might have been ``NULL``, in which case an non-``NULL`` sentinel value is + returned instead to differentiate between failure -- this means that this function + will sometimes return an invalid ``PyThreadState`` pointer). + + To visualize, this function is roughly equivalent to the following: + + .. code-block:: c + + PyThreadState * + PyThreadState_Ensure(PyInterpreterGuard *guard) + { + assert(guard != NULL); + PyInterpreterState *interp = PyInterpreterGuard_GetInterpreter(guard); + assert(interp != NULL); + + PyThreadState *current_tstate = PyThreadState_GetUnchecked(); + if (current_tstate == NULL) { + PyThreadState *last_used = PyGILState_GetThisThreadState(); + if (last_used != NULL) { + ++last_used->ensure_counter; + PyThreadState_Swap(last_used); + return NO_TSTATE_SENTINEL; + } + } else if (current_tstate->interp == interp) { + ++current_tstate->ensure_counter; + return current_tstate; + } + + PyThreadState *new_tstate = PyThreadState_New(interp); + if (new_tstate == NULL) { + return NULL; + } + + ++new_tstate->ensure_counter; + new_tstate->owned_by_pythreadstate_ensure = true; + PyThreadState_Swap(new_tstate); + return current_tstate == NULL ? NO_TSTATE_SENTINEL : current_tstate; + } + + .. versionadded:: next + + +.. c:function:: PyThreadState *PyThreadState_EnsureFromView(PyInterpreterView *view) + + Get an attached thread state for the interpreter referenced by *view*. + + *view* must not be ``NULL``. If the interpreter referenced by *view* has been + finalized or is currently finalizing, then this function returns ``NULL`` without + setting an exception. This function may also return ``NULL`` on other errors, + such memory allocation failure. + + The interpreter referenced by *view* will be implicitly guarded. The + guard will be released upon the corresponding :c:func:`PyThreadState_Release` + call. + + On success, this function will return the thread state that was previously attached. + If no thread state was previously attached, this returns a non-``NULL`` sentinel + value. The behavior of whether this function creates a thread state is + equivalent to that of :c:func:`PyThreadState_Ensure`. + + To visualize, function is roughly equivalent to the following: + + .. code-block:: c + + PyThreadState * + PyThreadState_EnsureFromView(PyInterpreterView *view) + { + assert(view != NULL); + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + if (guard == NULL) { + return NULL; + } + + PyThreadState *tstate = PyThreadState_Ensure(guard); + if (tstate == NULL) { + PyInterpreterGuard_Close(guard); + return NULL; + } + + if (tstate->guard == NULL) { + tstate->guard = guard; + } else { + PyInterpreterGuard_Close(guard); + } + + return tstate; + } + + +.. c:function:: void PyThreadState_Release(PyThreadState *tstate) + + Undo a :c:func:`PyThreadState_Ensure` call. This must be called exactly once + for each call to ``PyThreadState_Ensure``. + + This function will decrement an internal counter on the attached thread state. If + this counter ever reaches below zero, this function emits a fatal error (via + :c:func:`Py_FatalError`). + + If the attached thread state is owned by ``PyThreadState_Ensure``, then the + attached thread state will be deallocated and deleted upon the internal counter + reaching zero. Otherwise, nothing happens when the counter reaches zero. + + If *tstate* is non-``NULL``, it will be attached upon returning. + If *tstate* indicates that no prior thread state was attached, there will be + no attached thread state upon returning. + + To visualize, this function is roughly equivalent to the following: + + .. code-block:: c + + void + PyThreadState_Release(PyThreadState *old_tstate) + { + PyThreadState *current_tstate = PyThreadState_Get(); + assert(old_tstate != NULL); + assert(current_tstate != NULL); + assert(current_tstate->ensure_counter > 0); + if (--current_tstate->ensure_counter > 0) { + // There are remaining PyThreadState_Ensure() calls + // for this thread state. + return; + } + + assert(current_tstate->ensure_counter == 0); + if (old_tstate == NO_TSTATE_SENTINEL) { + // No thread state was attached prior the PyThreadState_Ensure() + // call. So, we can just destroy the current thread state and return. + assert(current_tstate->owned_by_pythreadstate_ensure); + PyThreadState_Clear(current_tstate); + PyThreadState_DeleteCurrent(); + return; + } + + if (tstate->guard != NULL) { + PyInterpreterGuard_Close(tstate->guard); + return; + } + + if (tstate->owned_by_pythreadstate_ensure) { + // The attached thread state was created by the initial PyThreadState_Ensure() + // call. It's our job to destroy it. + PyThreadState_Clear(current_tstate); + PyThreadState_DeleteCurrent(); + } + + PyThreadState_Swap(old_tstate); + } + +.. _legacy-api: +.. _gilstate: + +GIL-state APIs +-------------- + +The following APIs are generally not compatible with subinterpreters and +will hang the process during interpreter finalization (see +:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were +:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs +`. + + +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. + + This function has no way to return an error. As such, errors are either fatal + (that is, they send ``SIGABRT`` and crash the process; see + :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as + during interpreter finalization). + + .. warning:: + Calling this function when the interpreter is finalizing will + infinitely hang the thread, which may cause deadlocks. + :ref:`cautions-regarding-runtime-finalization` for more details. + + In addition, this function generally does not work with subinterpreters + when used from foreign threads, because this function has no way of + knowing which interpreter created the thread (and as such, will implicitly + pick the main interpreter). + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` instead. + + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GIL-state API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Release` instead. + + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`thread state` that was most recently :term:`attached + ` for this thread. (If the most recent thread state + has been deleted, this returns ``NULL``.) + + If the caller has an attached thread state, it is returned. + + In other terms, this function returns the thread state that will be used by + :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then + ``PyGILState_Ensure`` will create a new thread state. + + This function cannot fail. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + instead. + + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread has an :term:`attached thread state` + that matches the thread state returned by + :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread + state or it otherwise doesn't match, then this returns ``0``. + If the current Python process has ever created a subinterpreter, this function will *always* return ``1``. This is mainly a helper/diagnostic function. - - .. versionadded:: 3.4 - - .. soft-deprecated:: 3.15 - Use ``PyThreadState_GetUnchecked() != NULL`` instead. - - -.. _fork-and-threads: - -Cautions about fork() ---------------------- - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. This has a -concrete impact both on how locks must be handled and on all stored state -in CPython's runtime. - -The fact that only the "current" thread remains -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not -always able to. - -The fact that all other threads go away also means that CPython's -runtime state there must be cleaned up properly, which :func:`os.fork` -does. This means finalizing all other :c:type:`PyThreadState` objects -belonging to the current interpreter and all other -:c:type:`PyInterpreterState` objects. Due to this and the special -nature of the :ref:`"main" interpreter `, -:c:func:`fork` should only be called in that interpreter's "main" -thread, where the CPython global runtime was originally initialized. -The only exception is if :c:func:`exec` will be called immediately -after. - - -High-level APIs ---------------- - -These are the most commonly used types and functions when writing multi-threaded -C extensions. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is: - - .. c:member:: PyInterpreterState *interp - - This thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Detach the :term:`attached thread state` and return it. - The thread will have no :term:`thread state` upon returning. - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*. - The passed :term:`thread state` **should not** be :term:`attached `, - otherwise deadlock ensues. *tstate* will be attached upon returning. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the :term:`attached thread state`. If the thread has no attached - thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` - block), then this issues a fatal error (so that the caller needn't check - for ``NULL``). - - See also :c:func:`PyThreadState_GetUnchecked`. - -.. c:function:: PyThreadState* PyThreadState_GetUnchecked() - - Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a - fatal error if it is NULL. The caller is responsible to check if the result - is NULL. - - .. versionadded:: 3.13 - In Python 3.5 to 3.12, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*, and return the - :term:`thread state` that was attached prior to calling. - - This function is safe to call without an :term:`attached thread state`; it - will simply return ``NULL`` indicating that there was no prior thread state. - - .. seealso:: - :c:func:`PyEval_ReleaseThread` - - .. note:: - Similar to :c:func:`PyGILState_Ensure`, this function will hang the - thread if the runtime is finalizing. - - -Low-level APIs --------------- - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - An :term:`attached thread state` is not needed. - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a :term:`thread state` object. *tstate* - must be :term:`attached ` - - .. versionchanged:: 3.9 - This function now calls the :c:member:`!PyThreadState.on_delete` callback. - Previously, that happened in :c:func:`PyThreadState_Delete`. - - .. versionchanged:: 3.13 - The :c:member:`!PyThreadState.on_delete` callback was removed. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a :term:`thread state` object. *tstate* should not - be :term:`attached ` to any thread. - *tstate* must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: void PyThreadState_DeleteCurrent(void) - - Detach the :term:`attached thread state` (which must have been reset - with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. - - No :term:`thread state` will be :term:`attached ` upon - returning. - -.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) - - Get the current frame of the Python thread state *tstate*. - - Return a :term:`strong reference`. Return ``NULL`` if no frame is currently - executing. - - See also :c:func:`PyEval_GetFrame`. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - - Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) - - Get the interpreter of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) - - Suspend tracing and profiling in the Python thread state *tstate*. - - Resume them using the :c:func:`PyThreadState_LeaveTracing` function. - - .. versionadded:: 3.11 - - -.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) - - Resume tracing and profiling in the Python thread state *tstate* suspended - by the :c:func:`PyThreadState_EnterTracing` function. - - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. - - .. versionadded:: 3.11 - - -.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) - - Set the stack protection start address and stack protection size - of a Python thread state. - - On success, return ``0``. - On failure, set an exception and return ``-1``. - - CPython implements :ref:`recursion control ` for C code by raising - :py:exc:`RecursionError` when it notices that the machine execution stack is close - to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. - For this, it needs to know the location of the current thread's stack, which it - normally gets from the operating system. - When the stack is changed, for example using context switching techniques like the - Boost library's ``boost::context``, you must call - :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. - - Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before - or after changing the stack. - Do not call any other Python C API between the call and the stack - change. - - See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. - - .. versionadded:: 3.15 - - -.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) - - Reset the stack protection start address and stack protection size - of a Python thread state to the operating system defaults. - - See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. - - .. versionadded:: 3.15 - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no :term:`thread state` - is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no - thread state is attached. - - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - :term:`Attach ` *tstate* to the current thread, - which must not be ``NULL`` or already :term:`attached `. - - The calling thread must not already have an :term:`attached thread state`. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Detach the :term:`attached thread state`. - The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the :term:`attached thread state` --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -Asynchronous notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding an :term:`attached thread state` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the :term:`thread state ` is detached. - - This function doesn't need an :term:`attached thread state`. However, to call this - function in a subinterpreter, the caller must have an :term:`attached thread state`. - Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.9 - If this function is called in a subinterpreter, the function *func* is - now scheduled to be called from the subinterpreter, rather than being - called from the main interpreter. Each subinterpreter now has its own - list of scheduled calls. - - .. versionchanged:: 3.12 - This function now always schedules *func* to be run in the main - interpreter. - - -.. c:function:: int Py_MakePendingCalls(void) - - Execute all pending calls. This is usually executed automatically by the - interpreter. - - This function returns ``0`` on success, and returns ``-1`` with an exception - set on failure. - - If this is not called in the main thread of the main - interpreter, this function does nothing and returns ``0``. - The caller must hold an :term:`attached thread state`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.12 - This function only runs pending calls in the main interpreter. - - -.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) - - Schedule an exception to be raised asynchronously in a thread. - If the thread has a previously scheduled exception, it is overwritten. - - The *id* argument is the thread id of the target thread, as returned by - :c:func:`PyThread_get_thread_ident`. - *exc* is the class of the exception to be raised, or ``NULL`` to clear - the pending exception (if any). - - Return the number of affected thread states. - This is normally ``1`` if *id* is found, even when no change was - made (the given *exc* was already pending, or *exc* is ``NULL`` but - no exception is pending). - If the thread id isn't found, return ``0``. This raises no exceptions. - - To prevent naive misuse, you must write your own C extension to call this. - This function must be called with an :term:`attached thread state`. - This function does not steal any references to *exc*. - This function does not necessarily interrupt system calls such as - :py:func:`~time.sleep`. - - .. versionchanged:: 3.7 - The type of the *id* parameter changed from :c:expr:`long` to - :c:expr:`unsigned long`. - - -Operating system thread APIs -============================ - -.. c:macro:: PYTHREAD_INVALID_THREAD_ID - - Sentinel value for an invalid thread ID. - - This is currently equivalent to ``(unsigned long)-1``. - - -.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) - - Start function *func* in a new thread with argument *arg*. - The resulting thread is not intended to be joined. - - *func* must not be ``NULL``, but *arg* may be ``NULL``. - - On success, this function returns the identifier of the new thread; on failure, - this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: unsigned long PyThread_get_thread_ident(void) - - Return the identifier of the current thread, which will never be zero. - - This function cannot fail, and the caller does not need to hold an - :term:`attached thread state`. - - .. seealso:: - :py:func:`threading.get_ident` and :py:attr:`threading.Thread.ident` - expose this identifier to Python. - - -.. c:function:: PyObject *PyThread_GetInfo(void) - - Get general information about the current thread in the form of a - :ref:`struct sequence ` object. This information is - accessible as :py:attr:`sys.thread_info` in Python. - - On success, this returns a new :term:`strong reference` to the thread - information; on failure, this returns ``NULL`` with an exception set. - - The caller must hold an :term:`attached thread state`. - - -.. c:macro:: PY_HAVE_THREAD_NATIVE_ID - - This macro is defined when the system supports native thread IDs. - - -.. c:function:: unsigned long PyThread_get_thread_native_id(void) - - Get the native identifier of the current thread as it was assigned by the operating - system's kernel, which will never be less than zero. - - This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is - defined. - - This function cannot fail, and the caller does not need to hold an - :term:`attached thread state`. - - .. seealso:: - :py:func:`threading.get_native_id` - - -.. c:function:: void PyThread_exit_thread(void) - - Terminate the current thread. This function is generally considered unsafe - and should be avoided. It is kept solely for backwards compatibility. - - This function is only safe to call if all functions in the full call - stack are written to safely allow it. - - .. warning:: - - If the current system uses POSIX threads (also known as "pthreads"), - this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack - and call C++ destructors on some libc implementations. However, if a - ``noexcept`` function is reached, it may terminate the process. - Other systems, such as macOS, do unwinding. - - On Windows, this function calls ``_endthreadex()``, which kills the thread - without calling C++ destructors. - - In any case, there is a risk of corruption on the thread's stack. - - .. deprecated:: 3.14 - - -.. c:function:: void PyThread_init_thread(void) - - Initialize ``PyThread*`` APIs. Python executes this function automatically, - so there's little need to call it from an extension module. - - -.. c:function:: int PyThread_set_stacksize(size_t size) - - Set the stack size of the current thread to *size* bytes. - - This function returns ``0`` on success, ``-1`` if *size* is invalid, or - ``-2`` if the system does not support changing the stack size. This function - does not set exceptions. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: size_t PyThread_get_stacksize(void) - - Return the stack size of the current thread in bytes, or ``0`` if the system's - default stack size is in use. - - The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.4 + + .. soft-deprecated:: 3.15 + Use ``PyThreadState_GetUnchecked() != NULL`` instead. + + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + +Another important thing to note about threads is their behaviour in the face +of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains +means any locks held by other threads will never be released. Python solves +this for :func:`os.fork` by acquiring the locks it uses internally before +the fork, and releasing them afterwards. In addition, it resets any +:ref:`lock-objects` in the child. When extending or embedding Python, there +is no way to inform Python of additional (non-Python) locks that need to be +acquired before or reset after a fork. OS facilities such as +:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. +Additionally, when extending or embedding Python, calling :c:func:`fork` +directly rather than through :func:`os.fork` (and returning to or calling +into Python) may result in a deadlock by one of Python's internal locks +being held by a thread that is defunct after the fork. +:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not +always able to. + +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + + +High-level APIs +--------------- + +These are the most commonly used types and functions when writing multi-threaded +C extensions. + + +.. c:type:: PyThreadState + + This data structure represents the state of a single thread. The only public + data member is: + + .. c:member:: PyInterpreterState *interp + + This thread's interpreter state. + + +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + +.. c:function:: PyThreadState* PyEval_SaveThread() + + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. + + +.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached `, + otherwise deadlock ensues. *tstate* will be attached upon returning. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: PyThreadState* PyThreadState_Get() + + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). + + See also :c:func:`PyThreadState_GetUnchecked`. + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + + +.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso:: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + + +Low-level APIs +-------------- + +.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) + + Create a new thread state object belonging to the given interpreter object. + An :term:`attached thread state` is not needed. + +.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) + + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached ` + + .. versionchanged:: 3.9 + This function now calls the :c:member:`!PyThreadState.on_delete` callback. + Previously, that happened in :c:func:`PyThreadState_Delete`. + + .. versionchanged:: 3.13 + The :c:member:`!PyThreadState.on_delete` callback was removed. + + +.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) + + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached ` to any thread. + *tstate* must have been reset with a previous call to + :c:func:`PyThreadState_Clear`. + + +.. c:function:: void PyThreadState_DeleteCurrent(void) + + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + + No :term:`thread state` will be :term:`attached ` upon + returning. + +.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) + + Get the current frame of the Python thread state *tstate*. + + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently + executing. + + See also :c:func:`PyEval_GetFrame`. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) + + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) + + Get the interpreter of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyThreadState_GetDict() + + Return a dictionary in which extensions can store thread-specific state + information. Each extension should use a unique key to use to store state in + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached `. If this function returns + ``NULL``, no exception has been raised and the caller should assume no + thread state is attached. + + +.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) + + :term:`Attach ` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached `. + + The calling thread must not already have an :term:`attached thread state`. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + :c:func:`PyEval_RestoreThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) + + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is + reported. + + :c:func:`PyEval_SaveThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +Asynchronous notifications +========================== + +A mechanism is provided to make asynchronous notifications to the main +interpreter thread. These notifications take the form of a function +pointer and a void pointer argument. + + +.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) + + Schedule a function to be called from the main interpreter thread. On + success, ``0`` is returned and *func* is queued for being called in the + main thread. On failure, ``-1`` is returned without setting any exception. + + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: + + * on a :term:`bytecode` boundary; + * with the main thread holding an :term:`attached thread state` + (*func* can therefore use the full C API). + + *func* must return ``0`` on success, or ``-1`` on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the :term:`thread state ` is detached. + + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.9 + If this function is called in a subinterpreter, the function *func* is + now scheduled to be called from the subinterpreter, rather than being + called from the main interpreter. Each subinterpreter now has its own + list of scheduled calls. + + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + +.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) + + Schedule an exception to be raised asynchronously in a thread. + If the thread has a previously scheduled exception, it is overwritten. + + The *id* argument is the thread id of the target thread, as returned by + :c:func:`PyThread_get_thread_ident`. + *exc* is the class of the exception to be raised, or ``NULL`` to clear + the pending exception (if any). + + Return the number of affected thread states. + This is normally ``1`` if *id* is found, even when no change was + made (the given *exc* was already pending, or *exc* is ``NULL`` but + no exception is pending). + If the thread id isn't found, return ``0``. This raises no exceptions. + + To prevent naive misuse, you must write your own C extension to call this. + This function must be called with an :term:`attached thread state`. + This function does not steal any references to *exc*. + This function does not necessarily interrupt system calls such as + :py:func:`~time.sleep`. + + .. versionchanged:: 3.7 + The type of the *id* parameter changed from :c:expr:`long` to + :c:expr:`unsigned long`. + + +Operating system thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` and :py:attr:`threading.Thread.ident` + expose this identifier to Python. + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. From 74259f8c2ebeb017d11c18c00a3660a8a9b45529 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 29 Apr 2026 13:46:22 +0000 Subject: [PATCH 12/14] Fix lint and remove dead comment. --- Doc/c-api/interp-lifecycle.rst | 22 +- Doc/c-api/threads.rst | 2166 ++++++------ Doc/whatsnew/3.15.rst | 2 +- Python/pylifecycle.c | 6061 +++++++++++++++----------------- 4 files changed, 3979 insertions(+), 4272 deletions(-) diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index 7e3908ddde6f0c..8e8e2082f2057f 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -608,8 +608,8 @@ it possible to avoid these issues by temporarily preventing finalization: .. seealso:: - :pep:`788` explains the design, motivation and rationale - for these APIs. + :pep:`788` explains the design, motivation and rationale + for these APIs. .. c:type:: PyInterpreterGuard @@ -625,19 +625,19 @@ it possible to avoid these issues by temporarily preventing finalization: guards for that interpreter. This means that if you forget to close an interpreter guard, the process will **permanently hang** during finalization! - - Holding a guard for an interpreter is similar to holding a - :term:`strong reference` to a Python object, except finalization does not happen - automatically after all guards are released: it requires an explicit - :c:func:`Py_EndInterpreter` call. - + + Holding a guard for an interpreter is similar to holding a + :term:`strong reference` to a Python object, except finalization does not happen + automatically after all guards are released: it requires an explicit + :c:func:`Py_EndInterpreter` call. + .. versionadded:: next .. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromCurrent(void) Create a finalization guard for the current interpreter. This will prevent - finalization until the guard is closed. + finalization until the guard is closed. For example: @@ -770,8 +770,8 @@ deleted. This can be done using interpreter views. Failure indicates that the process is out of memory or that the main interpreter has finalized (or never existed). - Using this function in extension libraries is strongly discouraged, because - it typically compromises the library's subinterpreter support. It exists + Using this function in extension libraries is strongly discouraged, because + it typically compromises the library's subinterpreter support. It exists for exceptional cases where there is no other option (such as when a native threading library doesn't provide a ``void *arg`` parameter that could be used to store a ``PyInterpreterGuard`` or ``PyInterpreterView`` pointer). diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index f224d04589d739..08706a29f9a7f0 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -1,1083 +1,1083 @@ -.. highlight:: c - -.. _threads: - -Thread states and the global interpreter lock -============================================= - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -Unless on a :term:`free-threaded build` of :term:`CPython`, -the Python interpreter is generally not thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by a thread before -accessing Python objects. Without the lock, even the simplest operations -could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -As such, only a thread that holds the GIL may operate on Python objects or -invoke Python's C API. - -.. index:: single: setswitchinterval (in module sys) - -In order to emulate concurrency, the interpreter regularly tries to switch -threads between bytecode instructions (see :func:`sys.setswitchinterval`). -This is why locks are also necessary for thread-safety in pure-Python code. - -Additionally, the global interpreter lock is released around blocking I/O -operations, such as reading or writing to a file. From the C API, this is done -by :ref:`detaching the thread state `. - - -.. index:: - single: PyThreadState (C type) - -The Python interpreter keeps some thread-local information inside -a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. -Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state -referenced by this pointer is considered to be :term:`attached `. - -A thread can only have one :term:`attached thread state` at a time. An attached -thread state is typically analogous with holding the GIL, except on -free-threaded builds. On builds with the GIL enabled, attaching a thread state -will block until the GIL can be acquired. However, even on builds with the GIL -disabled, it is still required to have an attached thread state, as the interpreter -needs to keep track of which threads may access Python objects. - -.. note:: - - Even on the free-threaded build, attaching a thread state may block, as the - GIL can be re-enabled or threads might be temporarily suspended (such as during - a garbage collection). - -Generally, there will always be an attached thread state when using Python's -C API, including during embedding and when implementing methods, so it's uncommon -to need to set up a thread state on your own. Only in some specific cases, such -as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the -thread not have an attached thread state. -If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. - -If it turns out that you do need to create a thread state, it is recommended to -use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, -which will manage the thread state for you. - - -.. _detaching-thread-state: - -Detaching the thread state from extension code ----------------------------------------------- - -Most extension code manipulating the :term:`thread state` has the following simple -structure:: - - Save the thread state in a local variable. - ... Do some blocking I/O operation ... - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS (C macro) - single: Py_END_ALLOW_THREADS (C macro) - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. - -The block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ... Do some blocking I/O operation ... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread (C function) - single: PyEval_SaveThread (C function) - -Here is how these functions work: - -The attached thread state implies that the GIL is held for the interpreter. -To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored -in a local variable. - -By detaching the thread state, the GIL is released, which allows other threads -to attach to the interpreter and execute while the current thread performs -blocking I/O. When the I/O operation is complete, the old thread state is -reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until -the GIL can be acquired. - -.. note:: - Performing blocking I/O is the most common use case for detaching - the thread state, but it is also useful to call it over long-running - native code that doesn't need access to Python objects or Python's C API. - For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the - :term:`thread state ` when compressing or hashing - data. - -On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, -but **detaching the thread state is still required**, because the interpreter -periodically needs to block all threads to get a consistent view of Python objects -without the risk of race conditions. -For example, CPython currently suspends all threads for a short period of time -while running the garbage collector. - -.. warning:: - - Detaching the thread state can lead to unexpected behavior during interpreter - finalization. See :ref:`cautions-regarding-runtime-finalization` for more - details. - - -APIs -^^^^ - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - -.. note:: - - These macros are still necessary on the :term:`free-threaded build` to prevent - deadlocks. - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. - - -.. _non-python-created-threads: -.. _c-api-foreign-threads: - - -Using the C API from foreign threads ------------------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated with them, -However, when a thread is created from native code (for example, by a -third-party library with its own thread management), it doesn't hold an -attached thread state. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating a new thread state and attaching it. - -The easiest way to do this is through :c:func:`PyThreadState_Ensure` -or :c:func:`PyThreadState_EnsureFromView`. - -.. note:: - These functions require an argument pointing to the desired - interpreter; such a pointer can be acquired via a call to - :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or - :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) - from the function that creates the thread. If no pointer is available (such - as when the given native thread library doesn't provide a data argument), - :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main - interpreter, but note that this will make the code incompatible with - subinterpreters. - - -For example:: - - // The return value of PyInterpreterGuard_FromCurrent() from the - // function that created this thread. - PyInterpreterGuard *guard = thread_data->guard; - - // Create a new thread state for the interpreter. - PyThreadState *tstate = PyThreadState_Ensure(guard); - if (tstate == NULL) { - PyInterpreterGuard_Close(guard); - return; - } - - // We have a valid thread state -- perform Python actions here. - result = CallSomeFunction(); - // Evaluate result or handle exceptions. - - // Release the thread state. No calsl to the C API are allowed beyond this - // point. - PyThreadState_Release(tstate); - PyInterpreterGuard_Close(guard); - - -Some notes about this: - -1. In the above code, ``tstate`` is the *previously* attached thread state, not - the one that was just created! In some cases, ``PyThreadState_Ensure`` might - return an internal pointer, so it is **not** safe to treat ``tstate`` as a - valid thread state (that is, do not pass ``tstate`` to a function other than - ``PyThreadState_Release``). -2. Calling ``PyThreadState_Ensure`` might not always create a new thread state, - and calling ``PyThreadState_Release`` might not always detach it. These - functions may reuse an existing attached thread state, or may re-attach a - thread state that was previously attached for the current thread. - -.. seealso:: - :pep:`788` - -.. _c-api-attach-detach: - -Attaching/detaching thread states -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. c:function:: PyThreadState *PyThreadState_Ensure(PyInterpreterGuard *guard) - - Ensure that the thread has an attached thread state for the - interpreter protected by *guard*, and thus can safely invoke that - interpreter. - - It is OK to call this function if the thread already has an - attached thread state, as long as there is a subsequent call to - :c:func:`PyThreadState_Release` that matches this one. - - Nested calls to this function will only sometimes create a new - thread state. - - First, this function checks if an attached thread state is present. - If there is, this function then checks if the interpreter of that - thread state matches the interpreter guarded by *guard*. If that is - the case, this function simply marks the thread state as being used - by a ``PyThreadState_Ensure`` call and returns. - - If there is no attached thread state, then this function checks if any - thread state has been used by the current OS thread. (This is - returned by :c:func:`PyGILState_GetThisThreadState`.) - If there was, then this function checks if that thread state's interpreter - matches *guard*. If it does, it is re-attached and marked as used. - - Otherwise, if both of the above cases fail, a new thread state is created - for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. - - This function will return ``NULL`` to indicate a memory allocation failure, and - otherwise return a pointer to the thread state that was previously attached - (which might have been ``NULL``, in which case an non-``NULL`` sentinel value is - returned instead to differentiate between failure -- this means that this function - will sometimes return an invalid ``PyThreadState`` pointer). - - To visualize, this function is roughly equivalent to the following: - - .. code-block:: c - - PyThreadState * - PyThreadState_Ensure(PyInterpreterGuard *guard) - { - assert(guard != NULL); - PyInterpreterState *interp = PyInterpreterGuard_GetInterpreter(guard); - assert(interp != NULL); - - PyThreadState *current_tstate = PyThreadState_GetUnchecked(); - if (current_tstate == NULL) { - PyThreadState *last_used = PyGILState_GetThisThreadState(); - if (last_used != NULL) { - ++last_used->ensure_counter; - PyThreadState_Swap(last_used); - return NO_TSTATE_SENTINEL; - } - } else if (current_tstate->interp == interp) { - ++current_tstate->ensure_counter; - return current_tstate; - } - - PyThreadState *new_tstate = PyThreadState_New(interp); - if (new_tstate == NULL) { - return NULL; - } - - ++new_tstate->ensure_counter; - new_tstate->owned_by_pythreadstate_ensure = true; - PyThreadState_Swap(new_tstate); - return current_tstate == NULL ? NO_TSTATE_SENTINEL : current_tstate; - } - - .. versionadded:: next - - -.. c:function:: PyThreadState *PyThreadState_EnsureFromView(PyInterpreterView *view) - - Get an attached thread state for the interpreter referenced by *view*. - - *view* must not be ``NULL``. If the interpreter referenced by *view* has been - finalized or is currently finalizing, then this function returns ``NULL`` without - setting an exception. This function may also return ``NULL`` on other errors, - such memory allocation failure. - - The interpreter referenced by *view* will be implicitly guarded. The - guard will be released upon the corresponding :c:func:`PyThreadState_Release` - call. - - On success, this function will return the thread state that was previously attached. - If no thread state was previously attached, this returns a non-``NULL`` sentinel - value. The behavior of whether this function creates a thread state is - equivalent to that of :c:func:`PyThreadState_Ensure`. - - To visualize, function is roughly equivalent to the following: - - .. code-block:: c - - PyThreadState * - PyThreadState_EnsureFromView(PyInterpreterView *view) - { - assert(view != NULL); - PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); - if (guard == NULL) { - return NULL; - } - - PyThreadState *tstate = PyThreadState_Ensure(guard); - if (tstate == NULL) { - PyInterpreterGuard_Close(guard); - return NULL; - } - - if (tstate->guard == NULL) { - tstate->guard = guard; - } else { - PyInterpreterGuard_Close(guard); - } - - return tstate; - } - - -.. c:function:: void PyThreadState_Release(PyThreadState *tstate) - - Undo a :c:func:`PyThreadState_Ensure` call. This must be called exactly once - for each call to ``PyThreadState_Ensure``. - - This function will decrement an internal counter on the attached thread state. If - this counter ever reaches below zero, this function emits a fatal error (via - :c:func:`Py_FatalError`). - - If the attached thread state is owned by ``PyThreadState_Ensure``, then the - attached thread state will be deallocated and deleted upon the internal counter - reaching zero. Otherwise, nothing happens when the counter reaches zero. - - If *tstate* is non-``NULL``, it will be attached upon returning. - If *tstate* indicates that no prior thread state was attached, there will be - no attached thread state upon returning. - - To visualize, this function is roughly equivalent to the following: - - .. code-block:: c - - void - PyThreadState_Release(PyThreadState *old_tstate) - { - PyThreadState *current_tstate = PyThreadState_Get(); - assert(old_tstate != NULL); - assert(current_tstate != NULL); - assert(current_tstate->ensure_counter > 0); - if (--current_tstate->ensure_counter > 0) { - // There are remaining PyThreadState_Ensure() calls - // for this thread state. - return; - } - - assert(current_tstate->ensure_counter == 0); - if (old_tstate == NO_TSTATE_SENTINEL) { - // No thread state was attached prior the PyThreadState_Ensure() - // call. So, we can just destroy the current thread state and return. - assert(current_tstate->owned_by_pythreadstate_ensure); - PyThreadState_Clear(current_tstate); - PyThreadState_DeleteCurrent(); - return; - } - - if (tstate->guard != NULL) { - PyInterpreterGuard_Close(tstate->guard); - return; - } - - if (tstate->owned_by_pythreadstate_ensure) { - // The attached thread state was created by the initial PyThreadState_Ensure() - // call. It's our job to destroy it. - PyThreadState_Clear(current_tstate); - PyThreadState_DeleteCurrent(); - } - - PyThreadState_Swap(old_tstate); - } - -.. _legacy-api: -.. _gilstate: - -GIL-state APIs --------------- - -The following APIs are generally not compatible with subinterpreters and -will hang the process during interpreter finalization (see -:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were -:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs -`. - - -.. c:type:: PyGILState_STATE - - The type of the value returned by :c:func:`PyGILState_Ensure` and passed to - :c:func:`PyGILState_Release`. - - .. c:enumerator:: PyGILState_LOCKED - - The GIL was already held when :c:func:`PyGILState_Ensure` was called. - - .. c:enumerator:: PyGILState_UNLOCKED - - The GIL was not held when :c:func:`PyGILState_Ensure` was called. - - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. - - This function has no way to return an error. As such, errors are either fatal - (that is, they send ``SIGABRT`` and crash the process; see - :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as - during interpreter finalization). - - .. warning:: - Calling this function when the interpreter is finalizing will - infinitely hang the thread, which may cause deadlocks. - :ref:`cautions-regarding-runtime-finalization` for more details. - - In addition, this function generally does not work with subinterpreters - when used from foreign threads, because this function has no way of - knowing which interpreter created the thread (and as such, will implicitly - pick the main interpreter). - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - .. soft-deprecated:: 3.15 - Use :c:func:`PyThreadState_Ensure` or - :c:func:`PyThreadState_EnsureFromView` instead. - - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GIL-state API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - - .. soft-deprecated:: 3.15 - Use :c:func:`PyThreadState_Release` instead. - - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`thread state` that was most recently :term:`attached - ` for this thread. (If the most recent thread state - has been deleted, this returns ``NULL``.) - - If the caller has an attached thread state, it is returned. - - In other terms, this function returns the thread state that will be used by - :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then - ``PyGILState_Ensure`` will create a new thread state. - - This function cannot fail. - - .. soft-deprecated:: 3.15 - Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - instead. - - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread has an :term:`attached thread state` - that matches the thread state returned by - :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread - state or it otherwise doesn't match, then this returns ``0``. - - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. - - This is mainly a helper/diagnostic function. - - .. versionadded:: 3.4 - - .. soft-deprecated:: 3.15 - Use ``PyThreadState_GetUnchecked() != NULL`` instead. - - -.. _fork-and-threads: - -Cautions about fork() ---------------------- - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. This has a -concrete impact both on how locks must be handled and on all stored state -in CPython's runtime. - -The fact that only the "current" thread remains -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not -always able to. - -The fact that all other threads go away also means that CPython's -runtime state there must be cleaned up properly, which :func:`os.fork` -does. This means finalizing all other :c:type:`PyThreadState` objects -belonging to the current interpreter and all other -:c:type:`PyInterpreterState` objects. Due to this and the special -nature of the :ref:`"main" interpreter `, -:c:func:`fork` should only be called in that interpreter's "main" -thread, where the CPython global runtime was originally initialized. -The only exception is if :c:func:`exec` will be called immediately -after. - - -High-level APIs ---------------- - -These are the most commonly used types and functions when writing multi-threaded -C extensions. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is: - - .. c:member:: PyInterpreterState *interp - - This thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Detach the :term:`attached thread state` and return it. - The thread will have no :term:`thread state` upon returning. - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*. - The passed :term:`thread state` **should not** be :term:`attached `, - otherwise deadlock ensues. *tstate* will be attached upon returning. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the :term:`attached thread state`. If the thread has no attached - thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` - block), then this issues a fatal error (so that the caller needn't check - for ``NULL``). - - See also :c:func:`PyThreadState_GetUnchecked`. - -.. c:function:: PyThreadState* PyThreadState_GetUnchecked() - - Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a - fatal error if it is NULL. The caller is responsible to check if the result - is NULL. - - .. versionadded:: 3.13 - In Python 3.5 to 3.12, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*, and return the - :term:`thread state` that was attached prior to calling. - - This function is safe to call without an :term:`attached thread state`; it - will simply return ``NULL`` indicating that there was no prior thread state. - - .. seealso:: - :c:func:`PyEval_ReleaseThread` - - .. note:: - Similar to :c:func:`PyGILState_Ensure`, this function will hang the - thread if the runtime is finalizing. - - -Low-level APIs --------------- - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - An :term:`attached thread state` is not needed. - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a :term:`thread state` object. *tstate* - must be :term:`attached ` - - .. versionchanged:: 3.9 - This function now calls the :c:member:`!PyThreadState.on_delete` callback. - Previously, that happened in :c:func:`PyThreadState_Delete`. - - .. versionchanged:: 3.13 - The :c:member:`!PyThreadState.on_delete` callback was removed. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a :term:`thread state` object. *tstate* should not - be :term:`attached ` to any thread. - *tstate* must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: void PyThreadState_DeleteCurrent(void) - - Detach the :term:`attached thread state` (which must have been reset - with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. - - No :term:`thread state` will be :term:`attached ` upon - returning. - -.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) - - Get the current frame of the Python thread state *tstate*. - - Return a :term:`strong reference`. Return ``NULL`` if no frame is currently - executing. - - See also :c:func:`PyEval_GetFrame`. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - - Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) - - Get the interpreter of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) - - Suspend tracing and profiling in the Python thread state *tstate*. - - Resume them using the :c:func:`PyThreadState_LeaveTracing` function. - - .. versionadded:: 3.11 - - -.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) - - Resume tracing and profiling in the Python thread state *tstate* suspended - by the :c:func:`PyThreadState_EnterTracing` function. - - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. - - .. versionadded:: 3.11 - - -.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) - - Set the stack protection start address and stack protection size - of a Python thread state. - - On success, return ``0``. - On failure, set an exception and return ``-1``. - - CPython implements :ref:`recursion control ` for C code by raising - :py:exc:`RecursionError` when it notices that the machine execution stack is close - to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. - For this, it needs to know the location of the current thread's stack, which it - normally gets from the operating system. - When the stack is changed, for example using context switching techniques like the - Boost library's ``boost::context``, you must call - :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. - - Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before - or after changing the stack. - Do not call any other Python C API between the call and the stack - change. - - See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. - - .. versionadded:: 3.15 - - -.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) - - Reset the stack protection start address and stack protection size - of a Python thread state to the operating system defaults. - - See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. - - .. versionadded:: 3.15 - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no :term:`thread state` - is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no - thread state is attached. - - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - :term:`Attach ` *tstate* to the current thread, - which must not be ``NULL`` or already :term:`attached `. - - The calling thread must not already have an :term:`attached thread state`. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Detach the :term:`attached thread state`. - The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the :term:`attached thread state` --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -Asynchronous notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding an :term:`attached thread state` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the :term:`thread state ` is detached. - - This function doesn't need an :term:`attached thread state`. However, to call this - function in a subinterpreter, the caller must have an :term:`attached thread state`. - Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.9 - If this function is called in a subinterpreter, the function *func* is - now scheduled to be called from the subinterpreter, rather than being - called from the main interpreter. Each subinterpreter now has its own - list of scheduled calls. - - .. versionchanged:: 3.12 - This function now always schedules *func* to be run in the main - interpreter. - - -.. c:function:: int Py_MakePendingCalls(void) - - Execute all pending calls. This is usually executed automatically by the - interpreter. - - This function returns ``0`` on success, and returns ``-1`` with an exception - set on failure. - - If this is not called in the main thread of the main - interpreter, this function does nothing and returns ``0``. - The caller must hold an :term:`attached thread state`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.12 - This function only runs pending calls in the main interpreter. - - -.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) - - Schedule an exception to be raised asynchronously in a thread. - If the thread has a previously scheduled exception, it is overwritten. - - The *id* argument is the thread id of the target thread, as returned by - :c:func:`PyThread_get_thread_ident`. - *exc* is the class of the exception to be raised, or ``NULL`` to clear - the pending exception (if any). - - Return the number of affected thread states. - This is normally ``1`` if *id* is found, even when no change was - made (the given *exc* was already pending, or *exc* is ``NULL`` but - no exception is pending). - If the thread id isn't found, return ``0``. This raises no exceptions. - - To prevent naive misuse, you must write your own C extension to call this. - This function must be called with an :term:`attached thread state`. - This function does not steal any references to *exc*. - This function does not necessarily interrupt system calls such as - :py:func:`~time.sleep`. - - .. versionchanged:: 3.7 - The type of the *id* parameter changed from :c:expr:`long` to - :c:expr:`unsigned long`. - - -Operating system thread APIs -============================ - -.. c:macro:: PYTHREAD_INVALID_THREAD_ID - - Sentinel value for an invalid thread ID. - - This is currently equivalent to ``(unsigned long)-1``. - - -.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) - - Start function *func* in a new thread with argument *arg*. - The resulting thread is not intended to be joined. - - *func* must not be ``NULL``, but *arg* may be ``NULL``. - - On success, this function returns the identifier of the new thread; on failure, - this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: unsigned long PyThread_get_thread_ident(void) - - Return the identifier of the current thread, which will never be zero. - - This function cannot fail, and the caller does not need to hold an - :term:`attached thread state`. - - .. seealso:: - :py:func:`threading.get_ident` and :py:attr:`threading.Thread.ident` - expose this identifier to Python. - - -.. c:function:: PyObject *PyThread_GetInfo(void) - - Get general information about the current thread in the form of a - :ref:`struct sequence ` object. This information is - accessible as :py:attr:`sys.thread_info` in Python. - - On success, this returns a new :term:`strong reference` to the thread - information; on failure, this returns ``NULL`` with an exception set. - - The caller must hold an :term:`attached thread state`. - - -.. c:macro:: PY_HAVE_THREAD_NATIVE_ID - - This macro is defined when the system supports native thread IDs. - - -.. c:function:: unsigned long PyThread_get_thread_native_id(void) - - Get the native identifier of the current thread as it was assigned by the operating - system's kernel, which will never be less than zero. - - This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is - defined. - - This function cannot fail, and the caller does not need to hold an - :term:`attached thread state`. - - .. seealso:: - :py:func:`threading.get_native_id` - - -.. c:function:: void PyThread_exit_thread(void) - - Terminate the current thread. This function is generally considered unsafe - and should be avoided. It is kept solely for backwards compatibility. - - This function is only safe to call if all functions in the full call - stack are written to safely allow it. - - .. warning:: - - If the current system uses POSIX threads (also known as "pthreads"), - this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack - and call C++ destructors on some libc implementations. However, if a - ``noexcept`` function is reached, it may terminate the process. - Other systems, such as macOS, do unwinding. - - On Windows, this function calls ``_endthreadex()``, which kills the thread - without calling C++ destructors. - - In any case, there is a risk of corruption on the thread's stack. - - .. deprecated:: 3.14 - - -.. c:function:: void PyThread_init_thread(void) - - Initialize ``PyThread*`` APIs. Python executes this function automatically, - so there's little need to call it from an extension module. - - -.. c:function:: int PyThread_set_stacksize(size_t size) - - Set the stack size of the current thread to *size* bytes. - - This function returns ``0`` on success, ``-1`` if *size* is invalid, or - ``-2`` if the system does not support changing the stack size. This function - does not set exceptions. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: size_t PyThread_get_stacksize(void) - - Return the stack size of the current thread in bytes, or ``0`` if the system's - default stack size is in use. - - The caller does not need to hold an :term:`attached thread state`. +.. highlight:: c + +.. _threads: + +Thread states and the global interpreter lock +============================================= + +.. index:: + single: global interpreter lock + single: interpreter lock + single: lock, interpreter + +Unless on a :term:`free-threaded build` of :term:`CPython`, +the Python interpreter is generally not thread-safe. In order to support +multi-threaded Python programs, there's a global lock, called the :term:`global +interpreter lock` or :term:`GIL`, that must be held by a thread before +accessing Python objects. Without the lock, even the simplest operations +could cause problems in a multi-threaded program: for example, when +two threads simultaneously increment the reference count of the same object, the +reference count could end up being incremented only once instead of twice. + +As such, only a thread that holds the GIL may operate on Python objects or +invoke Python's C API. + +.. index:: single: setswitchinterval (in module sys) + +In order to emulate concurrency, the interpreter regularly tries to switch +threads between bytecode instructions (see :func:`sys.setswitchinterval`). +This is why locks are also necessary for thread-safety in pure-Python code. + +Additionally, the global interpreter lock is released around blocking I/O +operations, such as reading or writing to a file. From the C API, this is done +by :ref:`detaching the thread state `. + + +.. index:: + single: PyThreadState (C type) + +The Python interpreter keeps some thread-local information inside +a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached `. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the GIL, except on +free-threaded builds. On builds with the GIL enabled, attaching a thread state +will block until the GIL can be acquired. However, even on builds with the GIL +disabled, it is still required to have an attached thread state, as the interpreter +needs to keep track of which threads may access Python objects. + +.. note:: + + Even on the free-threaded build, attaching a thread state may block, as the + GIL can be re-enabled or threads might be temporarily suspended (such as during + a garbage collection). + +Generally, there will always be an attached thread state when using Python's +C API, including during embedding and when implementing methods, so it's uncommon +to need to set up a thread state on your own. Only in some specific cases, such +as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the +thread not have an attached thread state. +If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. + +If it turns out that you do need to create a thread state, it is recommended to +use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, +which will manage the thread state for you. + + +.. _detaching-thread-state: + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple +structure:: + + Save the thread state in a local variable. + ... Do some blocking I/O operation ... + Restore the thread state from the local variable. + +This is so common that a pair of macros exists to simplify it:: + + Py_BEGIN_ALLOW_THREADS + ... Do some blocking I/O operation ... + Py_END_ALLOW_THREADS + +.. index:: + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) + +The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a +hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the +block. + +The block above expands to the following code:: + + PyThreadState *_save; + + _save = PyEval_SaveThread(); + ... Do some blocking I/O operation ... + PyEval_RestoreThread(_save); + +.. index:: + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) + +Here is how these functions work: + +The attached thread state implies that the GIL is held for the interpreter. +To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored +in a local variable. + +By detaching the thread state, the GIL is released, which allows other threads +to attach to the interpreter and execute while the current thread performs +blocking I/O. When the I/O operation is complete, the old thread state is +reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until +the GIL can be acquired. + +.. note:: + Performing blocking I/O is the most common use case for detaching + the thread state, but it is also useful to call it over long-running + native code that doesn't need access to Python objects or Python's C API. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state ` when compressing or hashing + data. + +On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, +but **detaching the thread state is still required**, because the interpreter +periodically needs to block all threads to get a consistent view of Python objects +without the risk of race conditions. +For example, CPython currently suspends all threads for a short period of time +while running the garbage collector. + +.. warning:: + + Detaching the thread state can lead to unexpected behavior during interpreter + finalization. See :ref:`cautions-regarding-runtime-finalization` for more + details. + + +APIs +^^^^ + +The following macros are normally used without a trailing semicolon; look for +example usage in the Python source distribution. + +.. note:: + + These macros are still necessary on the :term:`free-threaded build` to prevent + deadlocks. + +.. c:macro:: Py_BEGIN_ALLOW_THREADS + + This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. + Note that it contains an opening brace; it must be matched with a following + :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this + macro. + + +.. c:macro:: Py_END_ALLOW_THREADS + + This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains + a closing brace; it must be matched with an earlier + :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of + this macro. + + +.. c:macro:: Py_BLOCK_THREADS + + This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. + + +.. c:macro:: Py_UNBLOCK_THREADS + + This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to + :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable + declaration. + + +.. _non-python-created-threads: +.. _c-api-foreign-threads: + + +Using the C API from foreign threads +------------------------------------ + +When threads are created using the dedicated Python APIs (such as the +:mod:`threading` module), a thread state is automatically associated with them, +However, when a thread is created from native code (for example, by a +third-party library with its own thread management), it doesn't hold an +attached thread state. + +If you need to call Python code from these threads (often this will be part +of a callback API provided by the aforementioned third-party library), +you must first register these threads with the interpreter by +creating a new thread state and attaching it. + +The easiest way to do this is through :c:func:`PyThreadState_Ensure` +or :c:func:`PyThreadState_EnsureFromView`. + +.. note:: + These functions require an argument pointing to the desired + interpreter; such a pointer can be acquired via a call to + :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or + :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) + from the function that creates the thread. If no pointer is available (such + as when the given native thread library doesn't provide a data argument), + :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main + interpreter, but note that this will make the code incompatible with + subinterpreters. + + +For example:: + + // The return value of PyInterpreterGuard_FromCurrent() from the + // function that created this thread. + PyInterpreterGuard *guard = thread_data->guard; + + // Create a new thread state for the interpreter. + PyThreadState *tstate = PyThreadState_Ensure(guard); + if (tstate == NULL) { + PyInterpreterGuard_Close(guard); + return; + } + + // We have a valid thread state -- perform Python actions here. + result = CallSomeFunction(); + // Evaluate result or handle exceptions. + + // Release the thread state. No calsl to the C API are allowed beyond this + // point. + PyThreadState_Release(tstate); + PyInterpreterGuard_Close(guard); + + +Some notes about this: + +1. In the above code, ``tstate`` is the *previously* attached thread state, not + the one that was just created! In some cases, ``PyThreadState_Ensure`` might + return an internal pointer, so it is **not** safe to treat ``tstate`` as a + valid thread state (that is, do not pass ``tstate`` to a function other than + ``PyThreadState_Release``). +2. Calling ``PyThreadState_Ensure`` might not always create a new thread state, + and calling ``PyThreadState_Release`` might not always detach it. These + functions may reuse an existing attached thread state, or may re-attach a + thread state that was previously attached for the current thread. + +.. seealso:: + :pep:`788` + +.. _c-api-attach-detach: + +Attaching/detaching thread states +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyThreadState *PyThreadState_Ensure(PyInterpreterGuard *guard) + + Ensure that the thread has an attached thread state for the + interpreter protected by *guard*, and thus can safely invoke that + interpreter. + + It is OK to call this function if the thread already has an + attached thread state, as long as there is a subsequent call to + :c:func:`PyThreadState_Release` that matches this one. + + Nested calls to this function will only sometimes create a new + thread state. + + First, this function checks if an attached thread state is present. + If there is, this function then checks if the interpreter of that + thread state matches the interpreter guarded by *guard*. If that is + the case, this function simply marks the thread state as being used + by a ``PyThreadState_Ensure`` call and returns. + + If there is no attached thread state, then this function checks if any + thread state has been used by the current OS thread. (This is + returned by :c:func:`PyGILState_GetThisThreadState`.) + If there was, then this function checks if that thread state's interpreter + matches *guard*. If it does, it is re-attached and marked as used. + + Otherwise, if both of the above cases fail, a new thread state is created + for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. + + This function will return ``NULL`` to indicate a memory allocation failure, and + otherwise return a pointer to the thread state that was previously attached + (which might have been ``NULL``, in which case an non-``NULL`` sentinel value is + returned instead to differentiate between failure -- this means that this function + will sometimes return an invalid ``PyThreadState`` pointer). + + To visualize, this function is roughly equivalent to the following: + + .. code-block:: c + + PyThreadState * + PyThreadState_Ensure(PyInterpreterGuard *guard) + { + assert(guard != NULL); + PyInterpreterState *interp = PyInterpreterGuard_GetInterpreter(guard); + assert(interp != NULL); + + PyThreadState *current_tstate = PyThreadState_GetUnchecked(); + if (current_tstate == NULL) { + PyThreadState *last_used = PyGILState_GetThisThreadState(); + if (last_used != NULL) { + ++last_used->ensure_counter; + PyThreadState_Swap(last_used); + return NO_TSTATE_SENTINEL; + } + } else if (current_tstate->interp == interp) { + ++current_tstate->ensure_counter; + return current_tstate; + } + + PyThreadState *new_tstate = PyThreadState_New(interp); + if (new_tstate == NULL) { + return NULL; + } + + ++new_tstate->ensure_counter; + new_tstate->owned_by_pythreadstate_ensure = true; + PyThreadState_Swap(new_tstate); + return current_tstate == NULL ? NO_TSTATE_SENTINEL : current_tstate; + } + + .. versionadded:: next + + +.. c:function:: PyThreadState *PyThreadState_EnsureFromView(PyInterpreterView *view) + + Get an attached thread state for the interpreter referenced by *view*. + + *view* must not be ``NULL``. If the interpreter referenced by *view* has been + finalized or is currently finalizing, then this function returns ``NULL`` without + setting an exception. This function may also return ``NULL`` on other errors, + such memory allocation failure. + + The interpreter referenced by *view* will be implicitly guarded. The + guard will be released upon the corresponding :c:func:`PyThreadState_Release` + call. + + On success, this function will return the thread state that was previously attached. + If no thread state was previously attached, this returns a non-``NULL`` sentinel + value. The behavior of whether this function creates a thread state is + equivalent to that of :c:func:`PyThreadState_Ensure`. + + To visualize, function is roughly equivalent to the following: + + .. code-block:: c + + PyThreadState * + PyThreadState_EnsureFromView(PyInterpreterView *view) + { + assert(view != NULL); + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + if (guard == NULL) { + return NULL; + } + + PyThreadState *tstate = PyThreadState_Ensure(guard); + if (tstate == NULL) { + PyInterpreterGuard_Close(guard); + return NULL; + } + + if (tstate->guard == NULL) { + tstate->guard = guard; + } else { + PyInterpreterGuard_Close(guard); + } + + return tstate; + } + + +.. c:function:: void PyThreadState_Release(PyThreadState *tstate) + + Undo a :c:func:`PyThreadState_Ensure` call. This must be called exactly once + for each call to ``PyThreadState_Ensure``. + + This function will decrement an internal counter on the attached thread state. If + this counter ever reaches below zero, this function emits a fatal error (via + :c:func:`Py_FatalError`). + + If the attached thread state is owned by ``PyThreadState_Ensure``, then the + attached thread state will be deallocated and deleted upon the internal counter + reaching zero. Otherwise, nothing happens when the counter reaches zero. + + If *tstate* is non-``NULL``, it will be attached upon returning. + If *tstate* indicates that no prior thread state was attached, there will be + no attached thread state upon returning. + + To visualize, this function is roughly equivalent to the following: + + .. code-block:: c + + void + PyThreadState_Release(PyThreadState *old_tstate) + { + PyThreadState *current_tstate = PyThreadState_Get(); + assert(old_tstate != NULL); + assert(current_tstate != NULL); + assert(current_tstate->ensure_counter > 0); + if (--current_tstate->ensure_counter > 0) { + // There are remaining PyThreadState_Ensure() calls + // for this thread state. + return; + } + + assert(current_tstate->ensure_counter == 0); + if (old_tstate == NO_TSTATE_SENTINEL) { + // No thread state was attached prior the PyThreadState_Ensure() + // call. So, we can just destroy the current thread state and return. + assert(current_tstate->owned_by_pythreadstate_ensure); + PyThreadState_Clear(current_tstate); + PyThreadState_DeleteCurrent(); + return; + } + + if (tstate->guard != NULL) { + PyInterpreterGuard_Close(tstate->guard); + return; + } + + if (tstate->owned_by_pythreadstate_ensure) { + // The attached thread state was created by the initial PyThreadState_Ensure() + // call. It's our job to destroy it. + PyThreadState_Clear(current_tstate); + PyThreadState_DeleteCurrent(); + } + + PyThreadState_Swap(old_tstate); + } + +.. _legacy-api: +.. _gilstate: + +GIL-state APIs +-------------- + +The following APIs are generally not compatible with subinterpreters and +will hang the process during interpreter finalization (see +:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were +:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs +`. + + +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. + + This function has no way to return an error. As such, errors are either fatal + (that is, they send ``SIGABRT`` and crash the process; see + :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as + during interpreter finalization). + + .. warning:: + Calling this function when the interpreter is finalizing will + infinitely hang the thread, which may cause deadlocks. + :ref:`cautions-regarding-runtime-finalization` for more details. + + In addition, this function generally does not work with subinterpreters + when used from foreign threads, because this function has no way of + knowing which interpreter created the thread (and as such, will implicitly + pick the main interpreter). + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` instead. + + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GIL-state API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Release` instead. + + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`thread state` that was most recently :term:`attached + ` for this thread. (If the most recent thread state + has been deleted, this returns ``NULL``.) + + If the caller has an attached thread state, it is returned. + + In other terms, this function returns the thread state that will be used by + :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then + ``PyGILState_Ensure`` will create a new thread state. + + This function cannot fail. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + instead. + + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread has an :term:`attached thread state` + that matches the thread state returned by + :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread + state or it otherwise doesn't match, then this returns ``0``. + + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. + + This is mainly a helper/diagnostic function. + + .. versionadded:: 3.4 + + .. soft-deprecated:: 3.15 + Use ``PyThreadState_GetUnchecked() != NULL`` instead. + + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + +Another important thing to note about threads is their behaviour in the face +of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains +means any locks held by other threads will never be released. Python solves +this for :func:`os.fork` by acquiring the locks it uses internally before +the fork, and releasing them afterwards. In addition, it resets any +:ref:`lock-objects` in the child. When extending or embedding Python, there +is no way to inform Python of additional (non-Python) locks that need to be +acquired before or reset after a fork. OS facilities such as +:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. +Additionally, when extending or embedding Python, calling :c:func:`fork` +directly rather than through :func:`os.fork` (and returning to or calling +into Python) may result in a deadlock by one of Python's internal locks +being held by a thread that is defunct after the fork. +:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not +always able to. + +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + + +High-level APIs +--------------- + +These are the most commonly used types and functions when writing multi-threaded +C extensions. + + +.. c:type:: PyThreadState + + This data structure represents the state of a single thread. The only public + data member is: + + .. c:member:: PyInterpreterState *interp + + This thread's interpreter state. + + +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + +.. c:function:: PyThreadState* PyEval_SaveThread() + + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. + + +.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached `, + otherwise deadlock ensues. *tstate* will be attached upon returning. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: PyThreadState* PyThreadState_Get() + + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). + + See also :c:func:`PyThreadState_GetUnchecked`. + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + + +.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso:: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + + +Low-level APIs +-------------- + +.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) + + Create a new thread state object belonging to the given interpreter object. + An :term:`attached thread state` is not needed. + +.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) + + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached ` + + .. versionchanged:: 3.9 + This function now calls the :c:member:`!PyThreadState.on_delete` callback. + Previously, that happened in :c:func:`PyThreadState_Delete`. + + .. versionchanged:: 3.13 + The :c:member:`!PyThreadState.on_delete` callback was removed. + + +.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) + + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached ` to any thread. + *tstate* must have been reset with a previous call to + :c:func:`PyThreadState_Clear`. + + +.. c:function:: void PyThreadState_DeleteCurrent(void) + + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + + No :term:`thread state` will be :term:`attached ` upon + returning. + +.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) + + Get the current frame of the Python thread state *tstate*. + + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently + executing. + + See also :c:func:`PyEval_GetFrame`. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) + + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) + + Get the interpreter of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyThreadState_GetDict() + + Return a dictionary in which extensions can store thread-specific state + information. Each extension should use a unique key to use to store state in + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached `. If this function returns + ``NULL``, no exception has been raised and the caller should assume no + thread state is attached. + + +.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) + + :term:`Attach ` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached `. + + The calling thread must not already have an :term:`attached thread state`. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + :c:func:`PyEval_RestoreThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) + + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is + reported. + + :c:func:`PyEval_SaveThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +Asynchronous notifications +========================== + +A mechanism is provided to make asynchronous notifications to the main +interpreter thread. These notifications take the form of a function +pointer and a void pointer argument. + + +.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) + + Schedule a function to be called from the main interpreter thread. On + success, ``0`` is returned and *func* is queued for being called in the + main thread. On failure, ``-1`` is returned without setting any exception. + + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: + + * on a :term:`bytecode` boundary; + * with the main thread holding an :term:`attached thread state` + (*func* can therefore use the full C API). + + *func* must return ``0`` on success, or ``-1`` on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the :term:`thread state ` is detached. + + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.9 + If this function is called in a subinterpreter, the function *func* is + now scheduled to be called from the subinterpreter, rather than being + called from the main interpreter. Each subinterpreter now has its own + list of scheduled calls. + + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + +.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) + + Schedule an exception to be raised asynchronously in a thread. + If the thread has a previously scheduled exception, it is overwritten. + + The *id* argument is the thread id of the target thread, as returned by + :c:func:`PyThread_get_thread_ident`. + *exc* is the class of the exception to be raised, or ``NULL`` to clear + the pending exception (if any). + + Return the number of affected thread states. + This is normally ``1`` if *id* is found, even when no change was + made (the given *exc* was already pending, or *exc* is ``NULL`` but + no exception is pending). + If the thread id isn't found, return ``0``. This raises no exceptions. + + To prevent naive misuse, you must write your own C extension to call this. + This function must be called with an :term:`attached thread state`. + This function does not steal any references to *exc*. + This function does not necessarily interrupt system calls such as + :py:func:`~time.sleep`. + + .. versionchanged:: 3.7 + The type of the *id* parameter changed from :c:expr:`long` to + :c:expr:`unsigned long`. + + +Operating system thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` and :py:attr:`threading.Thread.ident` + expose this identifier to Python. + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9914b030c06103..f1ea7e7b587898 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -456,7 +456,7 @@ In the C API, :term:`interpreter finalization ` can be problematic for many extensions, because :term:`attaching ` a thread state will permanently hang the thread, resulting in deadlocks and other spurious issues. Additionally, it has historically been impossible -to safely check whether an interpreter is alive before using it, leading to crashes +to safely check whether an interpreter is alive before using it, leading to crashes when a thread concurrently deletes an interpreter while another thread is trying to attach to it. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 57dd57fa586c79..ee3b2d4c06165a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1,18 +1,18 @@ /* Python interpreter top-level routines, including init/exit */ #include "Python.h" -#include "pycore_audit.h" // _PySys_ClearAuditHooks() -#include "pycore_call.h" // _PyObject_CallMethod() -#include "pycore_ceval.h" // _PyEval_FiniGIL() -#include "pycore_codecs.h" // _PyCodec_Lookup() -#include "pycore_context.h" // _PyContext_Init() -#include "pycore_dict.h" // _PyDict_Fini() -#include "pycore_exceptions.h" // _PyExc_InitTypes() -#include "pycore_fileutils.h" // _Py_ResetForceASCII() -#include "pycore_floatobject.h" // _PyFloat_InitTypes() -#include "pycore_freelist.h" // _PyObject_ClearFreeLists() -#include "pycore_global_objects_fini_generated.h" // _PyStaticObjects_CheckRefcnt() -#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_audit.h" // _PySys_ClearAuditHooks() +#include "pycore_call.h" // _PyObject_CallMethod() +#include "pycore_ceval.h" // _PyEval_FiniGIL() +#include "pycore_codecs.h" // _PyCodec_Lookup() +#include "pycore_context.h" // _PyContext_Init() +#include "pycore_dict.h" // _PyDict_Fini() +#include "pycore_exceptions.h" // _PyExc_InitTypes() +#include "pycore_fileutils.h" // _Py_ResetForceASCII() +#include "pycore_floatobject.h" // _PyFloat_InitTypes() +#include "pycore_freelist.h" // _PyObject_ClearFreeLists() +#include "pycore_global_objects_fini_generated.h" // _PyStaticObjects_CheckRefcnt() +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interpolation.h" // _PyInterpolation_InitTypes() #include "pycore_long.h" // _PyLong_InitTypes() #include "pycore_moduleobject.h" // _PyModule_InitModuleDictWatcher() @@ -44,54 +44,54 @@ #include "opcode.h" -#include // setlocale() -#include // getenv() +#include // setlocale() +#include // getenv() #ifdef HAVE_UNISTD_H -# include // isatty() +#include // isatty() #endif #if defined(__APPLE__) -# include -# include -# include +#include +#include +#include // The os_log unified logging APIs were introduced in macOS 10.12, iOS 10.0, // tvOS 10.0, and watchOS 3.0; -# if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE -# define HAS_APPLE_SYSTEM_LOG 1 -# elif defined(TARGET_OS_OSX) && TARGET_OS_OSX -# if defined(MAC_OS_X_VERSION_10_12) && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 -# define HAS_APPLE_SYSTEM_LOG 1 -# else -# define HAS_APPLE_SYSTEM_LOG 0 -# endif -# else -# define HAS_APPLE_SYSTEM_LOG 0 -# endif - -# if HAS_APPLE_SYSTEM_LOG -# include -# endif +#if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +#define HAS_APPLE_SYSTEM_LOG 1 +#elif defined(TARGET_OS_OSX) && TARGET_OS_OSX +#if defined(MAC_OS_X_VERSION_10_12) && \ + MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 +#define HAS_APPLE_SYSTEM_LOG 1 +#else +#define HAS_APPLE_SYSTEM_LOG 0 +#endif +#else +#define HAS_APPLE_SYSTEM_LOG 0 +#endif + +#if HAS_APPLE_SYSTEM_LOG +#include +#endif #endif #ifdef HAVE_SIGNAL_H -# include // SIG_IGN +#include // SIG_IGN #endif #ifdef HAVE_LANGINFO_H -# include // nl_langinfo(CODESET) +#include // nl_langinfo(CODESET) #endif #ifdef HAVE_FCNTL_H -# include // F_GETFD +#include // F_GETFD #endif #ifdef MS_WINDOWS -# undef BYTE +#undef BYTE #endif #define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) - /* Forward declarations */ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); @@ -107,76 +107,61 @@ static void wait_for_thread_shutdown(PyThreadState *tstate); static void finalize_subinterpreters(void); static void call_ll_exitfuncs(_PyRuntimeState *runtime); - /* The following places the `_PyRuntime` structure in a location that can be * found without any external information. This is meant to ease access to the * interpreter state for various runtime debugging tools, but is *not* an * officially supported feature */ /* Suppress deprecation warning for PyBytesObject.ob_shash */ -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS +_Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS -GENERATE_DEBUG_SECTION(PyRuntime, _PyRuntimeState _PyRuntime) -= _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); +GENERATE_DEBUG_SECTION(PyRuntime, _PyRuntimeState _PyRuntime) = + _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP + static int runtime_initialized = 0; -static int runtime_initialized = 0; - -PyStatus -_PyRuntime_Initialize(void) -{ - /* XXX We only initialize once in the process, which aligns with - the static initialization of the former globals now found in - _PyRuntime. However, _PyRuntime *should* be initialized with - every Py_Initialize() call, but doing so breaks the runtime. - This is because the runtime state is not properly finalized - currently. */ - if (runtime_initialized) { - return _PyStatus_OK(); - } - runtime_initialized = 1; +PyStatus _PyRuntime_Initialize(void) { + /* XXX We only initialize once in the process, which aligns with + the static initialization of the former globals now found in + _PyRuntime. However, _PyRuntime *should* be initialized with + every Py_Initialize() call, but doing so breaks the runtime. + This is because the runtime state is not properly finalized + currently. */ + if (runtime_initialized) { + return _PyStatus_OK(); + } + runtime_initialized = 1; - return _PyRuntimeState_Init(&_PyRuntime); + return _PyRuntimeState_Init(&_PyRuntime); } -void -_PyRuntime_Finalize(void) -{ - _PyRuntimeState_Fini(&_PyRuntime); - runtime_initialized = 0; +void _PyRuntime_Finalize(void) { + _PyRuntimeState_Fini(&_PyRuntime); + runtime_initialized = 0; } -int -Py_IsFinalizing(void) -{ - return _PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL; +int Py_IsFinalizing(void) { + return _PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL; } /* Hack to force loading of object files */ -int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t) = \ +int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t) = PyOS_mystrnicmp; /* Python/pystrcmp.o */ - /* APIs to access the initialization flags * * Can be called prior to Py_Initialize. */ -int -_Py_IsCoreInitialized(void) -{ - return _PyRuntimeState_GetCoreInitialized(&_PyRuntime); +int _Py_IsCoreInitialized(void) { + return _PyRuntimeState_GetCoreInitialized(&_PyRuntime); } -int -Py_IsInitialized(void) -{ - return _PyRuntimeState_GetInitialized(&_PyRuntime); +int Py_IsInitialized(void) { + return _PyRuntimeState_GetInitialized(&_PyRuntime); } - /* Helper functions to better handle the legacy C locale * * The legacy C locale assumes ASCII as the default text encoding, which @@ -197,32 +182,30 @@ Py_IsInitialized(void) * C locale and for any of the coercion target locales is "surrogateescape". */ -int -_Py_LegacyLocaleDetected(int warn) -{ +int _Py_LegacyLocaleDetected(int warn) { #ifndef MS_WINDOWS - if (!warn) { - const char *locale_override = getenv("LC_ALL"); - if (locale_override != NULL && *locale_override != '\0') { - /* Don't coerce C locale if the LC_ALL environment variable - is set */ - return 0; - } - } - - /* On non-Windows systems, the C locale is considered a legacy locale */ - /* XXX (ncoghlan): some platforms (notably Mac OS X) don't appear to treat - * the POSIX locale as a simple alias for the C locale, so - * we may also want to check for that explicitly. - */ - const char *ctype_loc = setlocale(LC_CTYPE, NULL); - if (ctype_loc == NULL) { - return 0; - } - return (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0); -#else - /* Windows uses code pages instead of locales, so no locale is legacy */ + if (!warn) { + const char *locale_override = getenv("LC_ALL"); + if (locale_override != NULL && *locale_override != '\0') { + /* Don't coerce C locale if the LC_ALL environment variable + is set */ + return 0; + } + } + + /* On non-Windows systems, the C locale is considered a legacy locale */ + /* XXX (ncoghlan): some platforms (notably Mac OS X) don't appear to treat + * the POSIX locale as a simple alias for the C locale, so + * we may also want to check for that explicitly. + */ + const char *ctype_loc = setlocale(LC_CTYPE, NULL); + if (ctype_loc == NULL) { return 0; + } + return (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0); +#else + /* Windows uses code pages instead of locales, so no locale is legacy */ + return 0; #endif } @@ -233,210 +216,193 @@ static const char *_C_LOCALE_WARNING = "C.utf8, or UTF-8 (if available) as alternative Unicode-compatible " "locales is recommended.\n"; -static void -emit_stderr_warning_for_legacy_locale(_PyRuntimeState *runtime) -{ - const PyPreConfig *preconfig = &runtime->preconfig; - if (preconfig->coerce_c_locale_warn && _Py_LegacyLocaleDetected(1)) { - PySys_FormatStderr("%s", _C_LOCALE_WARNING); - } +static void emit_stderr_warning_for_legacy_locale(_PyRuntimeState *runtime) { + const PyPreConfig *preconfig = &runtime->preconfig; + if (preconfig->coerce_c_locale_warn && _Py_LegacyLocaleDetected(1)) { + PySys_FormatStderr("%s", _C_LOCALE_WARNING); + } } -#endif /* !defined(MS_WINDOWS) */ +#endif /* !defined(MS_WINDOWS) */ typedef struct _CandidateLocale { - const char *locale_name; /* The locale to try as a coercion target */ + const char *locale_name; /* The locale to try as a coercion target */ } _LocaleCoercionTarget; static _LocaleCoercionTarget _TARGET_LOCALES[] = { - {"C.UTF-8"}, - {"C.utf8"}, - {"UTF-8"}, - {NULL} -}; + {"C.UTF-8"}, {"C.utf8"}, {"UTF-8"}, {NULL}}; - -int -_Py_IsLocaleCoercionTarget(const char *ctype_loc) -{ - const _LocaleCoercionTarget *target = NULL; - for (target = _TARGET_LOCALES; target->locale_name; target++) { - if (strcmp(ctype_loc, target->locale_name) == 0) { - return 1; - } +int _Py_IsLocaleCoercionTarget(const char *ctype_loc) { + const _LocaleCoercionTarget *target = NULL; + for (target = _TARGET_LOCALES; target->locale_name; target++) { + if (strcmp(ctype_loc, target->locale_name) == 0) { + return 1; } - return 0; + } + return 0; } - #ifdef PY_COERCE_C_LOCALE static const char C_LOCALE_COERCION_WARNING[] = "Python detected LC_CTYPE=C: LC_CTYPE coerced to %.20s (set another locale " "or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior).\n"; static int -_coerce_default_locale_settings(int warn, const _LocaleCoercionTarget *target) -{ - const char *newloc = target->locale_name; +_coerce_default_locale_settings(int warn, const _LocaleCoercionTarget *target) { + const char *newloc = target->locale_name; - /* Reset locale back to currently configured defaults */ - _Py_SetLocaleFromEnv(LC_ALL); + /* Reset locale back to currently configured defaults */ + _Py_SetLocaleFromEnv(LC_ALL); - /* Set the relevant locale environment variable */ - if (setenv("LC_CTYPE", newloc, 1)) { - fprintf(stderr, - "Error setting LC_CTYPE, skipping C locale coercion\n"); - return 0; - } - if (warn) { - fprintf(stderr, C_LOCALE_COERCION_WARNING, newloc); - } + /* Set the relevant locale environment variable */ + if (setenv("LC_CTYPE", newloc, 1)) { + fprintf(stderr, "Error setting LC_CTYPE, skipping C locale coercion\n"); + return 0; + } + if (warn) { + fprintf(stderr, C_LOCALE_COERCION_WARNING, newloc); + } - /* Reconfigure with the overridden environment variables */ - _Py_SetLocaleFromEnv(LC_ALL); - return 1; + /* Reconfigure with the overridden environment variables */ + _Py_SetLocaleFromEnv(LC_ALL); + return 1; } #endif -int -_Py_CoerceLegacyLocale(int warn) -{ - int coerced = 0; +int _Py_CoerceLegacyLocale(int warn) { + int coerced = 0; #ifdef PY_COERCE_C_LOCALE - char *oldloc = NULL; + char *oldloc = NULL; - oldloc = _PyMem_RawStrdup(setlocale(LC_CTYPE, NULL)); - if (oldloc == NULL) { - return coerced; - } + oldloc = _PyMem_RawStrdup(setlocale(LC_CTYPE, NULL)); + if (oldloc == NULL) { + return coerced; + } - const char *locale_override = getenv("LC_ALL"); - if (locale_override == NULL || *locale_override == '\0') { - /* LC_ALL is also not set (or is set to an empty string) */ - const _LocaleCoercionTarget *target = NULL; - for (target = _TARGET_LOCALES; target->locale_name; target++) { - const char *new_locale = setlocale(LC_CTYPE, - target->locale_name); - if (new_locale != NULL) { -#if !defined(_Py_FORCE_UTF8_LOCALE) && defined(HAVE_LANGINFO_H) && defined(CODESET) - /* Also ensure that nl_langinfo works in this locale */ - char *codeset = nl_langinfo(CODESET); - if (!codeset || *codeset == '\0') { - /* CODESET is not set or empty, so skip coercion */ - new_locale = NULL; - _Py_SetLocaleFromEnv(LC_CTYPE); - continue; - } -#endif - /* Successfully configured locale, so make it the default */ - coerced = _coerce_default_locale_settings(warn, target); - goto done; - } + const char *locale_override = getenv("LC_ALL"); + if (locale_override == NULL || *locale_override == '\0') { + /* LC_ALL is also not set (or is set to an empty string) */ + const _LocaleCoercionTarget *target = NULL; + for (target = _TARGET_LOCALES; target->locale_name; target++) { + const char *new_locale = setlocale(LC_CTYPE, target->locale_name); + if (new_locale != NULL) { +#if !defined(_Py_FORCE_UTF8_LOCALE) && defined(HAVE_LANGINFO_H) && \ + defined(CODESET) + /* Also ensure that nl_langinfo works in this locale */ + char *codeset = nl_langinfo(CODESET); + if (!codeset || *codeset == '\0') { + /* CODESET is not set or empty, so skip coercion */ + new_locale = NULL; + _Py_SetLocaleFromEnv(LC_CTYPE); + continue; } +#endif + /* Successfully configured locale, so make it the default */ + coerced = _coerce_default_locale_settings(warn, target); + goto done; + } } - /* No C locale warning here, as Py_Initialize will emit one later */ + } + /* No C locale warning here, as Py_Initialize will emit one later */ - setlocale(LC_CTYPE, oldloc); + setlocale(LC_CTYPE, oldloc); done: - PyMem_RawFree(oldloc); + PyMem_RawFree(oldloc); #endif - return coerced; + return coerced; } /* _Py_SetLocaleFromEnv() is a wrapper around setlocale(category, "") to * isolate the idiosyncrasies of different libc implementations. It reads the * appropriate environment variable and uses its value to select the locale for * 'category'. */ -char * -_Py_SetLocaleFromEnv(int category) -{ - char *res; +char *_Py_SetLocaleFromEnv(int category) { + char *res; #ifdef __ANDROID__ - const char *locale; - const char **pvar; + const char *locale; + const char **pvar; #ifdef PY_COERCE_C_LOCALE - const char *coerce_c_locale; + const char *coerce_c_locale; #endif - const char *utf8_locale = "C.UTF-8"; - const char *env_var_set[] = { - "LC_ALL", - "LC_CTYPE", - "LANG", - NULL, - }; - - /* Android setlocale(category, "") doesn't check the environment variables - * and incorrectly sets the "C" locale at API 24 and older APIs. We only - * check the environment variables listed in env_var_set. */ - for (pvar=env_var_set; *pvar; pvar++) { - locale = getenv(*pvar); - if (locale != NULL && *locale != '\0') { - if (strcmp(locale, utf8_locale) == 0 || - strcmp(locale, "en_US.UTF-8") == 0) { - return setlocale(category, utf8_locale); - } - return setlocale(category, "C"); - } - } - - /* Android uses UTF-8, so explicitly set the locale to C.UTF-8 if none of - * LC_ALL, LC_CTYPE, or LANG is set to a non-empty string. - * Quote from POSIX section "8.2 Internationalization Variables": - * "4. If the LANG environment variable is not set or is set to the empty - * string, the implementation-defined default locale shall be used." */ + const char *utf8_locale = "C.UTF-8"; + const char *env_var_set[] = { + "LC_ALL", + "LC_CTYPE", + "LANG", + NULL, + }; + + /* Android setlocale(category, "") doesn't check the environment variables + * and incorrectly sets the "C" locale at API 24 and older APIs. We only + * check the environment variables listed in env_var_set. */ + for (pvar = env_var_set; *pvar; pvar++) { + locale = getenv(*pvar); + if (locale != NULL && *locale != '\0') { + if (strcmp(locale, utf8_locale) == 0 || + strcmp(locale, "en_US.UTF-8") == 0) { + return setlocale(category, utf8_locale); + } + return setlocale(category, "C"); + } + } + + /* Android uses UTF-8, so explicitly set the locale to C.UTF-8 if none of + * LC_ALL, LC_CTYPE, or LANG is set to a non-empty string. + * Quote from POSIX section "8.2 Internationalization Variables": + * "4. If the LANG environment variable is not set or is set to the empty + * string, the implementation-defined default locale shall be used." */ #ifdef PY_COERCE_C_LOCALE - coerce_c_locale = getenv("PYTHONCOERCECLOCALE"); - if (coerce_c_locale == NULL || strcmp(coerce_c_locale, "0") != 0) { - /* Some other ported code may check the environment variables (e.g. in - * extension modules), so we make sure that they match the locale - * configuration */ - if (setenv("LC_CTYPE", utf8_locale, 1)) { - fprintf(stderr, "Warning: failed setting the LC_CTYPE " - "environment variable to %s\n", utf8_locale); - } - } + coerce_c_locale = getenv("PYTHONCOERCECLOCALE"); + if (coerce_c_locale == NULL || strcmp(coerce_c_locale, "0") != 0) { + /* Some other ported code may check the environment variables (e.g. in + * extension modules), so we make sure that they match the locale + * configuration */ + if (setenv("LC_CTYPE", utf8_locale, 1)) { + fprintf(stderr, + "Warning: failed setting the LC_CTYPE " + "environment variable to %s\n", + utf8_locale); + } + } #endif - res = setlocale(category, utf8_locale); + res = setlocale(category, utf8_locale); #else /* !defined(__ANDROID__) */ - res = setlocale(category, ""); + res = setlocale(category, ""); #endif - _Py_ResetForceASCII(); - return res; + _Py_ResetForceASCII(); + return res; } +static int interpreter_update_config(PyThreadState *tstate, + int only_update_path_config) { + const PyConfig *config = &tstate->interp->config; -static int -interpreter_update_config(PyThreadState *tstate, int only_update_path_config) -{ - const PyConfig *config = &tstate->interp->config; - - if (!only_update_path_config) { - PyStatus status = _PyConfig_Write(config, tstate->interp->runtime); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - return -1; - } + if (!only_update_path_config) { + PyStatus status = _PyConfig_Write(config, tstate->interp->runtime); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + return -1; } + } - if (_Py_IsMainInterpreter(tstate->interp)) { - PyStatus status = _PyPathConfig_UpdateGlobal(config); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - return -1; - } + if (_Py_IsMainInterpreter(tstate->interp)) { + PyStatus status = _PyPathConfig_UpdateGlobal(config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + return -1; } + } - tstate->interp->long_state.max_str_digits = config->int_max_str_digits; + tstate->interp->long_state.max_str_digits = config->int_max_str_digits; - // Update the sys module for the new configuration - if (_PySys_UpdateConfig(tstate) < 0) { - return -1; - } - return 0; + // Update the sys module for the new configuration + if (_PySys_UpdateConfig(tstate) < 0) { + return -1; + } + return 0; } - /* Global initializations. Can be undone by Py_Finalize(). Don't call this twice without an intervening Py_Finalize() call. @@ -449,1128 +415,1066 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config) */ -static PyStatus -pyinit_core_reconfigure(_PyRuntimeState *runtime, - PyThreadState **tstate_p, - const PyConfig *config) -{ - PyStatus status; - PyThreadState *tstate = _PyThreadState_GET(); - if (!tstate) { - return _PyStatus_ERR("failed to read thread state"); - } - *tstate_p = tstate; - - PyInterpreterState *interp = tstate->interp; - if (interp == NULL) { - return _PyStatus_ERR("can't make main interpreter"); - } - assert(interp->_ready); +static PyStatus pyinit_core_reconfigure(_PyRuntimeState *runtime, + PyThreadState **tstate_p, + const PyConfig *config) { + PyStatus status; + PyThreadState *tstate = _PyThreadState_GET(); + if (!tstate) { + return _PyStatus_ERR("failed to read thread state"); + } + *tstate_p = tstate; + + PyInterpreterState *interp = tstate->interp; + if (interp == NULL) { + return _PyStatus_ERR("can't make main interpreter"); + } + assert(interp->_ready); + + status = _PyConfig_Write(config, runtime); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyConfig_Write(config, runtime); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyConfig_Copy(&interp->config, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + config = _PyInterpreterState_GetConfig(interp); - status = _PyConfig_Copy(&interp->config, config); + if (config->_install_importlib) { + status = _PyPathConfig_UpdateGlobal(config); if (_PyStatus_EXCEPTION(status)) { - return status; - } - config = _PyInterpreterState_GetConfig(interp); - - if (config->_install_importlib) { - status = _PyPathConfig_UpdateGlobal(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + return status; } - return _PyStatus_OK(); + } + return _PyStatus_OK(); } #if defined(PYMALLOC_USE_HUGEPAGES) && defined(MS_WINDOWS) -static PyStatus -get_huge_pages_privilege(void) -{ - HANDLE hToken; - if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) - { - return _PyStatus_ERR("failed to open process token"); - } - TOKEN_PRIVILEGES tp; - if (!LookupPrivilegeValue(NULL, SE_LOCK_MEMORY_NAME, &tp.Privileges[0].Luid)) - { - CloseHandle(hToken); - return _PyStatus_ERR("failed to lookup SeLockMemoryPrivilege for huge pages"); - } - tp.PrivilegeCount = 1; - tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; - // AdjustTokenPrivileges can return with nonzero status (i.e. success) - // but without having all privileges adjusted (ERROR_NOT_ALL_ASSIGNED). - BOOL status = AdjustTokenPrivileges(hToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), NULL, NULL); - DWORD error = GetLastError(); - if (!status || (error != ERROR_SUCCESS)) - { - CloseHandle(hToken); - return _PyStatus_ERR( - "SeLockMemoryPrivilege not held; " - "grant it via Local Security Policy or disable PYTHON_PYMALLOC_HUGEPAGES"); - } - if (!CloseHandle(hToken)) - { - return _PyStatus_ERR("failed to close process token handle"); - } - return _PyStatus_OK(); +static PyStatus get_huge_pages_privilege(void) { + HANDLE hToken; + if (!OpenProcessToken(GetCurrentProcess(), + TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) { + return _PyStatus_ERR("failed to open process token"); + } + TOKEN_PRIVILEGES tp; + if (!LookupPrivilegeValue(NULL, SE_LOCK_MEMORY_NAME, + &tp.Privileges[0].Luid)) { + CloseHandle(hToken); + return _PyStatus_ERR( + "failed to lookup SeLockMemoryPrivilege for huge pages"); + } + tp.PrivilegeCount = 1; + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + // AdjustTokenPrivileges can return with nonzero status (i.e. success) + // but without having all privileges adjusted (ERROR_NOT_ALL_ASSIGNED). + BOOL status = AdjustTokenPrivileges(hToken, FALSE, &tp, + sizeof(TOKEN_PRIVILEGES), NULL, NULL); + DWORD error = GetLastError(); + if (!status || (error != ERROR_SUCCESS)) { + CloseHandle(hToken); + return _PyStatus_ERR("SeLockMemoryPrivilege not held; " + "grant it via Local Security Policy or disable " + "PYTHON_PYMALLOC_HUGEPAGES"); + } + if (!CloseHandle(hToken)) { + return _PyStatus_ERR("failed to close process token handle"); + } + return _PyStatus_OK(); } #endif -static PyStatus -pycore_init_runtime(_PyRuntimeState *runtime, - const PyConfig *config) -{ - if (_PyRuntimeState_GetInitialized(runtime)) { - return _PyStatus_ERR("main interpreter already initialized"); - } +static PyStatus pycore_init_runtime(_PyRuntimeState *runtime, + const PyConfig *config) { + if (_PyRuntimeState_GetInitialized(runtime)) { + return _PyStatus_ERR("main interpreter already initialized"); + } - PyStatus status = _PyConfig_Write(config, runtime); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + PyStatus status = _PyConfig_Write(config, runtime); + if (_PyStatus_EXCEPTION(status)) { + return status; + } #if defined(PYMALLOC_USE_HUGEPAGES) && defined(MS_WINDOWS) - if (runtime->allocators.use_hugepages) { - status = get_huge_pages_privilege(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + if (runtime->allocators.use_hugepages) { + status = get_huge_pages_privilege(); + if (_PyStatus_EXCEPTION(status)) { + return status; } + } #endif - /* Py_Finalize leaves _Py_Finalizing set in order to help daemon - * threads behave a little more gracefully at interpreter shutdown. - * We clobber it here so the new interpreter can start with a clean - * slate. - * - * However, this may still lead to misbehaviour if there are daemon - * threads still hanging around from a previous Py_Initialize/Finalize - * pair :( - */ - _PyRuntimeState_SetFinalizing(runtime, NULL); - - _Py_InitVersion(); - _Py_DumpTraceback_Init(); + /* Py_Finalize leaves _Py_Finalizing set in order to help daemon + * threads behave a little more gracefully at interpreter shutdown. + * We clobber it here so the new interpreter can start with a clean + * slate. + * + * However, this may still lead to misbehaviour if there are daemon + * threads still hanging around from a previous Py_Initialize/Finalize + * pair :( + */ + _PyRuntimeState_SetFinalizing(runtime, NULL); + + _Py_InitVersion(); + _Py_DumpTraceback_Init(); + + status = _Py_HashRandomization_Init(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _Py_HashRandomization_Init(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyImport_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyImport_Init(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyInterpreterState_Enable(runtime); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); +} + +static PyStatus init_interp_settings(PyInterpreterState *interp, + const PyInterpreterConfig *config) { + assert(interp->feature_flags == 0); + + if (config->use_main_obmalloc) { + interp->feature_flags |= Py_RTFLAGS_USE_MAIN_OBMALLOC; + } else if (!config->check_multi_interp_extensions) { + /* The reason: PyModuleDef.m_base.m_copy leaks objects between + interpreters. */ + return _PyStatus_ERR("per-interpreter obmalloc does not support " + "single-phase init extension modules"); + } +#ifdef Py_GIL_DISABLED + if (!_Py_IsMainInterpreter(interp) && + !config->check_multi_interp_extensions) { + return _PyStatus_ERR("The free-threaded build does not support " + "single-phase init extension modules in " + "subinterpreters"); + } +#endif - status = _PyInterpreterState_Enable(runtime); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); + if (config->allow_fork) { + interp->feature_flags |= Py_RTFLAGS_FORK; + } + if (config->allow_exec) { + interp->feature_flags |= Py_RTFLAGS_EXEC; + } + // Note that fork+exec is always allowed. + + if (config->allow_threads) { + interp->feature_flags |= Py_RTFLAGS_THREADS; + } + if (config->allow_daemon_threads) { + interp->feature_flags |= Py_RTFLAGS_DAEMON_THREADS; + } + + if (config->check_multi_interp_extensions) { + interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS; + } + + switch (config->gil) { + case PyInterpreterConfig_DEFAULT_GIL: + break; + case PyInterpreterConfig_SHARED_GIL: + break; + case PyInterpreterConfig_OWN_GIL: + break; + default: + return _PyStatus_ERR("invalid interpreter config 'gil' value"); + } + + return _PyStatus_OK(); +} + +static void init_interp_create_gil(PyThreadState *tstate, int gil) { + /* finalize_interp_delete() comment explains why _PyEval_FiniGIL() is + only called here. */ + // XXX This is broken with a per-interpreter GIL. + _PyEval_FiniGIL(tstate->interp); + + /* Auto-thread-state API */ + _PyGILState_SetTstate(tstate); + + int own_gil = (gil == PyInterpreterConfig_OWN_GIL); + + /* Create the GIL and take it */ + _PyEval_InitGIL(tstate, own_gil); +} + +static int builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, + PyObject *key, PyObject *new_value) { + PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef _Py_TIER2 + if (interp->rare_events.builtin_dict < + _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + _Py_Executors_InvalidateAll(interp, 1); + } +#endif + RARE_EVENT_INTERP_INC(interp, builtin_dict); + return 0; } +static PyStatus pycore_create_interpreter(_PyRuntimeState *runtime, + const PyConfig *src_config, + PyThreadState **tstate_p) { + PyStatus status; + PyInterpreterState *interp; + status = _PyInterpreterState_New(NULL, &interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + assert(interp != NULL); + assert(_Py_IsMainInterpreter(interp)); + _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME); + interp->_ready = 1; + + /* Initialize the module dict watcher early, before any modules are created */ + if (_PyModule_InitModuleDictWatcher(interp) != 0) { + return _PyStatus_ERR("failed to initialize module dict watcher"); + } + + status = _PyConfig_Copy(&interp->config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } -static PyStatus -init_interp_settings(PyInterpreterState *interp, - const PyInterpreterConfig *config) -{ - assert(interp->feature_flags == 0); + /* Auto-thread-state API */ + status = _PyGILState_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + + PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + // The main interpreter always has its own GIL and supports single-phase + // init extensions. + config.gil = PyInterpreterConfig_OWN_GIL; + config.check_multi_interp_extensions = 0; + status = init_interp_settings(interp, &config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - if (config->use_main_obmalloc) { - interp->feature_flags |= Py_RTFLAGS_USE_MAIN_OBMALLOC; - } - else if (!config->check_multi_interp_extensions) { - /* The reason: PyModuleDef.m_base.m_copy leaks objects between - interpreters. */ - return _PyStatus_ERR("per-interpreter obmalloc does not support " - "single-phase init extension modules"); - } -#ifdef Py_GIL_DISABLED - if (!_Py_IsMainInterpreter(interp) && - !config->check_multi_interp_extensions) - { - return _PyStatus_ERR("The free-threaded build does not support " - "single-phase init extension modules in " - "subinterpreters"); - } + // This could be done in init_interpreter() (in pystate.c) if it + // didn't depend on interp->feature_flags being set already. + status = _PyObject_InitState(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + +#ifdef Py_STATS + // initialize pystats. This must be done after the settings are loaded. + status = _PyStats_InterpInit(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } #endif - if (config->allow_fork) { - interp->feature_flags |= Py_RTFLAGS_FORK; - } - if (config->allow_exec) { - interp->feature_flags |= Py_RTFLAGS_EXEC; - } - // Note that fork+exec is always allowed. + // initialize the interp->obmalloc state. This must be done after + // the settings are loaded (so that feature_flags are set) but before + // any calls are made to obmalloc functions. + if (_PyMem_init_obmalloc(interp) < 0) { + return _PyStatus_NO_MEMORY(); + } - if (config->allow_threads) { - interp->feature_flags |= Py_RTFLAGS_THREADS; - } - if (config->allow_daemon_threads) { - interp->feature_flags |= Py_RTFLAGS_DAEMON_THREADS; - } + status = _PyTraceMalloc_Init(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - if (config->check_multi_interp_extensions) { - interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS; - } + PyThreadState *tstate = + _PyThreadState_New(interp, _PyThreadState_WHENCE_INIT); + if (tstate == NULL) { + return _PyStatus_ERR("can't make first thread"); + } + runtime->main_tstate = tstate; + _PyThreadState_Bind(tstate); - switch (config->gil) { - case PyInterpreterConfig_DEFAULT_GIL: break; - case PyInterpreterConfig_SHARED_GIL: break; - case PyInterpreterConfig_OWN_GIL: break; - default: - return _PyStatus_ERR("invalid interpreter config 'gil' value"); - } + init_interp_create_gil(tstate, config.gil); - return _PyStatus_OK(); + *tstate_p = tstate; + return _PyStatus_OK(); } +static PyStatus pycore_init_global_objects(PyInterpreterState *interp) { + PyStatus status; -static void -init_interp_create_gil(PyThreadState *tstate, int gil) -{ - /* finalize_interp_delete() comment explains why _PyEval_FiniGIL() is - only called here. */ - // XXX This is broken with a per-interpreter GIL. - _PyEval_FiniGIL(tstate->interp); + status = _PyUnicode_InitGlobalObjects(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - /* Auto-thread-state API */ - _PyGILState_SetTstate(tstate); + _PyUnicode_InitState(interp); - int own_gil = (gil == PyInterpreterConfig_OWN_GIL); + if (_Py_IsMainInterpreter(interp)) { + _Py_GetConstant_Init(); + } - /* Create the GIL and take it */ - _PyEval_InitGIL(tstate, own_gil); + return _PyStatus_OK(); } -static int -builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); -#ifdef _Py_TIER2 - if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { - _Py_Executors_InvalidateAll(interp, 1); - } -#endif - RARE_EVENT_INTERP_INC(interp, builtin_dict); - return 0; -} +static PyStatus pycore_init_types(PyInterpreterState *interp) { + PyStatus status; -static PyStatus -pycore_create_interpreter(_PyRuntimeState *runtime, - const PyConfig *src_config, - PyThreadState **tstate_p) -{ - PyStatus status; - PyInterpreterState *interp; - status = _PyInterpreterState_New(NULL, &interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - assert(interp != NULL); - assert(_Py_IsMainInterpreter(interp)); - _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME); - interp->_ready = 1; + status = _PyTypes_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - /* Initialize the module dict watcher early, before any modules are created */ - if (_PyModule_InitModuleDictWatcher(interp) != 0) { - return _PyStatus_ERR("failed to initialize module dict watcher"); - } + status = _PyLong_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyConfig_Copy(&interp->config, src_config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyUnicode_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - /* Auto-thread-state API */ - status = _PyGILState_Init(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyFloat_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - // The main interpreter always has its own GIL and supports single-phase - // init extensions. - config.gil = PyInterpreterConfig_OWN_GIL; - config.check_multi_interp_extensions = 0; - status = init_interp_settings(interp, &config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + if (_PyExc_InitTypes(interp) < 0) { + return _PyStatus_ERR("failed to initialize an exception type"); + } - // This could be done in init_interpreter() (in pystate.c) if it - // didn't depend on interp->feature_flags being set already. - status = _PyObject_InitState(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyExc_InitGlobalObjects(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } -#ifdef Py_STATS - // initialize pystats. This must be done after the settings are loaded. - status = _PyStats_InterpInit(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } -#endif + status = _PyExc_InitState(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - // initialize the interp->obmalloc state. This must be done after - // the settings are loaded (so that feature_flags are set) but before - // any calls are made to obmalloc functions. - if (_PyMem_init_obmalloc(interp) < 0) { - return _PyStatus_NO_MEMORY(); - } + status = _PyErr_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyTraceMalloc_Init(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyContext_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - PyThreadState *tstate = _PyThreadState_New(interp, - _PyThreadState_WHENCE_INIT); - if (tstate == NULL) { - return _PyStatus_ERR("can't make first thread"); - } - runtime->main_tstate = tstate; - _PyThreadState_Bind(tstate); + status = _PyXI_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - init_interp_create_gil(tstate, config.gil); + status = _PyInterpolation_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - *tstate_p = tstate; - return _PyStatus_OK(); -} + status = _PyDateTime_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + + return _PyStatus_OK(); +} + +static PyStatus pycore_init_builtins(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; + + PyObject *bimod = _PyBuiltin_Init(interp); + if (bimod == NULL) { + goto error; + } + + PyObject *modules = _PyImport_GetModules(interp); + if (_PyImport_FixupBuiltin(tstate, bimod, "builtins", modules) < 0) { + goto error; + } + + PyObject *builtins_dict = PyModule_GetDict(bimod); + if (builtins_dict == NULL) { + goto error; + } + interp->builtins = Py_NewRef(builtins_dict); + + PyObject *isinstance = + PyDict_GetItemWithError(builtins_dict, &_Py_ID(isinstance)); + if (!isinstance) { + goto error; + } + interp->callable_cache.isinstance = isinstance; + + PyObject *len = PyDict_GetItemWithError(builtins_dict, &_Py_ID(len)); + if (!len) { + goto error; + } + interp->callable_cache.len = len; + + PyObject *all = PyDict_GetItemWithError(builtins_dict, &_Py_ID(all)); + if (!all) { + goto error; + } + + PyObject *any = PyDict_GetItemWithError(builtins_dict, &_Py_ID(any)); + if (!any) { + goto error; + } + + interp->common_consts[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError; + interp->common_consts[CONSTANT_NOTIMPLEMENTEDERROR] = + PyExc_NotImplementedError; + interp->common_consts[CONSTANT_BUILTIN_TUPLE] = (PyObject *)&PyTuple_Type; + interp->common_consts[CONSTANT_BUILTIN_ALL] = all; + interp->common_consts[CONSTANT_BUILTIN_ANY] = any; + interp->common_consts[CONSTANT_BUILTIN_LIST] = (PyObject *)&PyList_Type; + interp->common_consts[CONSTANT_BUILTIN_SET] = (PyObject *)&PySet_Type; + + for (int i = 0; i < NUM_COMMON_CONSTANTS; i++) { + assert(interp->common_consts[i] != NULL); + } + + PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); + if (list_append == NULL) { + goto error; + } + interp->callable_cache.list_append = list_append; + + PyObject *object__getattribute__ = + _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__)); + if (object__getattribute__ == NULL) { + goto error; + } + interp->callable_cache.object__getattribute__ = object__getattribute__; + + if (_PyType_InitSlotDefs(interp) < 0) { + return _PyStatus_ERR("failed to init slotdefs"); + } + + if (_PyBuiltins_AddExceptions(bimod) < 0) { + return _PyStatus_ERR("failed to add exceptions to builtins"); + } + + interp->builtins_copy = PyDict_Copy(interp->builtins); + if (interp->builtins_copy == NULL) { + goto error; + } + Py_DECREF(bimod); + + if (_PyImport_InitDefaultImportFunc(interp) < 0) { + goto error; + } + + assert(!_PyErr_Occurred(tstate)); + return _PyStatus_OK(); +error: + Py_XDECREF(bimod); + return _PyStatus_ERR("can't initialize builtins module"); +} + +static PyStatus pycore_interp_init(PyThreadState *tstate) { + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (_tstate->c_stack_hard_limit == 0) { + _Py_InitializeRecursionLimits(tstate); + } + PyInterpreterState *interp = tstate->interp; + PyStatus status; + PyObject *sysmod = NULL; + + // Create singletons before the first PyType_Ready() call, since + // PyType_Ready() uses singletons like the Unicode empty string (tp_doc) + // and the empty tuple singletons (tp_bases). + status = pycore_init_global_objects(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } -static PyStatus -pycore_init_global_objects(PyInterpreterState *interp) -{ - PyStatus status; + status = _PyCode_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyUnicode_InitGlobalObjects(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyDtoa_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - _PyUnicode_InitState(interp); + // The GC must be initialized before the first GC collection. + status = _PyGC_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - if (_Py_IsMainInterpreter(interp)) { - _Py_GetConstant_Init(); - } + status = pycore_init_types(interp); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } - return _PyStatus_OK(); -} + if (_PyWarnings_InitState(interp) < 0) { + return _PyStatus_ERR("can't initialize warnings"); + } + status = _PyAtExit_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } -static PyStatus -pycore_init_types(PyInterpreterState *interp) -{ - PyStatus status; + status = _PySys_Create(tstate, &sysmod); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } - status = _PyTypes_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = pycore_init_builtins(tstate); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } - status = _PyLong_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyXI_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } - status = _PyUnicode_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + const PyConfig *config = _PyInterpreterState_GetConfig(interp); - status = _PyFloat_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyImport_InitCore(tstate, sysmod, config->_install_importlib); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } - if (_PyExc_InitTypes(interp) < 0) { - return _PyStatus_ERR("failed to initialize an exception type"); - } +done: + /* sys.modules['sys'] contains a strong reference to the module */ + Py_XDECREF(sysmod); + return status; +} - status = _PyExc_InitGlobalObjects(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } +static PyStatus pyinit_config(_PyRuntimeState *runtime, + PyThreadState **tstate_p, + const PyConfig *config) { + PyStatus status = pycore_init_runtime(runtime, config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyExc_InitState(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + PyThreadState *tstate; + status = pycore_create_interpreter(runtime, config, &tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + *tstate_p = tstate; - status = _PyErr_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = pycore_interp_init(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyContext_Init(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + /* Only when we get here is the runtime core fully initialized */ + _PyRuntimeState_SetCoreInitialized(runtime, 1); + return _PyStatus_OK(); +} - status = _PyXI_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } +PyStatus _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, + const _PyArgv *args) { + PyStatus status; - status = _PyInterpolation_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + if (src_config == NULL) { + return _PyStatus_ERR("preinitialization config is NULL"); + } - status = _PyDateTime_InitTypes(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyRuntime_Initialize(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyRuntimeState *runtime = &_PyRuntime; + if (runtime->preinitialized) { + /* If it's already configured: ignored the new configuration */ return _PyStatus_OK(); -} - -static PyStatus -pycore_init_builtins(PyThreadState *tstate) -{ - PyInterpreterState *interp = tstate->interp; - - PyObject *bimod = _PyBuiltin_Init(interp); - if (bimod == NULL) { - goto error; - } - - PyObject *modules = _PyImport_GetModules(interp); - if (_PyImport_FixupBuiltin(tstate, bimod, "builtins", modules) < 0) { - goto error; - } + } - PyObject *builtins_dict = PyModule_GetDict(bimod); - if (builtins_dict == NULL) { - goto error; - } - interp->builtins = Py_NewRef(builtins_dict); + /* Note: preinitializing remains 1 on error, it is only set to 0 + at exit on success. */ + runtime->preinitializing = 1; - PyObject *isinstance = PyDict_GetItemWithError(builtins_dict, &_Py_ID(isinstance)); - if (!isinstance) { - goto error; - } - interp->callable_cache.isinstance = isinstance; + PyPreConfig config; - PyObject *len = PyDict_GetItemWithError(builtins_dict, &_Py_ID(len)); - if (!len) { - goto error; - } - interp->callable_cache.len = len; + status = _PyPreConfig_InitFromPreConfig(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - PyObject *all = PyDict_GetItemWithError(builtins_dict, &_Py_ID(all)); - if (!all) { - goto error; - } + status = _PyPreConfig_Read(&config, args); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - PyObject *any = PyDict_GetItemWithError(builtins_dict, &_Py_ID(any)); - if (!any) { - goto error; - } + status = _PyPreConfig_Write(&config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - interp->common_consts[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError; - interp->common_consts[CONSTANT_NOTIMPLEMENTEDERROR] = PyExc_NotImplementedError; - interp->common_consts[CONSTANT_BUILTIN_TUPLE] = (PyObject*)&PyTuple_Type; - interp->common_consts[CONSTANT_BUILTIN_ALL] = all; - interp->common_consts[CONSTANT_BUILTIN_ANY] = any; - interp->common_consts[CONSTANT_BUILTIN_LIST] = (PyObject*)&PyList_Type; - interp->common_consts[CONSTANT_BUILTIN_SET] = (PyObject*)&PySet_Type; + runtime->preinitializing = 0; + runtime->preinitialized = 1; + return _PyStatus_OK(); +} - for (int i=0; i < NUM_COMMON_CONSTANTS; i++) { - assert(interp->common_consts[i] != NULL); - } +PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *src_config, + Py_ssize_t argc, char **argv) { + _PyArgv args = {.use_bytes_argv = 1, .argc = argc, .bytes_argv = argv}; + return _Py_PreInitializeFromPyArgv(src_config, &args); +} - PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); - if (list_append == NULL) { - goto error; - } - interp->callable_cache.list_append = list_append; +PyStatus Py_PreInitializeFromArgs(const PyPreConfig *src_config, + Py_ssize_t argc, wchar_t **argv) { + _PyArgv args = {.use_bytes_argv = 0, .argc = argc, .wchar_argv = argv}; + return _Py_PreInitializeFromPyArgv(src_config, &args); +} - PyObject *object__getattribute__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__)); - if (object__getattribute__ == NULL) { - goto error; - } - interp->callable_cache.object__getattribute__ = object__getattribute__; +PyStatus Py_PreInitialize(const PyPreConfig *src_config) { + return _Py_PreInitializeFromPyArgv(src_config, NULL); +} - if (_PyType_InitSlotDefs(interp) < 0) { - return _PyStatus_ERR("failed to init slotdefs"); - } +PyStatus _Py_PreInitializeFromConfig(const PyConfig *config, + const _PyArgv *args) { + assert(config != NULL); - if (_PyBuiltins_AddExceptions(bimod) < 0) { - return _PyStatus_ERR("failed to add exceptions to builtins"); - } + PyStatus status = _PyRuntime_Initialize(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyRuntimeState *runtime = &_PyRuntime; - interp->builtins_copy = PyDict_Copy(interp->builtins); - if (interp->builtins_copy == NULL) { - goto error; - } - Py_DECREF(bimod); + if (runtime->preinitialized) { + /* Already initialized: do nothing */ + return _PyStatus_OK(); + } - if (_PyImport_InitDefaultImportFunc(interp) < 0) { - goto error; - } + PyPreConfig preconfig; - assert(!_PyErr_Occurred(tstate)); - return _PyStatus_OK(); + _PyPreConfig_InitFromConfig(&preconfig, config); -error: - Py_XDECREF(bimod); - return _PyStatus_ERR("can't initialize builtins module"); + if (!config->parse_argv) { + return Py_PreInitialize(&preconfig); + } else if (args == NULL) { + _PyArgv config_args = {.use_bytes_argv = 0, + .argc = config->argv.length, + .wchar_argv = config->argv.items}; + return _Py_PreInitializeFromPyArgv(&preconfig, &config_args); + } else { + return _Py_PreInitializeFromPyArgv(&preconfig, args); + } } +/* Begin interpreter initialization + * + * On return, the first thread and interpreter state have been created, + * but the compiler, signal handling, multithreading and + * multiple interpreter support, and codec infrastructure are not yet + * available. + * + * The import system will support builtin and frozen modules only. + * The only supported io is writing to sys.stderr + * + * If any operation invoked by this function fails, a fatal error is + * issued and the function does not return. + * + * Any code invoked from this function should *not* assume it has access + * to the Python C API (unless the API is explicitly listed as being + * safe to call without calling Py_Initialize first) + */ +static PyStatus pyinit_core(_PyRuntimeState *runtime, + const PyConfig *src_config, + PyThreadState **tstate_p) { + PyStatus status; -static PyStatus -pycore_interp_init(PyThreadState *tstate) -{ - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if (_tstate->c_stack_hard_limit == 0) { - _Py_InitializeRecursionLimits(tstate); - } - PyInterpreterState *interp = tstate->interp; - PyStatus status; - PyObject *sysmod = NULL; + status = _Py_PreInitializeFromConfig(src_config, NULL); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - // Create singletons before the first PyType_Ready() call, since - // PyType_Ready() uses singletons like the Unicode empty string (tp_doc) - // and the empty tuple singletons (tp_bases). - status = pycore_init_global_objects(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + PyConfig config; + PyConfig_InitPythonConfig(&config); - status = _PyCode_Init(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyConfig_Copy(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } - status = _PyDtoa_Init(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - // The GC must be initialized before the first GC collection. - status = _PyGC_Init(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = pycore_init_types(interp); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - if (_PyWarnings_InitState(interp) < 0) { - return _PyStatus_ERR("can't initialize warnings"); - } - - status = _PyAtExit_Init(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = _PySys_Create(tstate, &sysmod); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - status = pycore_init_builtins(tstate); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - status = _PyXI_Init(interp); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - const PyConfig *config = _PyInterpreterState_GetConfig(interp); - - status = _PyImport_InitCore(tstate, sysmod, config->_install_importlib); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } + // Read the configuration, but don't compute the path configuration + // (it is computed in the main init). + status = _PyConfig_Read(&config, 0); + if (_PyStatus_EXCEPTION(status)) { + goto done; + } + + if (!runtime->core_initialized) { + status = pyinit_config(runtime, tstate_p, &config); + } else { + status = pyinit_core_reconfigure(runtime, tstate_p, &config); + } + if (_PyStatus_EXCEPTION(status)) { + goto done; + } done: - /* sys.modules['sys'] contains a strong reference to the module */ - Py_XDECREF(sysmod); - return status; -} - - -static PyStatus -pyinit_config(_PyRuntimeState *runtime, - PyThreadState **tstate_p, - const PyConfig *config) -{ - PyStatus status = pycore_init_runtime(runtime, config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - PyThreadState *tstate; - status = pycore_create_interpreter(runtime, config, &tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - *tstate_p = tstate; - - status = pycore_interp_init(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - /* Only when we get here is the runtime core fully initialized */ - _PyRuntimeState_SetCoreInitialized(runtime, 1); - return _PyStatus_OK(); -} - - -PyStatus -_Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) -{ - PyStatus status; - - if (src_config == NULL) { - return _PyStatus_ERR("preinitialization config is NULL"); - } - - status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - _PyRuntimeState *runtime = &_PyRuntime; - - if (runtime->preinitialized) { - /* If it's already configured: ignored the new configuration */ - return _PyStatus_OK(); - } - - /* Note: preinitializing remains 1 on error, it is only set to 0 - at exit on success. */ - runtime->preinitializing = 1; - - PyPreConfig config; - - status = _PyPreConfig_InitFromPreConfig(&config, src_config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = _PyPreConfig_Read(&config, args); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = _PyPreConfig_Write(&config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - runtime->preinitializing = 0; - runtime->preinitialized = 1; - return _PyStatus_OK(); -} - - -PyStatus -Py_PreInitializeFromBytesArgs(const PyPreConfig *src_config, Py_ssize_t argc, char **argv) -{ - _PyArgv args = {.use_bytes_argv = 1, .argc = argc, .bytes_argv = argv}; - return _Py_PreInitializeFromPyArgv(src_config, &args); -} - - -PyStatus -Py_PreInitializeFromArgs(const PyPreConfig *src_config, Py_ssize_t argc, wchar_t **argv) -{ - _PyArgv args = {.use_bytes_argv = 0, .argc = argc, .wchar_argv = argv}; - return _Py_PreInitializeFromPyArgv(src_config, &args); + PyConfig_Clear(&config); + return status; } - -PyStatus -Py_PreInitialize(const PyPreConfig *src_config) -{ - return _Py_PreInitializeFromPyArgv(src_config, NULL); -} - - -PyStatus -_Py_PreInitializeFromConfig(const PyConfig *config, - const _PyArgv *args) -{ - assert(config != NULL); - - PyStatus status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - _PyRuntimeState *runtime = &_PyRuntime; - - if (runtime->preinitialized) { - /* Already initialized: do nothing */ - return _PyStatus_OK(); - } - - PyPreConfig preconfig; - - _PyPreConfig_InitFromConfig(&preconfig, config); - - if (!config->parse_argv) { - return Py_PreInitialize(&preconfig); - } - else if (args == NULL) { - _PyArgv config_args = { - .use_bytes_argv = 0, - .argc = config->argv.length, - .wchar_argv = config->argv.items}; - return _Py_PreInitializeFromPyArgv(&preconfig, &config_args); - } - else { - return _Py_PreInitializeFromPyArgv(&preconfig, args); - } -} - - -/* Begin interpreter initialization - * - * On return, the first thread and interpreter state have been created, - * but the compiler, signal handling, multithreading and - * multiple interpreter support, and codec infrastructure are not yet - * available. - * - * The import system will support builtin and frozen modules only. - * The only supported io is writing to sys.stderr - * - * If any operation invoked by this function fails, a fatal error is - * issued and the function does not return. - * - * Any code invoked from this function should *not* assume it has access - * to the Python C API (unless the API is explicitly listed as being - * safe to call without calling Py_Initialize first) - */ -static PyStatus -pyinit_core(_PyRuntimeState *runtime, - const PyConfig *src_config, - PyThreadState **tstate_p) -{ - PyStatus status; - - status = _Py_PreInitializeFromConfig(src_config, NULL); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - PyConfig config; - PyConfig_InitPythonConfig(&config); - - status = _PyConfig_Copy(&config, src_config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - // Read the configuration, but don't compute the path configuration - // (it is computed in the main init). - status = _PyConfig_Read(&config, 0); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - if (!runtime->core_initialized) { - status = pyinit_config(runtime, tstate_p, &config); - } - else { - status = pyinit_core_reconfigure(runtime, tstate_p, &config); - } - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - -done: - PyConfig_Clear(&config); - return status; -} - - /* Py_Initialize() has already been called: update the main interpreter configuration. Example of bpo-34008: Py_Main() called after Py_Initialize(). */ -static PyStatus -pyinit_main_reconfigure(PyThreadState *tstate) -{ - if (interpreter_update_config(tstate, 0) < 0) { - return _PyStatus_ERR("fail to reconfigure Python"); - } - return _PyStatus_OK(); +static PyStatus pyinit_main_reconfigure(PyThreadState *tstate) { + if (interpreter_update_config(tstate, 0) < 0) { + return _PyStatus_ERR("fail to reconfigure Python"); + } + return _PyStatus_OK(); } - #ifdef Py_DEBUG // Equivalent to the Python code: // // for part in attr.split('.'): // obj = getattr(obj, part) -static PyObject* -presite_resolve_name(PyObject *obj, PyObject *attr) -{ - obj = Py_NewRef(obj); - attr = Py_NewRef(attr); - PyObject *res; - - while (1) { - Py_ssize_t len = PyUnicode_GET_LENGTH(attr); - Py_ssize_t pos = PyUnicode_FindChar(attr, '.', 0, len, 1); - if (pos < 0) { - break; - } +static PyObject *presite_resolve_name(PyObject *obj, PyObject *attr) { + obj = Py_NewRef(obj); + attr = Py_NewRef(attr); + PyObject *res; - PyObject *name = PyUnicode_Substring(attr, 0, pos); - if (name == NULL) { - goto error; - } - res = PyObject_GetAttr(obj, name); - Py_DECREF(name); - if (res == NULL) { - goto error; - } - Py_SETREF(obj, res); + while (1) { + Py_ssize_t len = PyUnicode_GET_LENGTH(attr); + Py_ssize_t pos = PyUnicode_FindChar(attr, '.', 0, len, 1); + if (pos < 0) { + break; + } - PyObject *suffix = PyUnicode_Substring(attr, pos + 1, len); - if (suffix == NULL) { - goto error; - } - Py_SETREF(attr, suffix); + PyObject *name = PyUnicode_Substring(attr, 0, pos); + if (name == NULL) { + goto error; + } + res = PyObject_GetAttr(obj, name); + Py_DECREF(name); + if (res == NULL) { + goto error; + } + Py_SETREF(obj, res); + + PyObject *suffix = PyUnicode_Substring(attr, pos + 1, len); + if (suffix == NULL) { + goto error; } + Py_SETREF(attr, suffix); + } - res = PyObject_GetAttr(obj, attr); - Py_DECREF(obj); - Py_DECREF(attr); - return res; + res = PyObject_GetAttr(obj, attr); + Py_DECREF(obj); + Py_DECREF(attr); + return res; error: - Py_DECREF(obj); - Py_DECREF(attr); - return NULL; + Py_DECREF(obj); + Py_DECREF(attr); + return NULL; } +static void run_presite(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; + const PyConfig *config = _PyInterpreterState_GetConfig(interp); -static void -run_presite(PyThreadState *tstate) -{ - PyInterpreterState *interp = tstate->interp; - const PyConfig *config = _PyInterpreterState_GetConfig(interp); - - if (!config->run_presite) { - return; - } + if (!config->run_presite) { + return; + } - PyObject *presite = PyUnicode_FromWideChar(config->run_presite, -1); - if (presite == NULL) { - fprintf(stderr, "Could not convert pre-site command to Unicode\n"); - _PyErr_Print(tstate); - return; - } - - // Accept "mod_name" and "mod_name:func_name" entry point syntax - Py_ssize_t len = PyUnicode_GET_LENGTH(presite); - Py_ssize_t pos = PyUnicode_FindChar(presite, ':', 0, len, 1); - PyObject *mod_name = NULL; - PyObject *func_name = NULL; - PyObject *module = NULL; - if (pos > 0) { - mod_name = PyUnicode_Substring(presite, 0, pos); - if (mod_name == NULL) { - goto error; - } + PyObject *presite = PyUnicode_FromWideChar(config->run_presite, -1); + if (presite == NULL) { + fprintf(stderr, "Could not convert pre-site command to Unicode\n"); + _PyErr_Print(tstate); + return; + } - func_name = PyUnicode_Substring(presite, pos + 1, len); - if (func_name == NULL) { - goto error; - } - } - else { - mod_name = Py_NewRef(presite); + // Accept "mod_name" and "mod_name:func_name" entry point syntax + Py_ssize_t len = PyUnicode_GET_LENGTH(presite); + Py_ssize_t pos = PyUnicode_FindChar(presite, ':', 0, len, 1); + PyObject *mod_name = NULL; + PyObject *func_name = NULL; + PyObject *module = NULL; + if (pos > 0) { + mod_name = PyUnicode_Substring(presite, 0, pos); + if (mod_name == NULL) { + goto error; } - // mod_name can contain dots (ex: "math.integer") - module = PyImport_Import(mod_name); - if (module == NULL) { - goto error; + func_name = PyUnicode_Substring(presite, pos + 1, len); + if (func_name == NULL) { + goto error; } + } else { + mod_name = Py_NewRef(presite); + } - if (func_name != NULL) { - PyObject *func = presite_resolve_name(module, func_name); - if (func == NULL) { - goto error; - } + // mod_name can contain dots (ex: "math.integer") + module = PyImport_Import(mod_name); + if (module == NULL) { + goto error; + } - PyObject *res = PyObject_CallNoArgs(func); - Py_DECREF(func); - if (res == NULL) { - goto error; - } - Py_DECREF(res); + if (func_name != NULL) { + PyObject *func = presite_resolve_name(module, func_name); + if (func == NULL) { + goto error; } - Py_DECREF(presite); - Py_DECREF(mod_name); - Py_XDECREF(func_name); - Py_DECREF(module); - return; + PyObject *res = PyObject_CallNoArgs(func); + Py_DECREF(func); + if (res == NULL) { + goto error; + } + Py_DECREF(res); + } + + Py_DECREF(presite); + Py_DECREF(mod_name); + Py_XDECREF(func_name); + Py_DECREF(module); + return; error: - fprintf(stderr, "pre-site failed:\n"); - _PyErr_Print(tstate); + fprintf(stderr, "pre-site failed:\n"); + _PyErr_Print(tstate); - Py_DECREF(presite); - Py_XDECREF(mod_name); - Py_XDECREF(func_name); - Py_XDECREF(module); + Py_DECREF(presite); + Py_XDECREF(mod_name); + Py_XDECREF(func_name); + Py_XDECREF(module); } #endif +static PyStatus init_interp_main(PyThreadState *tstate) { + assert(!_PyErr_Occurred(tstate)); -static PyStatus -init_interp_main(PyThreadState *tstate) -{ - assert(!_PyErr_Occurred(tstate)); - - PyStatus status; - int is_main_interp = _Py_IsMainInterpreter(tstate->interp); - PyInterpreterState *interp = tstate->interp; - const PyConfig *config = _PyInterpreterState_GetConfig(interp); - - if (!config->_install_importlib) { - /* Special mode for freeze_importlib: run with no import system - * - * This means anything which needs support from extension modules - * or pure Python code in the standard library won't work. - */ - if (is_main_interp) { - _PyRuntimeState_SetInitialized(interp->runtime, 1); - } - return _PyStatus_OK(); - } + PyStatus status; + int is_main_interp = _Py_IsMainInterpreter(tstate->interp); + PyInterpreterState *interp = tstate->interp; + const PyConfig *config = _PyInterpreterState_GetConfig(interp); - // Initialize the import-related configuration. - status = _PyConfig_InitImportConfig(&interp->config); - if (_PyStatus_EXCEPTION(status)) { - return status; + if (!config->_install_importlib) { + /* Special mode for freeze_importlib: run with no import system + * + * This means anything which needs support from extension modules + * or pure Python code in the standard library won't work. + */ + if (is_main_interp) { + _PyRuntimeState_SetInitialized(interp->runtime, 1); } + return _PyStatus_OK(); + } - if (interpreter_update_config(tstate, 1) < 0) { - return _PyStatus_ERR("failed to update the Python config"); - } + // Initialize the import-related configuration. + status = _PyConfig_InitImportConfig(&interp->config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyImport_InitExternal(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + if (interpreter_update_config(tstate, 1) < 0) { + return _PyStatus_ERR("failed to update the Python config"); + } - if (is_main_interp) { - /* initialize the faulthandler module */ - status = _PyFaulthandler_Init(config->faulthandler); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } + status = _PyImport_InitExternal(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } - status = _PyUnicode_InitEncodings(tstate); + if (is_main_interp) { + /* initialize the faulthandler module */ + status = _PyFaulthandler_Init(config->faulthandler); if (_PyStatus_EXCEPTION(status)) { - return status; + return status; } + } - if (is_main_interp) { - if (_PySignal_Init(config->install_signal_handlers) < 0) { - return _PyStatus_ERR("can't initialize signals"); - } - - if (config->tracemalloc) { - if (_PyTraceMalloc_Start(config->tracemalloc) < 0) { - return _PyStatus_ERR("can't start tracemalloc"); - } - } + status = _PyUnicode_InitEncodings(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } -#ifdef PY_HAVE_PERF_TRAMPOLINE - if (config->perf_profiling) { - _PyPerf_Callbacks *cur_cb; - if (config->perf_profiling == 1) { - cur_cb = &_Py_perfmap_callbacks; - } - else { - cur_cb = &_Py_perfmap_jit_callbacks; - } - if (_PyPerfTrampoline_SetCallbacks(cur_cb) < 0 || - _PyPerfTrampoline_Init(config->perf_profiling) < 0) { - return _PyStatus_ERR("can't initialize the perf trampoline"); - } - } -#endif + if (is_main_interp) { + if (_PySignal_Init(config->install_signal_handlers) < 0) { + return _PyStatus_ERR("can't initialize signals"); } - status = init_sys_streams(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; + if (config->tracemalloc) { + if (_PyTraceMalloc_Start(config->tracemalloc) < 0) { + return _PyStatus_ERR("can't start tracemalloc"); + } } - status = init_set_builtins_open(); - if (_PyStatus_EXCEPTION(status)) { - return status; +#ifdef PY_HAVE_PERF_TRAMPOLINE + if (config->perf_profiling) { + _PyPerf_Callbacks *cur_cb; + if (config->perf_profiling == 1) { + cur_cb = &_Py_perfmap_callbacks; + } else { + cur_cb = &_Py_perfmap_jit_callbacks; + } + if (_PyPerfTrampoline_SetCallbacks(cur_cb) < 0 || + _PyPerfTrampoline_Init(config->perf_profiling) < 0) { + return _PyStatus_ERR("can't initialize the perf trampoline"); + } } +#endif + } + + status = init_sys_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + + status = init_set_builtins_open(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } #ifdef __ANDROID__ - status = init_android_streams(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = init_android_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } #endif #if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG - if (config->use_system_logger) { - status = init_apple_streams(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + if (config->use_system_logger) { + status = init_apple_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; } + } #endif #ifdef Py_DEBUG - run_presite(tstate); + run_presite(tstate); #endif - status = add_main_module(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; + status = add_main_module(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + + if (is_main_interp) { + /* Initialize warnings. */ + PyObject *warnoptions; + if (PySys_GetOptionalAttrString("warnoptions", &warnoptions) < 0) { + return _PyStatus_ERR("can't initialize warnings"); + } + if (warnoptions != NULL && PyList_Check(warnoptions) && + PyList_Size(warnoptions) > 0) { + PyObject *warnings_module = PyImport_ImportModule("warnings"); + if (warnings_module == NULL) { + fprintf(stderr, "'import warnings' failed; traceback:\n"); + _PyErr_Print(tstate); + } + Py_XDECREF(warnings_module); } + Py_XDECREF(warnoptions); + } - if (is_main_interp) { - /* Initialize warnings. */ - PyObject *warnoptions; - if (PySys_GetOptionalAttrString("warnoptions", &warnoptions) < 0) { - return _PyStatus_ERR("can't initialize warnings"); - } - if (warnoptions != NULL && PyList_Check(warnoptions) && - PyList_Size(warnoptions) > 0) - { - PyObject *warnings_module = PyImport_ImportModule("warnings"); - if (warnings_module == NULL) { - fprintf(stderr, "'import warnings' failed; traceback:\n"); - _PyErr_Print(tstate); - } - Py_XDECREF(warnings_module); - } - Py_XDECREF(warnoptions); + if (config->site_import) { + status = init_import_site(); + if (_PyStatus_EXCEPTION(status)) { + return status; } + } - if (config->site_import) { - status = init_import_site(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + // Initialize lazy imports based on configuration. Do this after site + // module is imported to avoid circular imports during startup. + if (config->lazy_imports != -1) { + PyImport_LazyImportsMode lazy_mode; + if (config->lazy_imports == 1) { + lazy_mode = PyImport_LAZY_ALL; + } else { + lazy_mode = PyImport_LAZY_NONE; } - - // Initialize lazy imports based on configuration. Do this after site - // module is imported to avoid circular imports during startup. - if (config->lazy_imports != -1) { - PyImport_LazyImportsMode lazy_mode; - if (config->lazy_imports == 1) { - lazy_mode = PyImport_LAZY_ALL; - } - else { - lazy_mode = PyImport_LAZY_NONE; - } - if (PyImport_SetLazyImportsMode(lazy_mode) < 0) { - return _PyStatus_ERR("failed to set lazy imports mode"); - } + if (PyImport_SetLazyImportsMode(lazy_mode) < 0) { + return _PyStatus_ERR("failed to set lazy imports mode"); } - // If config->lazy_imports == -1, use the default mode, no change needed. + } + // If config->lazy_imports == -1, use the default mode, no change needed. - if (is_main_interp) { + if (is_main_interp) { #ifndef MS_WINDOWS - emit_stderr_warning_for_legacy_locale(interp->runtime); + emit_stderr_warning_for_legacy_locale(interp->runtime); #endif - } + } - // Turn on experimental tier 2 (uops-based) optimizer - // This is also needed when the JIT is enabled + // Turn on experimental tier 2 (uops-based) optimizer + // This is also needed when the JIT is enabled #ifdef _Py_TIER2 - if (is_main_interp) { - int enabled = 1; + if (is_main_interp) { + int enabled = 1; #if _Py_TIER2 & 2 - enabled = 0; + enabled = 0; #endif - char *env = Py_GETENV("PYTHON_JIT"); - if (env && *env != '\0') { - // PYTHON_JIT=0|1 overrides the default - enabled = *env != '0'; - } - if (enabled) { + char *env = Py_GETENV("PYTHON_JIT"); + if (env && *env != '\0') { + // PYTHON_JIT=0|1 overrides the default + enabled = *env != '0'; + } + if (enabled) { #ifdef _Py_JIT - // perf profiler works fine with tier 2 interpreter, so - // only checking for a "real JIT". - if (config->perf_profiling > 0) { - (void)PyErr_WarnEx( - PyExc_RuntimeWarning, - "JIT deactivated as perf profiling support is active", - 0); - } else + // perf profiler works fine with tier 2 interpreter, so + // only checking for a "real JIT". + if (config->perf_profiling > 0) { + (void)PyErr_WarnEx( + PyExc_RuntimeWarning, + "JIT deactivated as perf profiling support is active", 0); + } else #endif - { - interp->jit = true; - } - } + { + interp->jit = true; + } } + } #endif - if (!is_main_interp) { - // The main interpreter is handled in Py_Main(), for now. - if (config->sys_path_0 != NULL) { - PyObject *path0 = PyUnicode_FromWideChar(config->sys_path_0, -1); - if (path0 == NULL) { - return _PyStatus_ERR("can't initialize sys.path[0]"); - } - PyObject *sysdict = interp->sysdict; - if (sysdict == NULL) { - Py_DECREF(path0); - return _PyStatus_ERR("can't initialize sys.path[0]"); - } - PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); - if (sys_path == NULL) { - Py_DECREF(path0); - return _PyStatus_ERR("can't initialize sys.path[0]"); - } - int res = PyList_Insert(sys_path, 0, path0); - Py_DECREF(path0); - if (res) { - return _PyStatus_ERR("can't initialize sys.path[0]"); - } - } - } - - - interp->dict_state.watchers[0] = &builtins_dict_watcher; - if (PyDict_Watch(0, interp->builtins) != 0) { - return _PyStatus_ERR("failed to set builtin dict watcher"); - } - - assert(!_PyErr_Occurred(tstate)); - - if (is_main_interp) { - _PyRuntimeState_SetInitialized(interp->runtime, 1); - } - - return _PyStatus_OK(); + if (!is_main_interp) { + // The main interpreter is handled in Py_Main(), for now. + if (config->sys_path_0 != NULL) { + PyObject *path0 = PyUnicode_FromWideChar(config->sys_path_0, -1); + if (path0 == NULL) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sysdict = interp->sysdict; + if (sysdict == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); + if (sys_path == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + int res = PyList_Insert(sys_path, 0, path0); + Py_DECREF(path0); + if (res) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + } + } + + interp->dict_state.watchers[0] = &builtins_dict_watcher; + if (PyDict_Watch(0, interp->builtins) != 0) { + return _PyStatus_ERR("failed to set builtin dict watcher"); + } + + assert(!_PyErr_Occurred(tstate)); + + if (is_main_interp) { + _PyRuntimeState_SetInitialized(interp->runtime, 1); + } + + return _PyStatus_OK(); } - /* Update interpreter state based on supplied configuration settings * * After calling this function, most of the restrictions on the interpreter @@ -1582,463 +1486,414 @@ init_interp_main(PyThreadState *tstate) * Other errors should be reported as normal Python exceptions with a * non-zero return code. */ -static PyStatus -pyinit_main(PyThreadState *tstate) -{ - PyInterpreterState *interp = tstate->interp; - if (!_PyRuntimeState_GetCoreInitialized(interp->runtime)) { - return _PyStatus_ERR("runtime core not initialized"); - } - - if (_PyRuntimeState_GetInitialized(interp->runtime)) { - return pyinit_main_reconfigure(tstate); - } - - PyStatus status = init_interp_main(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); +static PyStatus pyinit_main(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; + if (!_PyRuntimeState_GetCoreInitialized(interp->runtime)) { + return _PyStatus_ERR("runtime core not initialized"); + } + + if (_PyRuntimeState_GetInitialized(interp->runtime)) { + return pyinit_main_reconfigure(tstate); + } + + PyStatus status = init_interp_main(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } +PyStatus Py_InitializeFromConfig(const PyConfig *config) { + if (config == NULL) { + return _PyStatus_ERR("initialization config is NULL"); + } -PyStatus -Py_InitializeFromConfig(const PyConfig *config) -{ - if (config == NULL) { - return _PyStatus_ERR("initialization config is NULL"); - } + PyStatus status; - PyStatus status; + status = _PyRuntime_Initialize(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyRuntimeState *runtime = &_PyRuntime; - status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - _PyRuntimeState *runtime = &_PyRuntime; + PyThreadState *tstate = NULL; + status = pyinit_core(runtime, config, &tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + config = _PyInterpreterState_GetConfig(tstate->interp); - PyThreadState *tstate = NULL; - status = pyinit_core(runtime, config, &tstate); + if (config->_init_main) { + status = pyinit_main(tstate); if (_PyStatus_EXCEPTION(status)) { - return status; - } - config = _PyInterpreterState_GetConfig(tstate->interp); - - if (config->_init_main) { - status = pyinit_main(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + return status; } + } - return _PyStatus_OK(); + return _PyStatus_OK(); } +void Py_InitializeEx(int install_sigs) { + if (Py_IsInitialized()) { + /* bpo-33932: Calling Py_Initialize() twice does nothing. */ + return; + } -void -Py_InitializeEx(int install_sigs) -{ - if (Py_IsInitialized()) { - /* bpo-33932: Calling Py_Initialize() twice does nothing. */ - return; - } - - PyStatus status; - PyConfig config; - _PyConfig_InitCompatConfig(&config); - - config.install_signal_handlers = install_sigs; + PyStatus status; + PyConfig config; + _PyConfig_InitCompatConfig(&config); - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (_PyStatus_EXCEPTION(status)) { - Py_ExitStatusException(status); - } -} + config.install_signal_handlers = install_sigs; -void -Py_Initialize(void) -{ - Py_InitializeEx(1); + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (_PyStatus_EXCEPTION(status)) { + Py_ExitStatusException(status); + } } +void Py_Initialize(void) { Py_InitializeEx(1); } -PyStatus -_Py_InitializeMain(void) -{ - PyStatus status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - PyThreadState *tstate = _PyThreadState_GET(); - return pyinit_main(tstate); -} - - -static void -finalize_modules_delete_special(PyThreadState *tstate, int verbose) -{ - // List of names to clear in sys - static const char * const sys_deletes[] = { - "path", "argv", "ps1", "ps2", "last_exc", - "last_type", "last_value", "last_traceback", - "__interactivehook__", - // path_hooks and path_importer_cache are cleared - // by _PyImport_FiniExternal(). - // XXX Clear meta_path in _PyImport_FiniCore(). - "meta_path", - NULL - }; - - static const char * const sys_files[] = { - "stdin", "__stdin__", - "stdout", "__stdout__", - "stderr", "__stderr__", - NULL - }; - - PyInterpreterState *interp = tstate->interp; +PyStatus _Py_InitializeMain(void) { + PyStatus status = _PyRuntime_Initialize(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + PyThreadState *tstate = _PyThreadState_GET(); + return pyinit_main(tstate); +} + +static void finalize_modules_delete_special(PyThreadState *tstate, + int verbose) { + // List of names to clear in sys + static const char *const sys_deletes[] = { + "path", "argv", "ps1", "ps2", "last_exc", "last_type", "last_value", + "last_traceback", "__interactivehook__", + // path_hooks and path_importer_cache are cleared + // by _PyImport_FiniExternal(). + // XXX Clear meta_path in _PyImport_FiniCore(). + "meta_path", NULL}; + + static const char *const sys_files[] = { + "stdin", "__stdin__", "stdout", "__stdout__", + "stderr", "__stderr__", NULL}; + + PyInterpreterState *interp = tstate->interp; + if (verbose) { + PySys_WriteStderr("# clear builtins._\n"); + } + if (PyDict_SetItemString(interp->builtins, "_", Py_None) < 0) { + PyErr_FormatUnraisable("Exception ignored while " + "setting builtin variable _"); + } + + const char *const *p; + for (p = sys_deletes; *p != NULL; p++) { + if (_PySys_ClearAttrString(interp, *p, verbose) < 0) { + PyErr_FormatUnraisable("Exception ignored while " + "clearing sys.%s", + *p); + } + } + for (p = sys_files; *p != NULL; p += 2) { + const char *name = p[0]; + const char *orig_name = p[1]; if (verbose) { - PySys_WriteStderr("# clear builtins._\n"); - } - if (PyDict_SetItemString(interp->builtins, "_", Py_None) < 0) { - PyErr_FormatUnraisable("Exception ignored while " - "setting builtin variable _"); - } - - const char * const *p; - for (p = sys_deletes; *p != NULL; p++) { - if (_PySys_ClearAttrString(interp, *p, verbose) < 0) { - PyErr_FormatUnraisable("Exception ignored while " - "clearing sys.%s", *p); - } - } - for (p = sys_files; *p != NULL; p+=2) { - const char *name = p[0]; - const char *orig_name = p[1]; - if (verbose) { - PySys_WriteStderr("# restore sys.%s\n", name); - } - PyObject *value; - if (PyDict_GetItemStringRef(interp->sysdict, orig_name, &value) < 0) { - PyErr_FormatUnraisable("Exception ignored while " - "restoring sys.%s", name); - } + PySys_WriteStderr("# restore sys.%s\n", name); + } + PyObject *value; + if (PyDict_GetItemStringRef(interp->sysdict, orig_name, &value) < 0) { + PyErr_FormatUnraisable("Exception ignored while " + "restoring sys.%s", + name); + } + if (value == NULL) { + value = Py_NewRef(Py_None); + } + if (PyDict_SetItemString(interp->sysdict, name, value) < 0) { + PyErr_FormatUnraisable("Exception ignored while " + "restoring sys.%s", + name); + } + Py_DECREF(value); + } +} + +static PyObject *finalize_remove_modules(PyObject *modules, int verbose) { + PyObject *weaklist = PyList_New(0); + if (weaklist == NULL) { + PyErr_FormatUnraisable("Exception ignored while removing modules"); + } + +#define STORE_MODULE_WEAKREF(name, mod) \ + if (weaklist != NULL) { \ + PyObject *wr = PyWeakref_NewRef(mod, NULL); \ + if (wr) { \ + PyObject *tup = _PyTuple_FromPair(name, wr); \ + if (!tup || PyList_Append(weaklist, tup) < 0) { \ + PyErr_FormatUnraisable("Exception ignored while removing modules"); \ + } \ + Py_XDECREF(tup); \ + Py_DECREF(wr); \ + } else { \ + PyErr_FormatUnraisable("Exception ignored while removing modules"); \ + } \ + } + +#define CLEAR_MODULE(name, mod) \ + if (PyModule_Check(mod)) { \ + if (verbose && PyUnicode_Check(name)) { \ + PySys_FormatStderr("# cleanup[2] removing %U\n", name); \ + } \ + STORE_MODULE_WEAKREF(name, mod); \ + if (PyObject_SetItem(modules, name, Py_None) < 0) { \ + PyErr_FormatUnraisable("Exception ignored while removing modules"); \ + } \ + } + + if (PyDict_CheckExact(modules)) { + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(modules, &pos, &key, &value)) { + CLEAR_MODULE(key, value); + } + } else { + PyObject *iterator = PyObject_GetIter(modules); + if (iterator == NULL) { + PyErr_FormatUnraisable("Exception ignored while removing modules"); + } else { + PyObject *key; + while ((key = PyIter_Next(iterator))) { + PyObject *value = PyObject_GetItem(modules, key); if (value == NULL) { - value = Py_NewRef(Py_None); - } - if (PyDict_SetItemString(interp->sysdict, name, value) < 0) { - PyErr_FormatUnraisable("Exception ignored while " - "restoring sys.%s", name); + PyErr_FormatUnraisable("Exception ignored while removing modules"); + Py_DECREF(key); + continue; } + CLEAR_MODULE(key, value); Py_DECREF(value); - } -} - - -static PyObject* -finalize_remove_modules(PyObject *modules, int verbose) -{ - PyObject *weaklist = PyList_New(0); - if (weaklist == NULL) { + Py_DECREF(key); + } + if (PyErr_Occurred()) { PyErr_FormatUnraisable("Exception ignored while removing modules"); + } + Py_DECREF(iterator); } - -#define STORE_MODULE_WEAKREF(name, mod) \ - if (weaklist != NULL) { \ - PyObject *wr = PyWeakref_NewRef(mod, NULL); \ - if (wr) { \ - PyObject *tup = _PyTuple_FromPair(name, wr); \ - if (!tup || PyList_Append(weaklist, tup) < 0) { \ - PyErr_FormatUnraisable("Exception ignored while removing modules"); \ - } \ - Py_XDECREF(tup); \ - Py_DECREF(wr); \ - } \ - else { \ - PyErr_FormatUnraisable("Exception ignored while removing modules"); \ - } \ - } - -#define CLEAR_MODULE(name, mod) \ - if (PyModule_Check(mod)) { \ - if (verbose && PyUnicode_Check(name)) { \ - PySys_FormatStderr("# cleanup[2] removing %U\n", name); \ - } \ - STORE_MODULE_WEAKREF(name, mod); \ - if (PyObject_SetItem(modules, name, Py_None) < 0) { \ - PyErr_FormatUnraisable("Exception ignored while removing modules"); \ - } \ - } - - if (PyDict_CheckExact(modules)) { - Py_ssize_t pos = 0; - PyObject *key, *value; - while (PyDict_Next(modules, &pos, &key, &value)) { - CLEAR_MODULE(key, value); - } - } - else { - PyObject *iterator = PyObject_GetIter(modules); - if (iterator == NULL) { - PyErr_FormatUnraisable("Exception ignored while removing modules"); - } - else { - PyObject *key; - while ((key = PyIter_Next(iterator))) { - PyObject *value = PyObject_GetItem(modules, key); - if (value == NULL) { - PyErr_FormatUnraisable("Exception ignored while removing modules"); - Py_DECREF(key); - continue; - } - CLEAR_MODULE(key, value); - Py_DECREF(value); - Py_DECREF(key); - } - if (PyErr_Occurred()) { - PyErr_FormatUnraisable("Exception ignored while removing modules"); - } - Py_DECREF(iterator); - } - } + } #undef CLEAR_MODULE #undef STORE_MODULE_WEAKREF - return weaklist; + return weaklist; } - -static void -finalize_clear_modules_dict(PyObject *modules) -{ - if (PyDict_CheckExact(modules)) { - PyDict_Clear(modules); - } - else { - if (PyObject_CallMethodNoArgs(modules, &_Py_ID(clear)) == NULL) { - PyErr_FormatUnraisable("Exception ignored while clearing sys.modules"); - } +static void finalize_clear_modules_dict(PyObject *modules) { + if (PyDict_CheckExact(modules)) { + PyDict_Clear(modules); + } else { + if (PyObject_CallMethodNoArgs(modules, &_Py_ID(clear)) == NULL) { + PyErr_FormatUnraisable("Exception ignored while clearing sys.modules"); } + } } +static void finalize_restore_builtins(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; + PyObject *dict = PyDict_Copy(interp->builtins); + if (dict == NULL) { + PyErr_FormatUnraisable("Exception ignored while restoring builtins"); + } + PyDict_Clear(interp->builtins); + if (PyDict_Update(interp->builtins, interp->builtins_copy)) { + PyErr_FormatUnraisable("Exception ignored while restoring builtins"); + } + Py_XDECREF(dict); +} -static void -finalize_restore_builtins(PyThreadState *tstate) -{ - PyInterpreterState *interp = tstate->interp; - PyObject *dict = PyDict_Copy(interp->builtins); - if (dict == NULL) { - PyErr_FormatUnraisable("Exception ignored while restoring builtins"); +static void finalize_modules_clear_weaklist(PyInterpreterState *interp, + PyObject *weaklist, int verbose) { + // First clear modules imported later + for (Py_ssize_t i = PyList_GET_SIZE(weaklist) - 1; i >= 0; i--) { + PyObject *tup = PyList_GET_ITEM(weaklist, i); + PyObject *name = PyTuple_GET_ITEM(tup, 0); + PyObject *mod = _PyWeakref_GET_REF(PyTuple_GET_ITEM(tup, 1)); + if (mod == NULL) { + continue; } - PyDict_Clear(interp->builtins); - if (PyDict_Update(interp->builtins, interp->builtins_copy)) { - PyErr_FormatUnraisable("Exception ignored while restoring builtins"); + assert(PyModule_Check(mod)); + PyObject *dict = _PyModule_GetDict(mod); // borrowed reference + if (dict == interp->builtins || dict == interp->sysdict) { + Py_DECREF(mod); + continue; } - Py_XDECREF(dict); -} - - -static void -finalize_modules_clear_weaklist(PyInterpreterState *interp, - PyObject *weaklist, int verbose) -{ - // First clear modules imported later - for (Py_ssize_t i = PyList_GET_SIZE(weaklist) - 1; i >= 0; i--) { - PyObject *tup = PyList_GET_ITEM(weaklist, i); - PyObject *name = PyTuple_GET_ITEM(tup, 0); - PyObject *mod = _PyWeakref_GET_REF(PyTuple_GET_ITEM(tup, 1)); - if (mod == NULL) { - continue; - } - assert(PyModule_Check(mod)); - PyObject *dict = _PyModule_GetDict(mod); // borrowed reference - if (dict == interp->builtins || dict == interp->sysdict) { - Py_DECREF(mod); - continue; - } - if (verbose && PyUnicode_Check(name)) { - PySys_FormatStderr("# cleanup[3] wiping %U\n", name); - } - _PyModule_Clear(mod); - Py_DECREF(mod); + if (verbose && PyUnicode_Check(name)) { + PySys_FormatStderr("# cleanup[3] wiping %U\n", name); } + _PyModule_Clear(mod); + Py_DECREF(mod); + } } +static void finalize_clear_sys_builtins_dict(PyInterpreterState *interp, + int verbose) { + // Clear sys dict + if (verbose) { + PySys_FormatStderr("# cleanup[3] wiping sys\n"); + } + _PyModule_ClearDict(interp->sysdict); -static void -finalize_clear_sys_builtins_dict(PyInterpreterState *interp, int verbose) -{ - // Clear sys dict - if (verbose) { - PySys_FormatStderr("# cleanup[3] wiping sys\n"); - } - _PyModule_ClearDict(interp->sysdict); - - // Clear builtins dict - if (verbose) { - PySys_FormatStderr("# cleanup[3] wiping builtins\n"); - } - _PyModule_ClearDict(interp->builtins); + // Clear builtins dict + if (verbose) { + PySys_FormatStderr("# cleanup[3] wiping builtins\n"); + } + _PyModule_ClearDict(interp->builtins); } - /* Clear modules, as good as we can */ // XXX Move most of this to import.c. -static void -finalize_modules(PyThreadState *tstate) -{ - PyInterpreterState *interp = tstate->interp; - - // Invalidate all executors and turn off JIT: - interp->jit = false; - interp->compiling = false; +static void finalize_modules(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; + + // Invalidate all executors and turn off JIT: + interp->jit = false; + interp->compiling = false; #ifdef _Py_TIER2 - _Py_Executors_InvalidateAll(interp, 0); - PyMem_Free(interp->executor_blooms); - PyMem_Free(interp->executor_ptrs); - interp->executor_blooms = NULL; - interp->executor_ptrs = NULL; - interp->executor_count = 0; - interp->executor_capacity = 0; + _Py_Executors_InvalidateAll(interp, 0); + PyMem_Free(interp->executor_blooms); + PyMem_Free(interp->executor_ptrs); + interp->executor_blooms = NULL; + interp->executor_ptrs = NULL; + interp->executor_count = 0; + interp->executor_capacity = 0; #endif - // Stop watching __builtin__ modifications - if (PyDict_Unwatch(0, interp->builtins) < 0) { - // might happen if interp is cleared before watching the __builtin__ - PyErr_Clear(); - } - PyObject *modules = _PyImport_GetModules(interp); - if (modules == NULL) { - // Already done - return; - } - int verbose = _PyInterpreterState_GetConfig(interp)->verbose; - - // Delete some special builtins._ and sys attributes first. These are - // common places where user values hide and people complain when their - // destructors fail. Since the modules containing them are - // deleted *last* of all, they would come too late in the normal - // destruction order. Sigh. + // Stop watching __builtin__ modifications + if (PyDict_Unwatch(0, interp->builtins) < 0) { + // might happen if interp is cleared before watching the __builtin__ + PyErr_Clear(); + } + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL) { + // Already done + return; + } + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; + + // Delete some special builtins._ and sys attributes first. These are + // common places where user values hide and people complain when their + // destructors fail. Since the modules containing them are + // deleted *last* of all, they would come too late in the normal + // destruction order. Sigh. + // + // XXX Perhaps these precautions are obsolete. Who knows? + finalize_modules_delete_special(tstate, verbose); + + // Remove all modules from sys.modules, hoping that garbage collection + // can reclaim most of them: set all sys.modules values to None. + // + // We prepare a list which will receive (name, weakref) tuples of + // modules when they are removed from sys.modules. The name is used + // for diagnosis messages (in verbose mode), while the weakref helps + // detect those modules which have been held alive. + PyObject *weaklist = finalize_remove_modules(modules, verbose); + + // Clear the modules dict + finalize_clear_modules_dict(modules); + + // Restore the original builtins dict, to ensure that any + // user data gets cleared. + finalize_restore_builtins(tstate); + + // Collect garbage + _PyGC_CollectNoFail(tstate); + + // Dump GC stats before it's too late, since it uses the warnings + // machinery. + _PyGC_DumpShutdownStats(interp); + + if (weaklist != NULL) { + // Now, if there are any modules left alive, clear their globals to + // minimize potential leaks. All C extension modules actually end + // up here, since they are kept alive in the interpreter state. // - // XXX Perhaps these precautions are obsolete. Who knows? - finalize_modules_delete_special(tstate, verbose); - - // Remove all modules from sys.modules, hoping that garbage collection - // can reclaim most of them: set all sys.modules values to None. + // The special treatment of "builtins" here is because even + // when it's not referenced as a module, its dictionary is + // referenced by almost every module's __builtins__. Since + // deleting a module clears its dictionary (even if there are + // references left to it), we need to delete the "builtins" + // module last. Likewise, we don't delete sys until the very + // end because it is implicitly referenced (e.g. by print). // - // We prepare a list which will receive (name, weakref) tuples of - // modules when they are removed from sys.modules. The name is used - // for diagnosis messages (in verbose mode), while the weakref helps - // detect those modules which have been held alive. - PyObject *weaklist = finalize_remove_modules(modules, verbose); - - // Clear the modules dict - finalize_clear_modules_dict(modules); - - // Restore the original builtins dict, to ensure that any - // user data gets cleared. - finalize_restore_builtins(tstate); - - // Collect garbage - _PyGC_CollectNoFail(tstate); - - // Dump GC stats before it's too late, since it uses the warnings - // machinery. - _PyGC_DumpShutdownStats(interp); - - if (weaklist != NULL) { - // Now, if there are any modules left alive, clear their globals to - // minimize potential leaks. All C extension modules actually end - // up here, since they are kept alive in the interpreter state. - // - // The special treatment of "builtins" here is because even - // when it's not referenced as a module, its dictionary is - // referenced by almost every module's __builtins__. Since - // deleting a module clears its dictionary (even if there are - // references left to it), we need to delete the "builtins" - // module last. Likewise, we don't delete sys until the very - // end because it is implicitly referenced (e.g. by print). - // - // Since dict is ordered in CPython 3.6+, modules are saved in - // importing order. First clear modules imported later. - finalize_modules_clear_weaklist(interp, weaklist, verbose); - Py_DECREF(weaklist); - } - - // Clear sys and builtins modules dict - finalize_clear_sys_builtins_dict(interp, verbose); - - // Clear module dict copies stored in the interpreter state: - // clear PyInterpreterState.modules_by_index and - // clear PyModuleDef.m_base.m_copy (of extensions not using the multi-phase - // initialization API) - _PyImport_ClearModulesByIndex(interp); - - // Clear the dict of lazily loaded module nname to submodule names - _PyImport_ClearLazyModules(interp); - - // Clear and delete the modules directory. Actual modules will - // still be there only if imported during the execution of some - // destructor. - _PyImport_ClearModules(interp); - - // Collect garbage once more - _PyGC_CollectNoFail(tstate); -} + // Since dict is ordered in CPython 3.6+, modules are saved in + // importing order. First clear modules imported later. + finalize_modules_clear_weaklist(interp, weaklist, verbose); + Py_DECREF(weaklist); + } + // Clear sys and builtins modules dict + finalize_clear_sys_builtins_dict(interp, verbose); -/* Flush stdout and stderr */ + // Clear module dict copies stored in the interpreter state: + // clear PyInterpreterState.modules_by_index and + // clear PyModuleDef.m_base.m_copy (of extensions not using the multi-phase + // initialization API) + _PyImport_ClearModulesByIndex(interp); -static int -file_is_closed(PyObject *fobj) -{ - int r; - PyObject *tmp = PyObject_GetAttrString(fobj, "closed"); - if (tmp == NULL) { - PyErr_Clear(); - return 0; - } - r = PyObject_IsTrue(tmp); - Py_DECREF(tmp); - if (r < 0) - PyErr_Clear(); - return r > 0; + // Clear the dict of lazily loaded module nname to submodule names + _PyImport_ClearLazyModules(interp); + + // Clear and delete the modules directory. Actual modules will + // still be there only if imported during the execution of some + // destructor. + _PyImport_ClearModules(interp); + + // Collect garbage once more + _PyGC_CollectNoFail(tstate); } +/* Flush stdout and stderr */ -static int -flush_std_files(void) -{ - PyObject *file; - int status = 0; +static int file_is_closed(PyObject *fobj) { + int r; + PyObject *tmp = PyObject_GetAttrString(fobj, "closed"); + if (tmp == NULL) { + PyErr_Clear(); + return 0; + } + r = PyObject_IsTrue(tmp); + Py_DECREF(tmp); + if (r < 0) + PyErr_Clear(); + return r > 0; +} - if (PySys_GetOptionalAttr(&_Py_ID(stdout), &file) < 0) { - status = -1; - } - else if (file != NULL && file != Py_None && !file_is_closed(file)) { - if (_PyFile_Flush(file) < 0) { - status = -1; - } - } - if (status < 0) { - PyErr_FormatUnraisable("Exception ignored while flushing sys.stdout"); - } - Py_XDECREF(file); +static int flush_std_files(void) { + PyObject *file; + int status = 0; - if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { - PyErr_Clear(); - status = -1; + if (PySys_GetOptionalAttr(&_Py_ID(stdout), &file) < 0) { + status = -1; + } else if (file != NULL && file != Py_None && !file_is_closed(file)) { + if (_PyFile_Flush(file) < 0) { + status = -1; } - else if (file != NULL && file != Py_None && !file_is_closed(file)) { - if (_PyFile_Flush(file) < 0) { - PyErr_Clear(); - status = -1; - } + } + if (status < 0) { + PyErr_FormatUnraisable("Exception ignored while flushing sys.stdout"); + } + Py_XDECREF(file); + + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { + PyErr_Clear(); + status = -1; + } else if (file != NULL && file != Py_None && !file_is_closed(file)) { + if (_PyFile_Flush(file) < 0) { + PyErr_Clear(); + status = -1; } - Py_XDECREF(file); + } + Py_XDECREF(file); - return status; + return status; } /* Undo the effect of Py_Initialize(). @@ -2055,106 +1910,97 @@ flush_std_files(void) */ - -static void -finalize_interp_types(PyInterpreterState *interp) -{ - _PyTypes_FiniExtTypes(interp); - _PyUnicode_FiniTypes(interp); - _PySys_FiniTypes(interp); - _PyXI_FiniTypes(interp); - _PyExc_Fini(interp); - _PyFloat_FiniType(interp); - _PyLong_FiniTypes(interp); - _PyThread_FiniType(interp); - // XXX fini collections module static types (_PyStaticType_Dealloc()) - // XXX fini IO module static types (_PyStaticType_Dealloc()) - _PyErr_FiniTypes(interp); - _PyTypes_FiniTypes(interp); - - _PyTypes_Fini(interp); +static void finalize_interp_types(PyInterpreterState *interp) { + _PyTypes_FiniExtTypes(interp); + _PyUnicode_FiniTypes(interp); + _PySys_FiniTypes(interp); + _PyXI_FiniTypes(interp); + _PyExc_Fini(interp); + _PyFloat_FiniType(interp); + _PyLong_FiniTypes(interp); + _PyThread_FiniType(interp); + // XXX fini collections module static types (_PyStaticType_Dealloc()) + // XXX fini IO module static types (_PyStaticType_Dealloc()) + _PyErr_FiniTypes(interp); + _PyTypes_FiniTypes(interp); + + _PyTypes_Fini(interp); #ifdef Py_GIL_DISABLED - _PyObject_FinalizeUniqueIdPool(interp); + _PyObject_FinalizeUniqueIdPool(interp); #endif - _PyCode_Fini(interp); + _PyCode_Fini(interp); - // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses - // a dict internally. - _PyUnicode_ClearInterned(interp); + // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses + // a dict internally. + _PyUnicode_ClearInterned(interp); - _PyUnicode_Fini(interp); + _PyUnicode_Fini(interp); #ifndef Py_GIL_DISABLED - // With Py_GIL_DISABLED: - // the freelists for the current thread state have already been cleared. - struct _Py_freelists *freelists = _Py_freelists_GET(); - _PyObject_ClearFreeLists(freelists, 1); + // With Py_GIL_DISABLED: + // the freelists for the current thread state have already been cleared. + struct _Py_freelists *freelists = _Py_freelists_GET(); + _PyObject_ClearFreeLists(freelists, 1); #endif #ifdef Py_DEBUG - _PyStaticObjects_CheckRefcnt(interp); + _PyStaticObjects_CheckRefcnt(interp); #endif } +static void finalize_interp_clear(PyThreadState *tstate) { + int is_main_interp = _Py_IsMainInterpreter(tstate->interp); -static void -finalize_interp_clear(PyThreadState *tstate) -{ - int is_main_interp = _Py_IsMainInterpreter(tstate->interp); + _PyXI_Fini(tstate->interp); + _PyExc_ClearExceptionGroupType(tstate->interp); + _Py_clear_generic_types(tstate->interp); + _PyTypes_FiniCachedDescriptors(tstate->interp); - _PyXI_Fini(tstate->interp); - _PyExc_ClearExceptionGroupType(tstate->interp); - _Py_clear_generic_types(tstate->interp); - _PyTypes_FiniCachedDescriptors(tstate->interp); + /* Clear interpreter state and all thread states */ + _PyInterpreterState_Clear(tstate); - /* Clear interpreter state and all thread states */ - _PyInterpreterState_Clear(tstate); - - /* Clear all loghooks */ - /* Both _PySys_Audit function and users still need PyObject, such as tuple. - Call _PySys_ClearAuditHooks when PyObject available. */ - if (is_main_interp) { - _PySys_ClearAuditHooks(tstate); - } + /* Clear all loghooks */ + /* Both _PySys_Audit function and users still need PyObject, such as tuple. + Call _PySys_ClearAuditHooks when PyObject available. */ + if (is_main_interp) { + _PySys_ClearAuditHooks(tstate); + } - if (is_main_interp) { - _Py_HashRandomization_Fini(); - _PyArg_Fini(); - _Py_ClearFileSystemEncoding(); - _PyPerfTrampoline_Fini(); - } + if (is_main_interp) { + _Py_HashRandomization_Fini(); + _PyArg_Fini(); + _Py_ClearFileSystemEncoding(); + _PyPerfTrampoline_Fini(); + } - finalize_interp_types(tstate->interp); + finalize_interp_types(tstate->interp); - /* Finalize dtoa at last so that finalizers calling repr of float doesn't crash */ - _PyDtoa_Fini(tstate->interp); + /* Finalize dtoa at last so that finalizers calling repr of float doesn't + * crash */ + _PyDtoa_Fini(tstate->interp); - /* Free any delayed free requests immediately */ - _PyMem_FiniDelayed(tstate->interp); + /* Free any delayed free requests immediately */ + _PyMem_FiniDelayed(tstate->interp); - /* finalize_interp_types may allocate Python objects so we may need to - abandon mimalloc segments again */ - _PyThreadState_ClearMimallocHeaps(tstate); + /* finalize_interp_types may allocate Python objects so we may need to + abandon mimalloc segments again */ + _PyThreadState_ClearMimallocHeaps(tstate); } +static void finalize_interp_delete(PyInterpreterState *interp) { + /* Cleanup auto-thread-state */ + _PyGILState_Fini(interp); -static void -finalize_interp_delete(PyInterpreterState *interp) -{ - /* Cleanup auto-thread-state */ - _PyGILState_Fini(interp); + /* We can't call _PyEval_FiniGIL() here because destroying the GIL lock can + fail when it is being awaited by another running daemon thread (see + bpo-9901). Instead pycore_create_interpreter() destroys the previously + created GIL, which ensures that Py_Initialize / Py_FinalizeEx can be + called multiple times. */ - /* We can't call _PyEval_FiniGIL() here because destroying the GIL lock can - fail when it is being awaited by another running daemon thread (see - bpo-9901). Instead pycore_create_interpreter() destroys the previously - created GIL, which ensures that Py_Initialize / Py_FinalizeEx can be - called multiple times. */ - - PyInterpreterState_Delete(interp); + PyInterpreterState_Delete(interp); } - /* Conceptually, there isn't a good reason for Py_Finalize() to be called in any other thread than the one where Py_Initialize() was called. Consequently, it would make sense to fail if the thread @@ -2163,609 +2009,578 @@ finalize_interp_delete(PyInterpreterState *interp) there may be users relying on the unconstrained behavior. Thus, we do our best here to accommodate that possibility. */ -static PyThreadState * -resolve_final_tstate(_PyRuntimeState *runtime) -{ - PyThreadState *main_tstate = runtime->main_tstate; - assert(main_tstate != NULL); - assert(main_tstate->thread_id == runtime->main_thread); - PyInterpreterState *main_interp = _PyInterpreterState_Main(); - assert(main_tstate->interp == main_interp); - - PyThreadState *tstate = _PyThreadState_GET(); - if (_Py_IsMainThread()) { - if (tstate != main_tstate) { - /* This implies that Py_Finalize() was called while - a non-main interpreter was active or while the main - tstate was temporarily swapped out with another. - Neither case should be allowed, but, until we get around - to fixing that (and Py_Exit()), we're letting it go. */ - (void)PyThreadState_Swap(main_tstate); - } - } - else { - /* This is another unfortunate case where Py_Finalize() was - called when it shouldn't have been. We can't simply switch - over to the main thread. At the least, however, we can make - sure the main interpreter is active. */ - if (!_Py_IsMainInterpreter(tstate->interp)) { - /* We don't go to the trouble of updating runtime->main_tstate - since it will be dead soon anyway. */ - main_tstate = - _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI); - if (main_tstate != NULL) { - _PyThreadState_Bind(main_tstate); - (void)PyThreadState_Swap(main_tstate); - } - else { - /* Fall back to the current tstate. It's better than nothing. */ - // XXX No it's not - main_tstate = tstate; - } - } - } - assert(main_tstate != NULL); - - /* We might want to warn if main_tstate->current_frame != NULL. */ - - return main_tstate; +static PyThreadState *resolve_final_tstate(_PyRuntimeState *runtime) { + PyThreadState *main_tstate = runtime->main_tstate; + assert(main_tstate != NULL); + assert(main_tstate->thread_id == runtime->main_thread); + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(main_tstate->interp == main_interp); + + PyThreadState *tstate = _PyThreadState_GET(); + if (_Py_IsMainThread()) { + if (tstate != main_tstate) { + /* This implies that Py_Finalize() was called while + a non-main interpreter was active or while the main + tstate was temporarily swapped out with another. + Neither case should be allowed, but, until we get around + to fixing that (and Py_Exit()), we're letting it go. */ + (void)PyThreadState_Swap(main_tstate); + } + } else { + /* This is another unfortunate case where Py_Finalize() was + called when it shouldn't have been. We can't simply switch + over to the main thread. At the least, however, we can make + sure the main interpreter is active. */ + if (!_Py_IsMainInterpreter(tstate->interp)) { + /* We don't go to the trouble of updating runtime->main_tstate + since it will be dead soon anyway. */ + main_tstate = _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI); + if (main_tstate != NULL) { + _PyThreadState_Bind(main_tstate); + (void)PyThreadState_Swap(main_tstate); + } else { + /* Fall back to the current tstate. It's better than nothing. */ + // XXX No it's not + main_tstate = tstate; + } + } + } + assert(main_tstate != NULL); + + /* We might want to warn if main_tstate->current_frame != NULL. */ + + return main_tstate; } #ifdef Py_GIL_DISABLED -#define ASSERT_WORLD_STOPPED(interp) assert(interp->runtime->stoptheworld.world_stopped) +#define ASSERT_WORLD_STOPPED(interp) \ + assert(interp->runtime->stoptheworld.world_stopped) #else #define ASSERT_WORLD_STOPPED(interp) #endif -static int -interp_has_threads(PyInterpreterState *interp) -{ - /* This needs to check for non-daemon threads only, otherwise we get stuck - * in an infinite loop. */ - assert(interp != NULL); - ASSERT_WORLD_STOPPED(interp); - assert(interp->threads.head != NULL); - if (interp->threads.head->next == NULL) { - // No other threads active, easy way out. - return 0; - } - - // We don't have to worry about locking this because the - // world is stopped. - _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { - if (tstate->_whence == _PyThreadState_WHENCE_THREADING) { - return 1; - } +static int interp_has_threads(PyInterpreterState *interp) { + /* This needs to check for non-daemon threads only, otherwise we get stuck + * in an infinite loop. */ + assert(interp != NULL); + ASSERT_WORLD_STOPPED(interp); + assert(interp->threads.head != NULL); + if (interp->threads.head->next == NULL) { + // No other threads active, easy way out. + return 0; + } + + // We don't have to worry about locking this because the + // world is stopped. + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + if (tstate->_whence == _PyThreadState_WHENCE_THREADING) { + return 1; } + } - return 0; + return 0; } -static int -interp_has_pending_calls(PyInterpreterState *interp) -{ - assert(interp != NULL); - ASSERT_WORLD_STOPPED(interp); - return interp->ceval.pending.npending != 0; +static int interp_has_pending_calls(PyInterpreterState *interp) { + assert(interp != NULL); + ASSERT_WORLD_STOPPED(interp); + return interp->ceval.pending.npending != 0; } -static int -interp_has_atexit_callbacks(PyInterpreterState *interp) -{ - assert(interp != NULL); - assert(interp->atexit.callbacks != NULL); - ASSERT_WORLD_STOPPED(interp); - assert(PyList_CheckExact(interp->atexit.callbacks)); - return PyList_GET_SIZE(interp->atexit.callbacks) != 0; +static int interp_has_atexit_callbacks(PyInterpreterState *interp) { + assert(interp != NULL); + assert(interp->atexit.callbacks != NULL); + ASSERT_WORLD_STOPPED(interp); + assert(PyList_CheckExact(interp->atexit.callbacks)); + return PyList_GET_SIZE(interp->atexit.callbacks) != 0; } -static int -runtime_has_subinterpreters(_PyRuntimeState *runtime) -{ - assert(runtime != NULL); - HEAD_LOCK(runtime); - PyInterpreterState *interp = runtime->interpreters.head; - HEAD_UNLOCK(runtime); - return interp->next != NULL; -} - -static void -make_pre_finalization_calls(PyThreadState *tstate, int subinterpreters) -{ - assert(tstate != NULL); - PyInterpreterState *interp = tstate->interp; - /* Each of these functions can start one another, e.g. a pending call - * could start a thread or vice versa. To ensure that we properly clean - * call everything, we run these in a loop until none of them run anything. */ - for (;;) { - assert(!interp->runtime->stoptheworld.world_stopped); - - // Wrap up existing "threading"-module-created, non-daemon threads. - wait_for_thread_shutdown(tstate); - - // Make any remaining pending calls. - _Py_FinishPendingCalls(tstate); - - /* The interpreter is still entirely intact at this point, and the - * exit funcs may be relying on that. In particular, if some thread - * or exit func is still waiting to do an import, the import machinery - * expects Py_IsInitialized() to return true. So don't say the - * runtime is uninitialized until after the exit funcs have run. - * Note that Threading.py uses an exit func to do a join on all the - * threads created thru it, so this also protects pending imports in - * the threads created via Threading. - */ - - _PyAtExit_Call(tstate->interp); - - if (subinterpreters) { - /* Clean up any lingering subinterpreters. - * Two preconditions need to be met here: - * 1. This has to happen before _PyRuntimeState_SetFinalizing is - * called, or else threads might get prematurely blocked. - * 2. The world must not be stopped, as finalizers can run. - */ - finalize_subinterpreters(); - } +static int runtime_has_subinterpreters(_PyRuntimeState *runtime) { + assert(runtime != NULL); + HEAD_LOCK(runtime); + PyInterpreterState *interp = runtime->interpreters.head; + HEAD_UNLOCK(runtime); + return interp->next != NULL; +} + +static void make_pre_finalization_calls(PyThreadState *tstate, + int subinterpreters) { + assert(tstate != NULL); + PyInterpreterState *interp = tstate->interp; + /* Each of these functions can start one another, e.g. a pending call + * could start a thread or vice versa. To ensure that we properly clean + * call everything, we run these in a loop until none of them run anything. */ + for (;;) { + assert(!interp->runtime->stoptheworld.world_stopped); + + // Wrap up existing "threading"-module-created, non-daemon threads. + wait_for_thread_shutdown(tstate); - /* Wait on finalization guards. - * - * To avoid eating CPU cycles, we use an event to signal when we reach - * zero remaining guards. But, this isn't atomic! This event can be reset - * later if another thread creates a new finalization guard. The actual - * atomic check is made below, when we hold the finalization guard lock. - * Again, this is purely an optimization to avoid overloading the CPU. - */ - if (_Py_atomic_load_ssize_relaxed(&interp->finalization_guards.countdown) > 0) { - for (;;) { - PyTime_t wait_ns = 1000 * 1000; // 1ms - if (PyEvent_WaitTimed(&interp->finalization_guards.done, wait_ns, /*detach=*/1)) { - break; - } - - // For debugging purposes, we emit a fatal error if someone - // CTRL^C'ed the process. - if (PyErr_CheckSignals()) { - PyErr_FormatUnraisable("Exception ignored while waiting on finalization guards"); - Py_FatalError("Interrupted while waiting on finalization guard"); - } - } + // Make any remaining pending calls. + _Py_FinishPendingCalls(tstate); + + /* The interpreter is still entirely intact at this point, and the + * exit funcs may be relying on that. In particular, if some thread + * or exit func is still waiting to do an import, the import machinery + * expects Py_IsInitialized() to return true. So don't say the + * runtime is uninitialized until after the exit funcs have run. + * Note that Threading.py uses an exit func to do a join on all the + * threads created thru it, so this also protects pending imports in + * the threads created via Threading. + */ + + _PyAtExit_Call(tstate->interp); + + if (subinterpreters) { + /* Clean up any lingering subinterpreters. + * Two preconditions need to be met here: + * 1. This has to happen before _PyRuntimeState_SetFinalizing is + * called, or else threads might get prematurely blocked. + * 2. The world must not be stopped, as finalizers can run. + */ + finalize_subinterpreters(); + } + + /* Wait on finalization guards. + * + * To avoid eating CPU cycles, we use an event to signal when we reach + * zero remaining guards. But, this isn't atomic! This event can be reset + * later if another thread creates a new finalization guard. The actual + * atomic check is made below, when we hold the finalization guard lock. + * Again, this is purely an optimization to avoid overloading the CPU. + */ + if (_Py_atomic_load_ssize_relaxed(&interp->finalization_guards.countdown) > + 0) { + for (;;) { + PyTime_t wait_ns = 1000 * 1000; // 1ms + if (PyEvent_WaitTimed(&interp->finalization_guards.done, wait_ns, + /*detach=*/1)) { + break; } - /* Stop the world to prevent other threads from creating threads or - * atexit callbacks. On the default build, this is simply locked by - * the GIL. For pending calls, we acquire the dedicated mutex, because - * Py_AddPendingCall() can be called without an attached thread state. - */ - - PyMutex_Lock(&interp->ceval.pending.mutex); - // XXX Why does _PyThreadState_DeleteList() rely on all interpreters - // being stopped? - _PyEval_StopTheWorldAll(interp->runtime); - _PyRWMutex_Lock(&interp->finalization_guards.lock); - int has_subinterpreters = subinterpreters - ? runtime_has_subinterpreters(interp->runtime) - : 0; - // TODO: The interpreter guard countdown isn't very efficient. We should - // wait on an event or something like that. - int should_continue = (interp_has_threads(interp) - || interp_has_atexit_callbacks(interp) - || interp_has_pending_calls(interp) - || has_subinterpreters - || _Py_atomic_load_ssize_acquire(&interp->finalization_guards.countdown) > 0); - if (!should_continue) { - break; + // For debugging purposes, we emit a fatal error if someone + // CTRL^C'ed the process. + if (PyErr_CheckSignals()) { + PyErr_FormatUnraisable( + "Exception ignored while waiting on finalization guards"); + Py_FatalError("Interrupted while waiting on finalization guard"); } - // Temporarily let other threads execute - _PyThreadState_Detach(tstate); - _PyRWMutex_Unlock(&interp->finalization_guards.lock); - _PyEval_StartTheWorldAll(interp->runtime); - PyMutex_Unlock(&interp->ceval.pending.mutex); - _PyThreadState_Attach(tstate); + } } - assert(PyMutex_IsLocked(&interp->ceval.pending.mutex)); - ASSERT_WORLD_STOPPED(interp); + + /* Stop the world to prevent other threads from creating threads or + * atexit callbacks. On the default build, this is simply locked by + * the GIL. For pending calls, we acquire the dedicated mutex, because + * Py_AddPendingCall() can be called without an attached thread state. + */ + + PyMutex_Lock(&interp->ceval.pending.mutex); + // XXX Why does _PyThreadState_DeleteList() rely on all interpreters + // being stopped? + _PyEval_StopTheWorldAll(interp->runtime); + _PyRWMutex_Lock(&interp->finalization_guards.lock); + int has_subinterpreters = + subinterpreters ? runtime_has_subinterpreters(interp->runtime) : 0; + int should_continue = + (interp_has_threads(interp) || interp_has_atexit_callbacks(interp) || + interp_has_pending_calls(interp) || has_subinterpreters || + _Py_atomic_load_ssize_acquire(&interp->finalization_guards.countdown) > + 0); + if (!should_continue) { + break; + } + // Temporarily let other threads execute + _PyThreadState_Detach(tstate); + _PyRWMutex_Unlock(&interp->finalization_guards.lock); + _PyEval_StartTheWorldAll(interp->runtime); + PyMutex_Unlock(&interp->ceval.pending.mutex); + _PyThreadState_Attach(tstate); + } + assert(PyMutex_IsLocked(&interp->ceval.pending.mutex)); + ASSERT_WORLD_STOPPED(interp); } -static int -_Py_Finalize(_PyRuntimeState *runtime) -{ - int status = 0; +static int _Py_Finalize(_PyRuntimeState *runtime) { + int status = 0; - /* Bail out early if already finalized (or never initialized). */ - if (!_PyRuntimeState_GetInitialized(runtime)) { - return status; - } + /* Bail out early if already finalized (or never initialized). */ + if (!_PyRuntimeState_GetInitialized(runtime)) { + return status; + } - /* Get final thread state pointer. */ - PyThreadState *tstate = resolve_final_tstate(runtime); + /* Get final thread state pointer. */ + PyThreadState *tstate = resolve_final_tstate(runtime); - // Block some operations. - tstate->interp->finalizing = 1; + // Block some operations. + tstate->interp->finalizing = 1; - // This call stops the world and takes the pending calls lock. - make_pre_finalization_calls(tstate, /*subinterpreters=*/1); + // This call stops the world and takes the pending calls lock. + make_pre_finalization_calls(tstate, /*subinterpreters=*/1); - assert(_PyThreadState_GET() == tstate); + assert(_PyThreadState_GET() == tstate); - /* Copy the core config, PyInterpreterState_Delete() free - the core config memory */ + /* Copy the core config, PyInterpreterState_Delete() free + the core config memory */ #ifdef Py_REF_DEBUG - int show_ref_count = tstate->interp->config.show_ref_count; + int show_ref_count = tstate->interp->config.show_ref_count; #endif #ifdef Py_TRACE_REFS - int dump_refs = tstate->interp->config.dump_refs; - wchar_t *dump_refs_file = tstate->interp->config.dump_refs_file; + int dump_refs = tstate->interp->config.dump_refs; + wchar_t *dump_refs_file = tstate->interp->config.dump_refs_file; #endif #ifdef WITH_PYMALLOC - int malloc_stats = tstate->interp->config.malloc_stats; + int malloc_stats = tstate->interp->config.malloc_stats; #endif - /* Ensure that remaining threads are detached */ - ASSERT_WORLD_STOPPED(tstate->interp); - - /* Remaining daemon threads will be trapped in PyThread_hang_thread - when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ - _PyInterpreterState_SetFinalizing(tstate->interp, tstate); - _PyRuntimeState_SetFinalizing(runtime, tstate); - _PyRuntimeState_SetInitialized(runtime, 0); - _PyRuntimeState_SetCoreInitialized(runtime, 0); - - // XXX Call something like _PyImport_Disable() here? - - /* Remove the state of all threads of the interpreter, except for the - current thread. In practice, only daemon threads should still be alive, - except if wait_for_thread_shutdown() has been cancelled by CTRL+C. - We start the world once we are the only thread state left, - before we call destructors. */ - PyThreadState *list = _PyThreadState_RemoveExcept(tstate); - for (PyThreadState *p = list; p != NULL; p = p->next) { - _PyThreadState_SetShuttingDown(p); - } - _PyRWMutex_Unlock(&tstate->interp->finalization_guards.lock); - _PyEval_StartTheWorldAll(runtime); - PyMutex_Unlock(&tstate->interp->ceval.pending.mutex); - - /* Clear frames of other threads to call objects destructors. Destructors - will be called in the current Python thread. Since - _PyRuntimeState_SetFinalizing() has been called, no other Python thread - can take the GIL at this point: if they try, they will hang in - _PyThreadState_HangThread. */ - _PyThreadState_DeleteList(list, /*is_after_fork=*/0); - - /* At this point no Python code should be running at all. - The only thread state left should be the main thread of the main - interpreter (AKA tstate), in which this code is running right now. - There may be other OS threads running but none of them will have - thread states associated with them, nor will be able to create - new thread states. - - Thus tstate is the only possible thread state from here on out. - It may still be used during finalization to run Python code as - needed or provide runtime state (e.g. sys.modules) but that will - happen sparingly. Furthermore, the order of finalization aims - to not need a thread (or interpreter) state as soon as possible. - */ - // XXX Make sure we are preventing the creating of any new thread states - // (or interpreters). - - /* Flush sys.stdout and sys.stderr */ - if (flush_std_files() < 0) { - status = -1; - } - - /* Disable signal handling */ - _PySignal_Fini(); - - /* Collect garbage. This may call finalizers; it's nice to call these - * before all modules are destroyed. - * XXX If a __del__ or weakref callback is triggered here, and tries to - * XXX import a module, bad things can happen, because Python no - * XXX longer believes it's initialized. - * XXX Fatal Python error: Interpreter not initialized (version mismatch?) - * XXX is easy to provoke that way. I've also seen, e.g., - * XXX Exception exceptions.ImportError: 'No module named sha' - * XXX in ignored - * XXX but I'm unclear on exactly how that one happens. In any case, - * XXX I haven't seen a real-life report of either of these. - */ - PyGC_Collect(); - - /* Destroy all modules */ - _PyImport_FiniExternal(tstate->interp); - finalize_modules(tstate); - - /* Print debug stats if any */ - _PyEval_Fini(); - - - /* Flush sys.stdout and sys.stderr (again, in case more was printed) */ - if (flush_std_files() < 0) { - status = -1; - } - - /* Collect final garbage. This disposes of cycles created by - * class definitions, for example. - * XXX This is disabled because it caused too many problems. If - * XXX a __del__ or weakref callback triggers here, Python code has - * XXX a hard time running, because even the sys module has been - * XXX cleared out (sys.stdout is gone, sys.excepthook is gone, etc). - * XXX One symptom is a sequence of information-free messages - * XXX coming from threads (if a __del__ or callback is invoked, - * XXX other threads can execute too, and any exception they encounter - * XXX triggers a comedy of errors as subsystem after subsystem - * XXX fails to find what it *expects* to find in sys to help report - * XXX the exception and consequent unexpected failures). I've also - * XXX seen segfaults then, after adding print statements to the - * XXX Python code getting called. - */ + /* Ensure that remaining threads are detached */ + ASSERT_WORLD_STOPPED(tstate->interp); + + /* Remaining daemon threads will be trapped in PyThread_hang_thread + when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ + _PyInterpreterState_SetFinalizing(tstate->interp, tstate); + _PyRuntimeState_SetFinalizing(runtime, tstate); + _PyRuntimeState_SetInitialized(runtime, 0); + _PyRuntimeState_SetCoreInitialized(runtime, 0); + + // XXX Call something like _PyImport_Disable() here? + + /* Remove the state of all threads of the interpreter, except for the + current thread. In practice, only daemon threads should still be alive, + except if wait_for_thread_shutdown() has been cancelled by CTRL+C. + We start the world once we are the only thread state left, + before we call destructors. */ + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + for (PyThreadState *p = list; p != NULL; p = p->next) { + _PyThreadState_SetShuttingDown(p); + } + _PyRWMutex_Unlock(&tstate->interp->finalization_guards.lock); + _PyEval_StartTheWorldAll(runtime); + PyMutex_Unlock(&tstate->interp->ceval.pending.mutex); + + /* Clear frames of other threads to call objects destructors. Destructors + will be called in the current Python thread. Since + _PyRuntimeState_SetFinalizing() has been called, no other Python thread + can take the GIL at this point: if they try, they will hang in + _PyThreadState_HangThread. */ + _PyThreadState_DeleteList(list, /*is_after_fork=*/0); + + /* At this point no Python code should be running at all. + The only thread state left should be the main thread of the main + interpreter (AKA tstate), in which this code is running right now. + There may be other OS threads running but none of them will have + thread states associated with them, nor will be able to create + new thread states. + + Thus tstate is the only possible thread state from here on out. + It may still be used during finalization to run Python code as + needed or provide runtime state (e.g. sys.modules) but that will + happen sparingly. Furthermore, the order of finalization aims + to not need a thread (or interpreter) state as soon as possible. + */ + // XXX Make sure we are preventing the creating of any new thread states + // (or interpreters). + + /* Flush sys.stdout and sys.stderr */ + if (flush_std_files() < 0) { + status = -1; + } + + /* Disable signal handling */ + _PySignal_Fini(); + + /* Collect garbage. This may call finalizers; it's nice to call these + * before all modules are destroyed. + * XXX If a __del__ or weakref callback is triggered here, and tries to + * XXX import a module, bad things can happen, because Python no + * XXX longer believes it's initialized. + * XXX Fatal Python error: Interpreter not initialized (version mismatch?) + * XXX is easy to provoke that way. I've also seen, e.g., + * XXX Exception exceptions.ImportError: 'No module named sha' + * XXX in ignored + * XXX but I'm unclear on exactly how that one happens. In any case, + * XXX I haven't seen a real-life report of either of these. + */ + PyGC_Collect(); + + /* Destroy all modules */ + _PyImport_FiniExternal(tstate->interp); + finalize_modules(tstate); + + /* Print debug stats if any */ + _PyEval_Fini(); + + /* Flush sys.stdout and sys.stderr (again, in case more was printed) */ + if (flush_std_files() < 0) { + status = -1; + } + + /* Collect final garbage. This disposes of cycles created by + * class definitions, for example. + * XXX This is disabled because it caused too many problems. If + * XXX a __del__ or weakref callback triggers here, Python code has + * XXX a hard time running, because even the sys module has been + * XXX cleared out (sys.stdout is gone, sys.excepthook is gone, etc). + * XXX One symptom is a sequence of information-free messages + * XXX coming from threads (if a __del__ or callback is invoked, + * XXX other threads can execute too, and any exception they encounter + * XXX triggers a comedy of errors as subsystem after subsystem + * XXX fails to find what it *expects* to find in sys to help report + * XXX the exception and consequent unexpected failures). I've also + * XXX seen segfaults then, after adding print statements to the + * XXX Python code getting called. + */ #if 0 _PyGC_CollectIfEnabled(); #endif - /* Disable tracemalloc after all Python objects have been destroyed, - so it is possible to use tracemalloc in objects destructor. */ - _PyTraceMalloc_Fini(); + /* Disable tracemalloc after all Python objects have been destroyed, + so it is possible to use tracemalloc in objects destructor. */ + _PyTraceMalloc_Fini(); - /* Finalize any remaining import state */ - // XXX Move these up to where finalize_modules() is currently. - _PyImport_FiniCore(tstate->interp); - _PyImport_Fini(); + /* Finalize any remaining import state */ + // XXX Move these up to where finalize_modules() is currently. + _PyImport_FiniCore(tstate->interp); + _PyImport_Fini(); - /* unload faulthandler module */ - _PyFaulthandler_Fini(); + /* unload faulthandler module */ + _PyFaulthandler_Fini(); - /* dump hash stats */ - _PyHash_Fini(); + /* dump hash stats */ + _PyHash_Fini(); #ifdef Py_TRACE_REFS - /* Display all objects still alive -- this can invoke arbitrary - * __repr__ overrides, so requires a mostly-intact interpreter. - * Alas, a lot of stuff may still be alive now that will be cleaned - * up later. - */ - - FILE *dump_refs_fp = NULL; - if (dump_refs_file != NULL) { - dump_refs_fp = _Py_wfopen(dump_refs_file, L"w"); - if (dump_refs_fp == NULL) { - fprintf(stderr, "PYTHONDUMPREFSFILE: cannot create file: %ls\n", dump_refs_file); - } - } - - if (dump_refs) { - _Py_PrintReferences(tstate->interp, stderr); - } - - if (dump_refs_fp != NULL) { - _Py_PrintReferences(tstate->interp, dump_refs_fp); - } + /* Display all objects still alive -- this can invoke arbitrary + * __repr__ overrides, so requires a mostly-intact interpreter. + * Alas, a lot of stuff may still be alive now that will be cleaned + * up later. + */ + + FILE *dump_refs_fp = NULL; + if (dump_refs_file != NULL) { + dump_refs_fp = _Py_wfopen(dump_refs_file, L"w"); + if (dump_refs_fp == NULL) { + fprintf(stderr, "PYTHONDUMPREFSFILE: cannot create file: %ls\n", + dump_refs_file); + } + } + + if (dump_refs) { + _Py_PrintReferences(tstate->interp, stderr); + } + + if (dump_refs_fp != NULL) { + _Py_PrintReferences(tstate->interp, dump_refs_fp); + } #endif /* Py_TRACE_REFS */ - /* At this point there's almost no other Python code that will run, - nor interpreter state needed. The only possibility is the - finalizers of the objects stored on tstate (and tstate->interp), - which are triggered via finalize_interp_clear(). + /* At this point there's almost no other Python code that will run, + nor interpreter state needed. The only possibility is the + finalizers of the objects stored on tstate (and tstate->interp), + which are triggered via finalize_interp_clear(). - For now we operate as though none of those finalizers actually - need an operational thread state or interpreter. In reality, - those finalizers may rely on some part of tstate or - tstate->interp, and/or may raise exceptions - or otherwise fail. - */ - // XXX Do this sooner during finalization. - // XXX Ensure finalizer errors are handled properly. + For now we operate as though none of those finalizers actually + need an operational thread state or interpreter. In reality, + those finalizers may rely on some part of tstate or + tstate->interp, and/or may raise exceptions + or otherwise fail. + */ + // XXX Do this sooner during finalization. + // XXX Ensure finalizer errors are handled properly. - finalize_interp_clear(tstate); + finalize_interp_clear(tstate); #ifdef Py_TRACE_REFS - /* Display addresses (& refcnts) of all objects still alive. - * An address can be used to find the repr of the object, printed - * above by _Py_PrintReferences. */ - if (dump_refs) { - _Py_PrintReferenceAddresses(tstate->interp, stderr); - } - if (dump_refs_fp != NULL) { - _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); - fclose(dump_refs_fp); - } + /* Display addresses (& refcnts) of all objects still alive. + * An address can be used to find the repr of the object, printed + * above by _Py_PrintReferences. */ + if (dump_refs) { + _Py_PrintReferenceAddresses(tstate->interp, stderr); + } + if (dump_refs_fp != NULL) { + _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); + fclose(dump_refs_fp); + } #endif /* Py_TRACE_REFS */ #ifdef WITH_PYMALLOC - if (malloc_stats) { - _PyObject_DebugMallocStats(stderr); - } + if (malloc_stats) { + _PyObject_DebugMallocStats(stderr); + } #endif - finalize_interp_delete(tstate->interp); + finalize_interp_delete(tstate->interp); #ifdef Py_REF_DEBUG - if (show_ref_count) { - _PyDebug_PrintTotalRefs(); - } - _Py_FinalizeRefTotal(runtime); + if (show_ref_count) { + _PyDebug_PrintTotalRefs(); + } + _Py_FinalizeRefTotal(runtime); #endif - _Py_FinalizeAllocatedBlocks(runtime); + _Py_FinalizeAllocatedBlocks(runtime); - call_ll_exitfuncs(runtime); + call_ll_exitfuncs(runtime); - _PyRuntime_Finalize(); - return status; -} - -int -Py_FinalizeEx(void) -{ - return _Py_Finalize(&_PyRuntime); + _PyRuntime_Finalize(); + return status; } -void -Py_Finalize(void) -{ - (void)_Py_Finalize(&_PyRuntime); -} +int Py_FinalizeEx(void) { return _Py_Finalize(&_PyRuntime); } +void Py_Finalize(void) { (void)_Py_Finalize(&_PyRuntime); } /* Create and initialize a new interpreter and thread, and return the new thread. This requires that Py_Initialize() has been called first. - Unsuccessful initialization yields a NULL pointer. Note that *no* - exception information is available even in this case -- the - exception information is held in the thread, and there is no - thread. - - Locking: as above. - -*/ - -static PyStatus -new_interpreter(PyThreadState **tstate_p, - const PyInterpreterConfig *config, long whence) -{ - PyStatus status; - - status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - _PyRuntimeState *runtime = &_PyRuntime; - - if (!_PyRuntimeState_GetInitialized(runtime)) { - return _PyStatus_ERR("Py_Initialize must be called first"); - } - - /* Issue #10915, #15751: The GIL API doesn't work with multiple - interpreters: disable PyGILState_Check(). */ - _Py_atomic_store_int_relaxed(&runtime->gilstate.check_enabled, 0); - - // XXX Might new_interpreter() have been called without the GIL held? - PyThreadState *save_tstate = _PyThreadState_GET(); - PyThreadState *tstate = NULL; - PyInterpreterState *interp; - status = _PyInterpreterState_New(save_tstate, &interp); - if (interp == NULL) { - goto error; - } - _PyInterpreterState_SetWhence(interp, whence); - interp->_ready = 1; - - /* Initialize the module dict watcher early, before any modules are created */ - if (_PyModule_InitModuleDictWatcher(interp) != 0) { - goto error; - } - - /* From this point until the init_interp_create_gil() call, - we must not do anything that requires that the GIL be held - (or otherwise exist). That applies whether or not the new - interpreter has its own GIL (e.g. the main interpreter). */ - if (save_tstate != NULL) { - _PyThreadState_Detach(save_tstate); - } + Unsuccessful initialization yields a NULL pointer. Note that *no* + exception information is available even in this case -- the + exception information is held in the thread, and there is no + thread. - /* Copy the current interpreter config into the new interpreter */ - const PyConfig *src_config; - if (save_tstate != NULL) { - src_config = _PyInterpreterState_GetConfig(save_tstate->interp); - } - else - { - /* No current thread state, copy from the main interpreter */ - PyInterpreterState *main_interp = _PyInterpreterState_Main(); - src_config = _PyInterpreterState_GetConfig(main_interp); - } + Locking: as above. - /* This does not require that the GIL be held. */ - status = _PyConfig_Copy(&interp->config, src_config); - if (_PyStatus_EXCEPTION(status)) { - goto error; - } +*/ - /* This does not require that the GIL be held. */ - status = init_interp_settings(interp, config); - if (_PyStatus_EXCEPTION(status)) { - goto error; - } +static PyStatus new_interpreter(PyThreadState **tstate_p, + const PyInterpreterConfig *config, + long whence) { + PyStatus status; - // This could be done in init_interpreter() (in pystate.c) if it - // didn't depend on interp->feature_flags being set already. - status = _PyObject_InitState(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + status = _PyRuntime_Initialize(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyRuntimeState *runtime = &_PyRuntime; + + if (!_PyRuntimeState_GetInitialized(runtime)) { + return _PyStatus_ERR("Py_Initialize must be called first"); + } + + /* Issue #10915, #15751: The GIL API doesn't work with multiple + interpreters: disable PyGILState_Check(). */ + _Py_atomic_store_int_relaxed(&runtime->gilstate.check_enabled, 0); + + // XXX Might new_interpreter() have been called without the GIL held? + PyThreadState *save_tstate = _PyThreadState_GET(); + PyThreadState *tstate = NULL; + PyInterpreterState *interp; + status = _PyInterpreterState_New(save_tstate, &interp); + if (interp == NULL) { + goto error; + } + _PyInterpreterState_SetWhence(interp, whence); + interp->_ready = 1; + + /* Initialize the module dict watcher early, before any modules are created */ + if (_PyModule_InitModuleDictWatcher(interp) != 0) { + goto error; + } + + /* From this point until the init_interp_create_gil() call, + we must not do anything that requires that the GIL be held + (or otherwise exist). That applies whether or not the new + interpreter has its own GIL (e.g. the main interpreter). */ + if (save_tstate != NULL) { + _PyThreadState_Detach(save_tstate); + } + + /* Copy the current interpreter config into the new interpreter */ + const PyConfig *src_config; + if (save_tstate != NULL) { + src_config = _PyInterpreterState_GetConfig(save_tstate->interp); + } else { + /* No current thread state, copy from the main interpreter */ + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + src_config = _PyInterpreterState_GetConfig(main_interp); + } + + /* This does not require that the GIL be held. */ + status = _PyConfig_Copy(&interp->config, src_config); + if (_PyStatus_EXCEPTION(status)) { + goto error; + } + + /* This does not require that the GIL be held. */ + status = init_interp_settings(interp, config); + if (_PyStatus_EXCEPTION(status)) { + goto error; + } + + // This could be done in init_interpreter() (in pystate.c) if it + // didn't depend on interp->feature_flags being set already. + status = _PyObject_InitState(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } #ifdef Py_STATS - // initialize pystats. This must be done after the settings are loaded. - status = _PyStats_InterpInit(interp); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + // initialize pystats. This must be done after the settings are loaded. + status = _PyStats_InterpInit(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } #endif - // initialize the interp->obmalloc state. This must be done after - // the settings are loaded (so that feature_flags are set) but before - // any calls are made to obmalloc functions. - if (_PyMem_init_obmalloc(interp) < 0) { - status = _PyStatus_NO_MEMORY(); - goto error; - } + // initialize the interp->obmalloc state. This must be done after + // the settings are loaded (so that feature_flags are set) but before + // any calls are made to obmalloc functions. + if (_PyMem_init_obmalloc(interp) < 0) { + status = _PyStatus_NO_MEMORY(); + goto error; + } - tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INIT); - if (tstate == NULL) { - status = _PyStatus_NO_MEMORY(); - goto error; - } + tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_INIT); + if (tstate == NULL) { + status = _PyStatus_NO_MEMORY(); + goto error; + } - _PyThreadState_Bind(tstate); - init_interp_create_gil(tstate, config->gil); + _PyThreadState_Bind(tstate); + init_interp_create_gil(tstate, config->gil); - /* No objects have been created yet. */ + /* No objects have been created yet. */ - status = pycore_interp_init(tstate); - if (_PyStatus_EXCEPTION(status)) { - goto error; - } + status = pycore_interp_init(tstate); + if (_PyStatus_EXCEPTION(status)) { + goto error; + } - status = init_interp_main(tstate); - if (_PyStatus_EXCEPTION(status)) { - goto error; - } + status = init_interp_main(tstate); + if (_PyStatus_EXCEPTION(status)) { + goto error; + } - *tstate_p = tstate; - return _PyStatus_OK(); + *tstate_p = tstate; + return _PyStatus_OK(); error: - *tstate_p = NULL; - if (tstate != NULL) { - Py_EndInterpreter(tstate); - } else if (interp != NULL) { - PyInterpreterState_Delete(interp); - } - if (save_tstate != NULL) { - _PyThreadState_Attach(save_tstate); - } - return status; + *tstate_p = NULL; + if (tstate != NULL) { + Py_EndInterpreter(tstate); + } else if (interp != NULL) { + PyInterpreterState_Delete(interp); + } + if (save_tstate != NULL) { + _PyThreadState_Attach(save_tstate); + } + return status; } -PyStatus -Py_NewInterpreterFromConfig(PyThreadState **tstate_p, - const PyInterpreterConfig *config) -{ - long whence = _PyInterpreterState_WHENCE_CAPI; - return new_interpreter(tstate_p, config, whence); +PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, + const PyInterpreterConfig *config) { + long whence = _PyInterpreterState_WHENCE_CAPI; + return new_interpreter(tstate_p, config, whence); } -PyThreadState * -Py_NewInterpreter(void) -{ - PyThreadState *tstate = NULL; - long whence = _PyInterpreterState_WHENCE_LEGACY_CAPI; - const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - PyStatus status = new_interpreter(&tstate, &config, whence); - if (_PyStatus_EXCEPTION(status)) { - Py_ExitStatusException(status); - } - return tstate; +PyThreadState *Py_NewInterpreter(void) { + PyThreadState *tstate = NULL; + long whence = _PyInterpreterState_WHENCE_LEGACY_CAPI; + const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + PyStatus status = new_interpreter(&tstate, &config, whence); + if (_PyStatus_EXCEPTION(status)) { + Py_ExitStatusException(status); + } + return tstate; } /* Delete an interpreter. This requires that the given thread state @@ -2779,603 +2594,560 @@ Py_NewInterpreter(void) */ -void -Py_EndInterpreter(PyThreadState *tstate) -{ - PyInterpreterState *interp = tstate->interp; - - if (tstate != _PyThreadState_GET()) { - Py_FatalError("thread is not current"); - } - if (tstate->current_frame != tstate->base_frame) { - Py_FatalError("thread still has a frame"); - } - interp->finalizing = 1; - - // This call stops the world and takes the pending calls lock. - make_pre_finalization_calls(tstate, /*subinterpreters=*/0); - - ASSERT_WORLD_STOPPED(interp); - /* Remaining daemon threads will automatically exit - when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ - _PyInterpreterState_SetFinalizing(interp, tstate); - - PyThreadState *list = _PyThreadState_RemoveExcept(tstate); - for (PyThreadState *p = list; p != NULL; p = p->next) { - _PyThreadState_SetShuttingDown(p); +void Py_EndInterpreter(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; + + if (tstate != _PyThreadState_GET()) { + Py_FatalError("thread is not current"); + } + if (tstate->current_frame != tstate->base_frame) { + Py_FatalError("thread still has a frame"); + } + interp->finalizing = 1; + + // This call stops the world and takes the pending calls lock. + make_pre_finalization_calls(tstate, /*subinterpreters=*/0); + + ASSERT_WORLD_STOPPED(interp); + /* Remaining daemon threads will automatically exit + when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ + _PyInterpreterState_SetFinalizing(interp, tstate); + + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + for (PyThreadState *p = list; p != NULL; p = p->next) { + _PyThreadState_SetShuttingDown(p); + } + + _PyRWMutex_Unlock(&interp->finalization_guards.lock); + _PyEval_StartTheWorldAll(interp->runtime); + PyMutex_Unlock(&interp->ceval.pending.mutex); + _PyThreadState_DeleteList(list, /*is_after_fork=*/0); + + // XXX Call something like _PyImport_Disable() here? + + _PyImport_FiniExternal(tstate->interp); + finalize_modules(tstate); + _PyImport_FiniCore(tstate->interp); + + finalize_interp_clear(tstate); + finalize_interp_delete(tstate->interp); +} + +int _Py_IsInterpreterFinalizing(PyInterpreterState *interp) { + /* We check the runtime first since, in a daemon thread, + interp might be dangling pointer. */ + PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime); + if (finalizing == NULL) { + finalizing = _PyInterpreterState_GetFinalizing(interp); + } + return finalizing != NULL; +} + +static void finalize_subinterpreters(void) { + PyThreadState *final_tstate = _PyThreadState_GET(); + PyInterpreterState *main_interp = _PyInterpreterState_Main(); + assert(final_tstate->interp == main_interp); + _PyRuntimeState *runtime = main_interp->runtime; + assert(!runtime->stoptheworld.world_stopped); + assert(_PyRuntimeState_GetFinalizing(runtime) == NULL); + struct pyinterpreters *interpreters = &runtime->interpreters; + + /* Get the first interpreter in the list. */ + HEAD_LOCK(runtime); + PyInterpreterState *interp = interpreters->head; + if (interp == main_interp) { + interp = interp->next; + } + HEAD_UNLOCK(runtime); + + /* Bail out if there are no subinterpreters left. */ + if (interp == NULL) { + return; + } + + /* Warn the user if they forgot to clean up subinterpreters. */ + (void)PyErr_WarnEx(PyExc_RuntimeWarning, + "remaining subinterpreters; " + "close them with Interpreter.close()", + 0); + + /* Swap out the current tstate, which we know must belong + to the main interpreter. */ + _PyThreadState_Detach(final_tstate); + + /* Clean up all remaining subinterpreters. */ + while (interp != NULL) { + /* Make a tstate for finalization. */ + PyThreadState *tstate = + _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); + if (tstate == NULL) { + // XXX Some graceful way to always get a thread state? + Py_FatalError("thread state allocation failed"); } - _PyRWMutex_Unlock(&interp->finalization_guards.lock); - _PyEval_StartTheWorldAll(interp->runtime); - PyMutex_Unlock(&interp->ceval.pending.mutex); - _PyThreadState_DeleteList(list, /*is_after_fork=*/0); - - // XXX Call something like _PyImport_Disable() here? - - _PyImport_FiniExternal(tstate->interp); - finalize_modules(tstate); - _PyImport_FiniCore(tstate->interp); - - finalize_interp_clear(tstate); - finalize_interp_delete(tstate->interp); -} - -int -_Py_IsInterpreterFinalizing(PyInterpreterState *interp) -{ - /* We check the runtime first since, in a daemon thread, - interp might be dangling pointer. */ - PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime); - if (finalizing == NULL) { - finalizing = _PyInterpreterState_GetFinalizing(interp); - } - return finalizing != NULL; -} + /* Enter the subinterpreter. */ + _PyThreadState_Attach(tstate); -static void -finalize_subinterpreters(void) -{ - PyThreadState *final_tstate = _PyThreadState_GET(); - PyInterpreterState *main_interp = _PyInterpreterState_Main(); - assert(final_tstate->interp == main_interp); - _PyRuntimeState *runtime = main_interp->runtime; - assert(!runtime->stoptheworld.world_stopped); - assert(_PyRuntimeState_GetFinalizing(runtime) == NULL); - struct pyinterpreters *interpreters = &runtime->interpreters; + /* Destroy the subinterpreter. */ + Py_EndInterpreter(tstate); + assert(_PyThreadState_GET() == NULL); - /* Get the first interpreter in the list. */ + /* Advance to the next interpreter. */ HEAD_LOCK(runtime); - PyInterpreterState *interp = interpreters->head; + interp = interpreters->head; if (interp == main_interp) { - interp = interp->next; + interp = interp->next; } HEAD_UNLOCK(runtime); + } - /* Bail out if there are no subinterpreters left. */ - if (interp == NULL) { - return; - } - - /* Warn the user if they forgot to clean up subinterpreters. */ - (void)PyErr_WarnEx( - PyExc_RuntimeWarning, - "remaining subinterpreters; " - "close them with Interpreter.close()", - 0); - - /* Swap out the current tstate, which we know must belong - to the main interpreter. */ - _PyThreadState_Detach(final_tstate); - - /* Clean up all remaining subinterpreters. */ - while (interp != NULL) { - /* Make a tstate for finalization. */ - PyThreadState *tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); - if (tstate == NULL) { - // XXX Some graceful way to always get a thread state? - Py_FatalError("thread state allocation failed"); - } - - /* Enter the subinterpreter. */ - _PyThreadState_Attach(tstate); - - /* Destroy the subinterpreter. */ - Py_EndInterpreter(tstate); - assert(_PyThreadState_GET() == NULL); - - /* Advance to the next interpreter. */ - HEAD_LOCK(runtime); - interp = interpreters->head; - if (interp == main_interp) { - interp = interp->next; - } - HEAD_UNLOCK(runtime); - } - - /* Switch back to the main interpreter. */ - _PyThreadState_Attach(final_tstate); + /* Switch back to the main interpreter. */ + _PyThreadState_Attach(final_tstate); } - /* Add the __main__ module */ -static PyStatus -add_main_module(PyInterpreterState *interp) -{ - PyObject *m, *d; - m = PyImport_AddModuleObject(&_Py_ID(__main__)); - if (m == NULL) - return _PyStatus_ERR("can't create __main__ module"); +static PyStatus add_main_module(PyInterpreterState *interp) { + PyObject *m, *d; + m = PyImport_AddModuleObject(&_Py_ID(__main__)); + if (m == NULL) + return _PyStatus_ERR("can't create __main__ module"); - d = PyModule_GetDict(m); + d = PyModule_GetDict(m); - int has_builtins = PyDict_ContainsString(d, "__builtins__"); - if (has_builtins < 0) { - return _PyStatus_ERR("Failed to test __main__.__builtins__"); - } - if (!has_builtins) { - PyObject *bimod = PyImport_ImportModule("builtins"); - if (bimod == NULL) { - return _PyStatus_ERR("Failed to retrieve builtins module"); - } - if (PyDict_SetItemString(d, "__builtins__", bimod) < 0) { - return _PyStatus_ERR("Failed to initialize __main__.__builtins__"); - } - Py_DECREF(bimod); + int has_builtins = PyDict_ContainsString(d, "__builtins__"); + if (has_builtins < 0) { + return _PyStatus_ERR("Failed to test __main__.__builtins__"); + } + if (!has_builtins) { + PyObject *bimod = PyImport_ImportModule("builtins"); + if (bimod == NULL) { + return _PyStatus_ERR("Failed to retrieve builtins module"); } - - /* Main is a little special - BuiltinImporter is the most appropriate - * initial setting for its __loader__ attribute. A more suitable value - * will be set if __main__ gets further initialized later in the startup - * process. - */ - PyObject *loader; - if (PyDict_GetItemStringRef(d, "__loader__", &loader) < 0) { - return _PyStatus_ERR("Failed to test __main__.__loader__"); - } - int has_loader = !(loader == NULL || loader == Py_None); - Py_XDECREF(loader); - if (!has_loader) { - PyObject *loader = _PyImport_GetImportlibLoader(interp, - "BuiltinImporter"); - if (loader == NULL) { - return _PyStatus_ERR("Failed to retrieve BuiltinImporter"); - } - if (PyDict_SetItemString(d, "__loader__", loader) < 0) { - return _PyStatus_ERR("Failed to initialize __main__.__loader__"); - } - Py_DECREF(loader); + if (PyDict_SetItemString(d, "__builtins__", bimod) < 0) { + return _PyStatus_ERR("Failed to initialize __main__.__builtins__"); } - return _PyStatus_OK(); + Py_DECREF(bimod); + } + + /* Main is a little special - BuiltinImporter is the most appropriate + * initial setting for its __loader__ attribute. A more suitable value + * will be set if __main__ gets further initialized later in the startup + * process. + */ + PyObject *loader; + if (PyDict_GetItemStringRef(d, "__loader__", &loader) < 0) { + return _PyStatus_ERR("Failed to test __main__.__loader__"); + } + int has_loader = !(loader == NULL || loader == Py_None); + Py_XDECREF(loader); + if (!has_loader) { + PyObject *loader = _PyImport_GetImportlibLoader(interp, "BuiltinImporter"); + if (loader == NULL) { + return _PyStatus_ERR("Failed to retrieve BuiltinImporter"); + } + if (PyDict_SetItemString(d, "__loader__", loader) < 0) { + return _PyStatus_ERR("Failed to initialize __main__.__loader__"); + } + Py_DECREF(loader); + } + return _PyStatus_OK(); } /* Import the site module (not into __main__ though) */ -static PyStatus -init_import_site(void) -{ - PyObject *m; - m = PyImport_ImportModule("site"); - if (m == NULL) { - return _PyStatus_ERR("Failed to import the site module"); - } - Py_DECREF(m); - return _PyStatus_OK(); +static PyStatus init_import_site(void) { + PyObject *m; + m = PyImport_ImportModule("site"); + if (m == NULL) { + return _PyStatus_ERR("Failed to import the site module"); + } + Py_DECREF(m); + return _PyStatus_OK(); } /* returns Py_None if the fd is not valid */ -static PyObject* -create_stdio(const PyConfig *config, PyObject* io, - int fd, int write_mode, const char* name, - const wchar_t* encoding, const wchar_t* errors) -{ - PyObject *buf = NULL, *stream = NULL, *text = NULL, *raw = NULL, *res; - const char* mode; - const char* newline; - PyObject *line_buffering, *write_through; - int buffering, isatty; - const int buffered_stdio = config->buffered_stdio; - - if (!_Py_IsValidFD(fd)) { - Py_RETURN_NONE; - } - - /* stdin is always opened in buffered mode, first because it shouldn't - make a difference in common use cases, second because TextIOWrapper - depends on the presence of a read1() method which only exists on - buffered streams. - */ - if (!buffered_stdio && write_mode) - buffering = 0; - else - buffering = -1; - if (write_mode) - mode = "wb"; - else - mode = "rb"; - buf = _PyObject_CallMethod(io, &_Py_ID(open), "isiOOOO", - fd, mode, buffering, - Py_None, Py_None, /* encoding, errors */ - Py_None, Py_False); /* newline, closefd */ - if (buf == NULL) - goto error; - - if (buffering) { - raw = PyObject_GetAttr(buf, &_Py_ID(raw)); - if (raw == NULL) - goto error; - } - else { - raw = Py_NewRef(buf); - } +static PyObject *create_stdio(const PyConfig *config, PyObject *io, int fd, + int write_mode, const char *name, + const wchar_t *encoding, const wchar_t *errors) { + PyObject *buf = NULL, *stream = NULL, *text = NULL, *raw = NULL, *res; + const char *mode; + const char *newline; + PyObject *line_buffering, *write_through; + int buffering, isatty; + const int buffered_stdio = config->buffered_stdio; + + if (!_Py_IsValidFD(fd)) { + Py_RETURN_NONE; + } + + /* stdin is always opened in buffered mode, first because it shouldn't + make a difference in common use cases, second because TextIOWrapper + depends on the presence of a read1() method which only exists on + buffered streams. + */ + if (!buffered_stdio && write_mode) + buffering = 0; + else + buffering = -1; + if (write_mode) + mode = "wb"; + else + mode = "rb"; + buf = _PyObject_CallMethod(io, &_Py_ID(open), "isiOOOO", fd, mode, buffering, + Py_None, Py_None, /* encoding, errors */ + Py_None, Py_False); /* newline, closefd */ + if (buf == NULL) + goto error; + + if (buffering) { + raw = PyObject_GetAttr(buf, &_Py_ID(raw)); + if (raw == NULL) + goto error; + } else { + raw = Py_NewRef(buf); + } #ifdef HAVE_WINDOWS_CONSOLE_IO - /* Windows console IO is always UTF-8 encoded */ - PyTypeObject *winconsoleio_type = (PyTypeObject *)PyImport_ImportModuleAttr( - &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); - if (winconsoleio_type == NULL) { - goto error; - } - int is_subclass = PyObject_TypeCheck(raw, winconsoleio_type); - Py_DECREF(winconsoleio_type); - if (is_subclass) { - encoding = L"utf-8"; - } + /* Windows console IO is always UTF-8 encoded */ + PyTypeObject *winconsoleio_type = (PyTypeObject *)PyImport_ImportModuleAttr( + &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + if (winconsoleio_type == NULL) { + goto error; + } + int is_subclass = PyObject_TypeCheck(raw, winconsoleio_type); + Py_DECREF(winconsoleio_type); + if (is_subclass) { + encoding = L"utf-8"; + } #endif - text = PyUnicode_FromString(name); - if (text == NULL || PyObject_SetAttr(raw, &_Py_ID(name), text) < 0) - goto error; - res = PyObject_CallMethodNoArgs(raw, &_Py_ID(isatty)); - if (res == NULL) - goto error; - isatty = PyObject_IsTrue(res); - Py_DECREF(res); - if (isatty == -1) - goto error; - if (!buffered_stdio) - write_through = Py_True; - else - write_through = Py_False; - if (buffered_stdio && (isatty || fd == fileno(stderr))) - line_buffering = Py_True; - else - line_buffering = Py_False; - - Py_CLEAR(raw); - Py_CLEAR(text); + text = PyUnicode_FromString(name); + if (text == NULL || PyObject_SetAttr(raw, &_Py_ID(name), text) < 0) + goto error; + res = PyObject_CallMethodNoArgs(raw, &_Py_ID(isatty)); + if (res == NULL) + goto error; + isatty = PyObject_IsTrue(res); + Py_DECREF(res); + if (isatty == -1) + goto error; + if (!buffered_stdio) + write_through = Py_True; + else + write_through = Py_False; + if (buffered_stdio && (isatty || fd == fileno(stderr))) + line_buffering = Py_True; + else + line_buffering = Py_False; + + Py_CLEAR(raw); + Py_CLEAR(text); #ifdef MS_WINDOWS - /* sys.stdin: enable universal newline mode, translate "\r\n" and "\r" - newlines to "\n". - sys.stdout and sys.stderr: translate "\n" to "\r\n". */ - newline = NULL; + /* sys.stdin: enable universal newline mode, translate "\r\n" and "\r" + newlines to "\n". + sys.stdout and sys.stderr: translate "\n" to "\r\n". */ + newline = NULL; #else - /* sys.stdin: split lines at "\n". - sys.stdout and sys.stderr: don't translate newlines (use "\n"). */ - newline = "\n"; + /* sys.stdin: split lines at "\n". + sys.stdout and sys.stderr: don't translate newlines (use "\n"). */ + newline = "\n"; #endif - PyObject *encoding_str = PyUnicode_FromWideChar(encoding, -1); - if (encoding_str == NULL) { - Py_CLEAR(buf); - goto error; - } - - PyObject *errors_str = PyUnicode_FromWideChar(errors, -1); - if (errors_str == NULL) { - Py_CLEAR(buf); - Py_CLEAR(encoding_str); - goto error; - } + PyObject *encoding_str = PyUnicode_FromWideChar(encoding, -1); + if (encoding_str == NULL) { + Py_CLEAR(buf); + goto error; + } - stream = _PyObject_CallMethod(io, &_Py_ID(TextIOWrapper), "OOOsOO", - buf, encoding_str, errors_str, - newline, line_buffering, write_through); + PyObject *errors_str = PyUnicode_FromWideChar(errors, -1); + if (errors_str == NULL) { Py_CLEAR(buf); Py_CLEAR(encoding_str); - Py_CLEAR(errors_str); - if (stream == NULL) - goto error; - - if (write_mode) - mode = "w"; - else - mode = "r"; - text = PyUnicode_FromString(mode); - if (!text || PyObject_SetAttr(stream, &_Py_ID(mode), text) < 0) - goto error; - Py_CLEAR(text); - return stream; + goto error; + } + + stream = _PyObject_CallMethod(io, &_Py_ID(TextIOWrapper), "OOOsOO", buf, + encoding_str, errors_str, newline, + line_buffering, write_through); + Py_CLEAR(buf); + Py_CLEAR(encoding_str); + Py_CLEAR(errors_str); + if (stream == NULL) + goto error; + + if (write_mode) + mode = "w"; + else + mode = "r"; + text = PyUnicode_FromString(mode); + if (!text || PyObject_SetAttr(stream, &_Py_ID(mode), text) < 0) + goto error; + Py_CLEAR(text); + return stream; error: - Py_XDECREF(buf); - Py_XDECREF(stream); - Py_XDECREF(text); - Py_XDECREF(raw); - - if (PyErr_ExceptionMatches(PyExc_OSError) && !_Py_IsValidFD(fd)) { - /* Issue #24891: the file descriptor was closed after the first - _Py_IsValidFD() check was called. Ignore the OSError and set the - stream to None. */ - PyErr_Clear(); - Py_RETURN_NONE; - } - return NULL; + Py_XDECREF(buf); + Py_XDECREF(stream); + Py_XDECREF(text); + Py_XDECREF(raw); + + if (PyErr_ExceptionMatches(PyExc_OSError) && !_Py_IsValidFD(fd)) { + /* Issue #24891: the file descriptor was closed after the first + _Py_IsValidFD() check was called. Ignore the OSError and set the + stream to None. */ + PyErr_Clear(); + Py_RETURN_NONE; + } + return NULL; } /* Set builtins.open to io.open */ -static PyStatus -init_set_builtins_open(void) -{ - PyObject *wrapper; - PyObject *bimod = NULL; - PyStatus res = _PyStatus_OK(); +static PyStatus init_set_builtins_open(void) { + PyObject *wrapper; + PyObject *bimod = NULL; + PyStatus res = _PyStatus_OK(); - if (!(bimod = PyImport_ImportModule("builtins"))) { - goto error; - } + if (!(bimod = PyImport_ImportModule("builtins"))) { + goto error; + } - if (!(wrapper = PyImport_ImportModuleAttrString("_io", "open"))) { - goto error; - } + if (!(wrapper = PyImport_ImportModuleAttrString("_io", "open"))) { + goto error; + } - /* Set builtins.open */ - if (PyObject_SetAttrString(bimod, "open", wrapper) == -1) { - Py_DECREF(wrapper); - goto error; - } + /* Set builtins.open */ + if (PyObject_SetAttrString(bimod, "open", wrapper) == -1) { Py_DECREF(wrapper); - goto done; + goto error; + } + Py_DECREF(wrapper); + goto done; error: - res = _PyStatus_ERR("can't initialize io.open"); + res = _PyStatus_ERR("can't initialize io.open"); done: - Py_XDECREF(bimod); - return res; + Py_XDECREF(bimod); + return res; } - /* Create sys.stdin, sys.stdout and sys.stderr */ -static PyStatus -init_sys_streams(PyThreadState *tstate) -{ - PyObject *iomod = NULL; - PyObject *std = NULL; - int fd; - PyObject * encoding_attr; - PyStatus res = _PyStatus_OK(); - const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); - - /* Check that stdin is not a directory - Using shell redirection, you can redirect stdin to a directory, - crashing the Python interpreter. Catch this common mistake here - and output a useful error message. Note that under MS Windows, - the shell already prevents that. */ +static PyStatus init_sys_streams(PyThreadState *tstate) { + PyObject *iomod = NULL; + PyObject *std = NULL; + int fd; + PyObject *encoding_attr; + PyStatus res = _PyStatus_OK(); + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + + /* Check that stdin is not a directory + Using shell redirection, you can redirect stdin to a directory, + crashing the Python interpreter. Catch this common mistake here + and output a useful error message. Note that under MS Windows, + the shell already prevents that. */ #ifndef MS_WINDOWS - struct _Py_stat_struct sb; - if (_Py_fstat_noraise(fileno(stdin), &sb) == 0 && - S_ISDIR(sb.st_mode)) { - return _PyStatus_ERR(" is a directory, cannot continue"); - } + struct _Py_stat_struct sb; + if (_Py_fstat_noraise(fileno(stdin), &sb) == 0 && S_ISDIR(sb.st_mode)) { + return _PyStatus_ERR(" is a directory, cannot continue"); + } #endif - if (!(iomod = PyImport_ImportModule("_io"))) { - goto error; - } - - /* Set sys.stdin */ - fd = fileno(stdin); - /* Under some conditions stdin, stdout and stderr may not be connected - * and fileno() may point to an invalid file descriptor. For example - * GUI apps don't have valid standard streams by default. - */ - std = create_stdio(config, iomod, fd, 0, "", - config->stdio_encoding, - config->stdio_errors); - if (std == NULL) - goto error; - PySys_SetObject("__stdin__", std); - _PySys_SetAttr(&_Py_ID(stdin), std); - Py_DECREF(std); - - /* Set sys.stdout */ - fd = fileno(stdout); - std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, - config->stdio_errors); - if (std == NULL) - goto error; - PySys_SetObject("__stdout__", std); - _PySys_SetAttr(&_Py_ID(stdout), std); - Py_DECREF(std); + if (!(iomod = PyImport_ImportModule("_io"))) { + goto error; + } + + /* Set sys.stdin */ + fd = fileno(stdin); + /* Under some conditions stdin, stdout and stderr may not be connected + * and fileno() may point to an invalid file descriptor. For example + * GUI apps don't have valid standard streams by default. + */ + std = create_stdio(config, iomod, fd, 0, "", config->stdio_encoding, + config->stdio_errors); + if (std == NULL) + goto error; + PySys_SetObject("__stdin__", std); + _PySys_SetAttr(&_Py_ID(stdin), std); + Py_DECREF(std); + + /* Set sys.stdout */ + fd = fileno(stdout); + std = create_stdio(config, iomod, fd, 1, "", config->stdio_encoding, + config->stdio_errors); + if (std == NULL) + goto error; + PySys_SetObject("__stdout__", std); + _PySys_SetAttr(&_Py_ID(stdout), std); + Py_DECREF(std); #if 1 /* Disable this if you have trouble debugging bootstrap stuff */ - /* Set sys.stderr, replaces the preliminary stderr */ - fd = fileno(stderr); - std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, - L"backslashreplace"); - if (std == NULL) - goto error; - - /* Same as hack above, pre-import stderr's codec to avoid recursion - when import.c tries to write to stderr in verbose mode. */ - encoding_attr = PyObject_GetAttrString(std, "encoding"); - if (encoding_attr != NULL) { - const char *std_encoding = PyUnicode_AsUTF8(encoding_attr); - if (std_encoding != NULL) { - PyObject *codec_info = _PyCodec_Lookup(std_encoding); - Py_XDECREF(codec_info); - } - Py_DECREF(encoding_attr); - } - _PyErr_Clear(tstate); /* Not a fatal error if codec isn't available */ - - if (PySys_SetObject("__stderr__", std) < 0) { - Py_DECREF(std); - goto error; - } - if (_PySys_SetAttr(&_Py_ID(stderr), std) < 0) { - Py_DECREF(std); - goto error; - } + /* Set sys.stderr, replaces the preliminary stderr */ + fd = fileno(stderr); + std = create_stdio(config, iomod, fd, 1, "", config->stdio_encoding, + L"backslashreplace"); + if (std == NULL) + goto error; + + /* Same as hack above, pre-import stderr's codec to avoid recursion + when import.c tries to write to stderr in verbose mode. */ + encoding_attr = PyObject_GetAttrString(std, "encoding"); + if (encoding_attr != NULL) { + const char *std_encoding = PyUnicode_AsUTF8(encoding_attr); + if (std_encoding != NULL) { + PyObject *codec_info = _PyCodec_Lookup(std_encoding); + Py_XDECREF(codec_info); + } + Py_DECREF(encoding_attr); + } + _PyErr_Clear(tstate); /* Not a fatal error if codec isn't available */ + + if (PySys_SetObject("__stderr__", std) < 0) { + Py_DECREF(std); + goto error; + } + if (_PySys_SetAttr(&_Py_ID(stderr), std) < 0) { Py_DECREF(std); + goto error; + } + Py_DECREF(std); #endif - goto done; + goto done; error: - res = _PyStatus_ERR("can't initialize sys standard streams"); + res = _PyStatus_ERR("can't initialize sys standard streams"); done: - Py_XDECREF(iomod); - return res; + Py_XDECREF(iomod); + return res; } - #ifdef __ANDROID__ #include -static PyObject * -android_log_write_impl(PyObject *self, PyObject *args) -{ - int prio = 0; - const char *tag = NULL; - const char *text = NULL; - if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) { - return NULL; - } +static PyObject *android_log_write_impl(PyObject *self, PyObject *args) { + int prio = 0; + const char *tag = NULL; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) { + return NULL; + } - // Despite its name, this function is part of the public API - // (https://developer.android.com/ndk/reference/group/logging). - __android_log_write(prio, tag, text); - Py_RETURN_NONE; + // Despite its name, this function is part of the public API + // (https://developer.android.com/ndk/reference/group/logging). + __android_log_write(prio, tag, text); + Py_RETURN_NONE; } - static PyMethodDef android_log_write_method = { - "android_log_write", android_log_write_impl, METH_VARARGS -}; - - -static PyStatus -init_android_streams(PyThreadState *tstate) -{ - PyStatus status = _PyStatus_OK(); - PyObject *_android_support = NULL; - PyObject *android_log_write = NULL; - PyObject *result = NULL; - - _android_support = PyImport_ImportModule("_android_support"); - if (_android_support == NULL) { - goto error; - } - - android_log_write = PyCFunction_New(&android_log_write_method, NULL); - if (android_log_write == NULL) { - goto error; - } - - // These log priorities match those used by Java's System.out and System.err. - result = PyObject_CallMethod( - _android_support, "init_streams", "Oii", - android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN); - if (result == NULL) { - goto error; - } - - goto done; + "android_log_write", android_log_write_impl, METH_VARARGS}; + +static PyStatus init_android_streams(PyThreadState *tstate) { + PyStatus status = _PyStatus_OK(); + PyObject *_android_support = NULL; + PyObject *android_log_write = NULL; + PyObject *result = NULL; + + _android_support = PyImport_ImportModule("_android_support"); + if (_android_support == NULL) { + goto error; + } + + android_log_write = PyCFunction_New(&android_log_write_method, NULL); + if (android_log_write == NULL) { + goto error; + } + + // These log priorities match those used by Java's System.out and System.err. + result = PyObject_CallMethod(_android_support, "init_streams", "Oii", + android_log_write, ANDROID_LOG_INFO, + ANDROID_LOG_WARN); + if (result == NULL) { + goto error; + } + + goto done; error: - _PyErr_Print(tstate); - status = _PyStatus_ERR("failed to initialize Android streams"); + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Android streams"); done: - Py_XDECREF(result); - Py_XDECREF(android_log_write); - Py_XDECREF(_android_support); - return status; + Py_XDECREF(result); + Py_XDECREF(android_log_write); + Py_XDECREF(_android_support); + return status; } -#endif // __ANDROID__ +#endif // __ANDROID__ #if defined(__APPLE__) && HAS_APPLE_SYSTEM_LOG -static PyObject * -apple_log_write_impl(PyObject *self, PyObject *args) -{ - int logtype = 0; - const char *text = NULL; - if (!PyArg_ParseTuple(args, "iy", &logtype, &text)) { - return NULL; - } +static PyObject *apple_log_write_impl(PyObject *self, PyObject *args) { + int logtype = 0; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "iy", &logtype, &text)) { + return NULL; + } - // Pass the user-provided text through explicit %s formatting - // to avoid % literals being interpreted as a formatting directive. - os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); - Py_RETURN_NONE; + // Pass the user-provided text through explicit %s formatting + // to avoid % literals being interpreted as a formatting directive. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + Py_RETURN_NONE; } - static PyMethodDef apple_log_write_method = { - "apple_log_write", apple_log_write_impl, METH_VARARGS -}; - - -static PyStatus -init_apple_streams(PyThreadState *tstate) -{ - PyStatus status = _PyStatus_OK(); - PyObject *_apple_support = NULL; - PyObject *apple_log_write = NULL; - PyObject *result = NULL; - - _apple_support = PyImport_ImportModule("_apple_support"); - if (_apple_support == NULL) { - goto error; - } - - apple_log_write = PyCFunction_New(&apple_log_write_method, NULL); - if (apple_log_write == NULL) { - goto error; - } - - // Initialize the logging streams, sending stdout -> Default; stderr -> Error - result = PyObject_CallMethod( - _apple_support, "init_streams", "Oii", - apple_log_write, OS_LOG_TYPE_DEFAULT, OS_LOG_TYPE_ERROR); - if (result == NULL) { - goto error; - } - goto done; + "apple_log_write", apple_log_write_impl, METH_VARARGS}; + +static PyStatus init_apple_streams(PyThreadState *tstate) { + PyStatus status = _PyStatus_OK(); + PyObject *_apple_support = NULL; + PyObject *apple_log_write = NULL; + PyObject *result = NULL; + + _apple_support = PyImport_ImportModule("_apple_support"); + if (_apple_support == NULL) { + goto error; + } + + apple_log_write = PyCFunction_New(&apple_log_write_method, NULL); + if (apple_log_write == NULL) { + goto error; + } + + // Initialize the logging streams, sending stdout -> Default; stderr -> Error + result = PyObject_CallMethod(_apple_support, "init_streams", "Oii", + apple_log_write, OS_LOG_TYPE_DEFAULT, + OS_LOG_TYPE_ERROR); + if (result == NULL) { + goto error; + } + goto done; error: - _PyErr_Print(tstate); - status = _PyStatus_ERR("failed to initialize Apple log streams"); + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Apple log streams"); done: - Py_XDECREF(result); - Py_XDECREF(apple_log_write); - Py_XDECREF(_apple_support); - return status; + Py_XDECREF(result); + Py_XDECREF(apple_log_write); + Py_XDECREF(_apple_support); + return status; } -#endif // __APPLE__ && HAS_APPLE_SYSTEM_LOG - +#endif // __APPLE__ && HAS_APPLE_SYSTEM_LOG -static void -_Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, - PyThreadState *tstate) -{ - PUTS(fd, "\n"); +static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, + PyThreadState *tstate) { + PUTS(fd, "\n"); - /* display the current Python stack */ + /* display the current Python stack */ #ifndef Py_GIL_DISABLED - _Py_DumpTracebackThreads(fd, interp, tstate); + _Py_DumpTracebackThreads(fd, interp, tstate); #else - _Py_DumpTraceback(fd, tstate); + _Py_DumpTraceback(fd, tstate); #endif } @@ -3387,146 +3159,126 @@ _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, Return 1 if the traceback was displayed, 0 otherwise. */ -static int -_Py_FatalError_PrintExc(PyThreadState *tstate) -{ - PyObject *exc = _PyErr_GetRaisedException(tstate); - if (exc == NULL) { - /* No current exception */ - return 0; - } - - PyObject *ferr; - if (PySys_GetOptionalAttr(&_Py_ID(stderr), &ferr) < 0) { - _PyErr_Clear(tstate); - } - if (ferr == NULL || ferr == Py_None) { - /* sys.stderr is not set yet or set to None, - no need to try to display the exception */ - Py_XDECREF(ferr); - Py_DECREF(exc); - return 0; - } +static int _Py_FatalError_PrintExc(PyThreadState *tstate) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (exc == NULL) { + /* No current exception */ + return 0; + } + + PyObject *ferr; + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &ferr) < 0) { + _PyErr_Clear(tstate); + } + if (ferr == NULL || ferr == Py_None) { + /* sys.stderr is not set yet or set to None, + no need to try to display the exception */ + Py_XDECREF(ferr); + Py_DECREF(exc); + return 0; + } - PyErr_DisplayException(exc); + PyErr_DisplayException(exc); - PyObject *tb = PyException_GetTraceback(exc); - int has_tb = (tb != NULL) && (tb != Py_None); - Py_XDECREF(tb); - Py_DECREF(exc); + PyObject *tb = PyException_GetTraceback(exc); + int has_tb = (tb != NULL) && (tb != Py_None); + Py_XDECREF(tb); + Py_DECREF(exc); - /* sys.stderr may be buffered: call sys.stderr.flush() */ - if (_PyFile_Flush(ferr) < 0) { - _PyErr_Clear(tstate); - } - Py_DECREF(ferr); + /* sys.stderr may be buffered: call sys.stderr.flush() */ + if (_PyFile_Flush(ferr) < 0) { + _PyErr_Clear(tstate); + } + Py_DECREF(ferr); - return has_tb; + return has_tb; } /* Print fatal error message and abort */ #ifdef MS_WINDOWS -static void -fatal_output_debug(const char *msg) -{ - /* buffer of 256 bytes allocated on the stack */ - WCHAR buffer[256 / sizeof(WCHAR)]; - size_t buflen = Py_ARRAY_LENGTH(buffer) - 1; - size_t msglen; - - OutputDebugStringW(L"Fatal Python error: "); - - msglen = strlen(msg); - while (msglen) { - size_t i; - - if (buflen > msglen) { - buflen = msglen; - } +static void fatal_output_debug(const char *msg) { + /* buffer of 256 bytes allocated on the stack */ + WCHAR buffer[256 / sizeof(WCHAR)]; + size_t buflen = Py_ARRAY_LENGTH(buffer) - 1; + size_t msglen; - /* Convert the message to wchar_t. This uses a simple one-to-one - conversion, assuming that the this error message actually uses - ASCII only. If this ceases to be true, we will have to convert. */ - for (i=0; i < buflen; ++i) { - buffer[i] = msg[i]; - } - buffer[i] = L'\0'; - OutputDebugStringW(buffer); - - msg += buflen; - msglen -= buflen; - } - OutputDebugStringW(L"\n"); -} -#endif + OutputDebugStringW(L"Fatal Python error: "); + msglen = strlen(msg); + while (msglen) { + size_t i; -static void -fatal_error_dump_runtime(int fd, _PyRuntimeState *runtime) -{ - PUTS(fd, "Python runtime state: "); - PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(runtime); - if (finalizing) { - PUTS(fd, "finalizing (tstate=0x"); - _Py_DumpHexadecimal(fd, (uintptr_t)finalizing, sizeof(finalizing) * 2); - PUTS(fd, ")"); - } - else if (_PyRuntimeState_GetInitialized(runtime)) { - PUTS(fd, "initialized"); - } - else if (_PyRuntimeState_GetCoreInitialized(runtime)) { - PUTS(fd, "core initialized"); - } - else if (runtime->preinitialized) { - PUTS(fd, "preinitialized"); + if (buflen > msglen) { + buflen = msglen; } - else if (runtime->preinitializing) { - PUTS(fd, "preinitializing"); - } - else { - PUTS(fd, "unknown"); + + /* Convert the message to wchar_t. This uses a simple one-to-one + conversion, assuming that the this error message actually uses + ASCII only. If this ceases to be true, we will have to convert. */ + for (i = 0; i < buflen; ++i) { + buffer[i] = msg[i]; } - PUTS(fd, "\n"); -} + buffer[i] = L'\0'; + OutputDebugStringW(buffer); + msg += buflen; + msglen -= buflen; + } + OutputDebugStringW(L"\n"); +} +#endif -static inline void _Py_NO_RETURN -fatal_error_exit(int status) -{ - if (status < 0) { +static void fatal_error_dump_runtime(int fd, _PyRuntimeState *runtime) { + PUTS(fd, "Python runtime state: "); + PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(runtime); + if (finalizing) { + PUTS(fd, "finalizing (tstate=0x"); + _Py_DumpHexadecimal(fd, (uintptr_t)finalizing, sizeof(finalizing) * 2); + PUTS(fd, ")"); + } else if (_PyRuntimeState_GetInitialized(runtime)) { + PUTS(fd, "initialized"); + } else if (_PyRuntimeState_GetCoreInitialized(runtime)) { + PUTS(fd, "core initialized"); + } else if (runtime->preinitialized) { + PUTS(fd, "preinitialized"); + } else if (runtime->preinitializing) { + PUTS(fd, "preinitializing"); + } else { + PUTS(fd, "unknown"); + } + PUTS(fd, "\n"); +} + +static inline void _Py_NO_RETURN fatal_error_exit(int status) { + if (status < 0) { #if defined(MS_WINDOWS) && defined(Py_DEBUG) - DebugBreak(); + DebugBreak(); #endif - abort(); - } - else { - exit(status); - } + abort(); + } else { + exit(status); + } } -static inline int -acquire_dict_lock_for_dump(PyObject *obj) -{ +static inline int acquire_dict_lock_for_dump(PyObject *obj) { #ifdef Py_GIL_DISABLED - PyMutex *mutex = &obj->ob_mutex; - if (_PyMutex_LockTimed(mutex, 0, 0) == PY_LOCK_ACQUIRED) { - return 1; - } - return 0; -#else + PyMutex *mutex = &obj->ob_mutex; + if (_PyMutex_LockTimed(mutex, 0, 0) == PY_LOCK_ACQUIRED) { return 1; + } + return 0; +#else + return 1; #endif } -static inline void -release_dict_lock_for_dump(PyObject *obj) -{ +static inline void release_dict_lock_for_dump(PyObject *obj) { #ifdef Py_GIL_DISABLED - PyMutex *mutex = &obj->ob_mutex; - // We can not call PyMutex_Unlock because it's not async-signal-safe. - // So not to wake up other threads, we just use a simple atomic store in here. - _Py_atomic_store_uint8(&mutex->_bits, _Py_UNLOCKED); + PyMutex *mutex = &obj->ob_mutex; + // We can not call PyMutex_Unlock because it's not async-signal-safe. + // So not to wake up other threads, we just use a simple atomic store in here. + _Py_atomic_store_uint8(&mutex->_bits, _Py_UNLOCKED); #endif } @@ -3536,440 +3288,397 @@ release_dict_lock_for_dump(PyObject *obj) // This function is called by a signal handler in faulthandler: avoid memory // allocations and keep the implementation simple. For example, the list is not // sorted on purpose. -void -_Py_DumpExtensionModules(int fd, PyInterpreterState *interp) -{ - if (interp == NULL) { - return; - } - PyObject *modules = _PyImport_GetModules(interp); - if (modules == NULL || !PyDict_Check(modules)) { - return; - } +void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) { + if (interp == NULL) { + return; + } + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL || !PyDict_Check(modules)) { + return; + } - Py_ssize_t pos; - PyObject *key, *value; + Py_ssize_t pos; + PyObject *key, *value; - // Avoid PyDict_GetItemString() which calls PyUnicode_FromString(), - // memory cannot be allocated on the heap in a signal handler. - // Iterate on the dict instead. - PyObject *stdlib_module_names = NULL; - if (interp->sysdict != NULL) { - pos = 0; - if (!acquire_dict_lock_for_dump(interp->sysdict)) { - // If we cannot acquire the lock, just don't dump the list of extension modules. - return; - } - while (_PyDict_Next(interp->sysdict, &pos, &key, &value, NULL)) { - if (PyUnicode_Check(key) - && PyUnicode_CompareWithASCIIString(key, "stdlib_module_names") == 0) { - stdlib_module_names = value; - break; - } + // Avoid PyDict_GetItemString() which calls PyUnicode_FromString(), + // memory cannot be allocated on the heap in a signal handler. + // Iterate on the dict instead. + PyObject *stdlib_module_names = NULL; + if (interp->sysdict != NULL) { + pos = 0; + if (!acquire_dict_lock_for_dump(interp->sysdict)) { + // If we cannot acquire the lock, just don't dump the list of extension + // modules. + return; + } + while (_PyDict_Next(interp->sysdict, &pos, &key, &value, NULL)) { + if (PyUnicode_Check(key) && + PyUnicode_CompareWithASCIIString(key, "stdlib_module_names") == 0) { + stdlib_module_names = value; + break; + } + } + release_dict_lock_for_dump(interp->sysdict); + } + // If we failed to get sys.stdlib_module_names or it's not a frozenset, + // don't exclude stdlib modules. + if (stdlib_module_names != NULL && !PyFrozenSet_Check(stdlib_module_names)) { + stdlib_module_names = NULL; + } + + // List extensions + int header = 1; + Py_ssize_t count = 0; + pos = 0; + if (!acquire_dict_lock_for_dump(modules)) { + // If we cannot acquire the lock, just don't dump the list of extension + // modules. + return; + } + while (_PyDict_Next(modules, &pos, &key, &value, NULL)) { + if (!PyUnicode_Check(key)) { + continue; + } + if (!_PyModule_IsExtension(value)) { + continue; + } + // Use the module name from the sys.modules key, + // don't attempt to get the module object name. + if (stdlib_module_names != NULL) { + int is_stdlib_ext = 0; + + Py_ssize_t i = 0; + PyObject *item; + Py_hash_t hash; + // if stdlib_module_names is not NULL, it is always a frozenset. + while (_PySet_NextEntry(stdlib_module_names, &i, &item, &hash)) { + if (!PyUnicode_Check(item)) { + continue; + } + Py_ssize_t len = PyUnicode_GET_LENGTH(item); + if (PyUnicode_Tailmatch(key, item, 0, len, -1) == 1) { + Py_ssize_t key_len = PyUnicode_GET_LENGTH(key); + if (key_len == len) { + is_stdlib_ext = 1; + break; + } + assert(key_len > len); + + // Ignore sub-modules of stdlib packages. For example, + // ignore "math.integer" if key starts with "math.". + Py_UCS4 ch = PyUnicode_ReadChar(key, len); + if (ch == '.') { + is_stdlib_ext = 1; + break; + } } - release_dict_lock_for_dump(interp->sysdict); - } - // If we failed to get sys.stdlib_module_names or it's not a frozenset, - // don't exclude stdlib modules. - if (stdlib_module_names != NULL && !PyFrozenSet_Check(stdlib_module_names)) { - stdlib_module_names = NULL; + } + if (is_stdlib_ext) { + // Ignore stdlib extension + continue; + } } - // List extensions - int header = 1; - Py_ssize_t count = 0; - pos = 0; - if (!acquire_dict_lock_for_dump(modules)) { - // If we cannot acquire the lock, just don't dump the list of extension modules. - return; + if (header) { + PUTS(fd, "\nExtension modules: "); + header = 0; + } else { + PUTS(fd, ", "); } - while (_PyDict_Next(modules, &pos, &key, &value, NULL)) { - if (!PyUnicode_Check(key)) { - continue; - } - if (!_PyModule_IsExtension(value)) { - continue; - } - // Use the module name from the sys.modules key, - // don't attempt to get the module object name. - if (stdlib_module_names != NULL) { - int is_stdlib_ext = 0; - - Py_ssize_t i = 0; - PyObject *item; - Py_hash_t hash; - // if stdlib_module_names is not NULL, it is always a frozenset. - while (_PySet_NextEntry(stdlib_module_names, &i, &item, &hash)) { - if (!PyUnicode_Check(item)) { - continue; - } - Py_ssize_t len = PyUnicode_GET_LENGTH(item); - if (PyUnicode_Tailmatch(key, item, 0, len, -1) == 1) { - Py_ssize_t key_len = PyUnicode_GET_LENGTH(key); - if (key_len == len) { - is_stdlib_ext = 1; - break; - } - assert(key_len > len); - - // Ignore sub-modules of stdlib packages. For example, - // ignore "math.integer" if key starts with "math.". - Py_UCS4 ch = PyUnicode_ReadChar(key, len); - if (ch == '.') { - is_stdlib_ext = 1; - break; - } - } - } - if (is_stdlib_ext) { - // Ignore stdlib extension - continue; - } - } - - if (header) { - PUTS(fd, "\nExtension modules: "); - header = 0; - } - else { - PUTS(fd, ", "); - } - _Py_DumpASCII(fd, key); - count++; - } - release_dict_lock_for_dump(modules); + _Py_DumpASCII(fd, key); + count++; + } + release_dict_lock_for_dump(modules); - if (count) { - PUTS(fd, " (total: "); - _Py_DumpDecimal(fd, count); - PUTS(fd, ")"); - PUTS(fd, "\n"); - } + if (count) { + PUTS(fd, " (total: "); + _Py_DumpDecimal(fd, count); + PUTS(fd, ")"); + PUTS(fd, "\n"); + } } +static void _Py_NO_RETURN fatal_error(int fd, int header, const char *prefix, + const char *msg, int status) { + static int reentrant = 0; -static void _Py_NO_RETURN -fatal_error(int fd, int header, const char *prefix, const char *msg, - int status) -{ - static int reentrant = 0; + if (reentrant) { + /* Py_FatalError() caused a second fatal error. + Example: flush_std_files() raises a recursion error. */ + fatal_error_exit(status); + } + reentrant = 1; - if (reentrant) { - /* Py_FatalError() caused a second fatal error. - Example: flush_std_files() raises a recursion error. */ - fatal_error_exit(status); + if (header) { + PUTS(fd, "Fatal Python error: "); + if (prefix) { + PUTS(fd, prefix); + PUTS(fd, ": "); } - reentrant = 1; - - if (header) { - PUTS(fd, "Fatal Python error: "); - if (prefix) { - PUTS(fd, prefix); - PUTS(fd, ": "); - } - if (msg) { - PUTS(fd, msg); - } - else { - PUTS(fd, ""); - } - PUTS(fd, "\n"); + if (msg) { + PUTS(fd, msg); + } else { + PUTS(fd, ""); } + PUTS(fd, "\n"); + } - _PyRuntimeState *runtime = &_PyRuntime; - fatal_error_dump_runtime(fd, runtime); - - /* Check if the current thread has a Python thread state - and holds the GIL. - - tss_tstate is NULL if Py_FatalError() is called from a C thread which - has no Python thread state. - - tss_tstate != tstate if the current Python thread does not hold the GIL. - */ - PyThreadState *tstate = _PyThreadState_GET(); - PyInterpreterState *interp = NULL; - PyThreadState *tss_tstate = PyGILState_GetThisThreadState(); - if (tstate != NULL) { - interp = tstate->interp; - } - else if (tss_tstate != NULL) { - interp = tss_tstate->interp; - } - int has_tstate_and_gil = (tss_tstate != NULL && tss_tstate == tstate); - - if (has_tstate_and_gil) { - /* If an exception is set, print the exception with its traceback */ - if (!_Py_FatalError_PrintExc(tss_tstate)) { - /* No exception is set, or an exception is set without traceback */ - _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate); - } - } - else { - _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate); - } + _PyRuntimeState *runtime = &_PyRuntime; + fatal_error_dump_runtime(fd, runtime); - _Py_DumpExtensionModules(fd, interp); + /* Check if the current thread has a Python thread state + and holds the GIL. - /* The main purpose of faulthandler is to display the traceback. - This function already did its best to display a traceback. - Disable faulthandler to prevent writing a second traceback - on abort(). */ - _PyFaulthandler_Fini(); + tss_tstate is NULL if Py_FatalError() is called from a C thread which + has no Python thread state. - /* Check if the current Python thread hold the GIL */ - if (has_tstate_and_gil) { - /* Flush sys.stdout and sys.stderr */ - flush_std_files(); - } + tss_tstate != tstate if the current Python thread does not hold the GIL. + */ + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = NULL; + PyThreadState *tss_tstate = PyGILState_GetThisThreadState(); + if (tstate != NULL) { + interp = tstate->interp; + } else if (tss_tstate != NULL) { + interp = tss_tstate->interp; + } + int has_tstate_and_gil = (tss_tstate != NULL && tss_tstate == tstate); + + if (has_tstate_and_gil) { + /* If an exception is set, print the exception with its traceback */ + if (!_Py_FatalError_PrintExc(tss_tstate)) { + /* No exception is set, or an exception is set without traceback */ + _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate); + } + } else { + _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate); + } + + _Py_DumpExtensionModules(fd, interp); + + /* The main purpose of faulthandler is to display the traceback. + This function already did its best to display a traceback. + Disable faulthandler to prevent writing a second traceback + on abort(). */ + _PyFaulthandler_Fini(); + + /* Check if the current Python thread hold the GIL */ + if (has_tstate_and_gil) { + /* Flush sys.stdout and sys.stderr */ + flush_std_files(); + } #ifdef MS_WINDOWS - fatal_output_debug(msg); + fatal_output_debug(msg); #endif /* MS_WINDOWS */ - fatal_error_exit(status); + fatal_error_exit(status); } - #undef Py_FatalError -void _Py_NO_RETURN -Py_FatalError(const char *msg) -{ - fatal_error(fileno(stderr), 1, NULL, msg, -1); +void _Py_NO_RETURN Py_FatalError(const char *msg) { + fatal_error(fileno(stderr), 1, NULL, msg, -1); } - -void _Py_NO_RETURN -_Py_FatalErrorFunc(const char *func, const char *msg) -{ - fatal_error(fileno(stderr), 1, func, msg, -1); +void _Py_NO_RETURN _Py_FatalErrorFunc(const char *func, const char *msg) { + fatal_error(fileno(stderr), 1, func, msg, -1); } +void _Py_NO_RETURN _Py_FatalErrorFormat(const char *func, const char *format, + ...) { + static int reentrant = 0; + if (reentrant) { + /* _Py_FatalErrorFormat() caused a second fatal error */ + fatal_error_exit(-1); + } + reentrant = 1; -void _Py_NO_RETURN -_Py_FatalErrorFormat(const char *func, const char *format, ...) -{ - static int reentrant = 0; - if (reentrant) { - /* _Py_FatalErrorFormat() caused a second fatal error */ - fatal_error_exit(-1); - } - reentrant = 1; - - FILE *stream = stderr; - const int fd = fileno(stream); - PUTS(fd, "Fatal Python error: "); - if (func) { - PUTS(fd, func); - PUTS(fd, ": "); - } + FILE *stream = stderr; + const int fd = fileno(stream); + PUTS(fd, "Fatal Python error: "); + if (func) { + PUTS(fd, func); + PUTS(fd, ": "); + } - va_list vargs; - va_start(vargs, format); - vfprintf(stream, format, vargs); - va_end(vargs); + va_list vargs; + va_start(vargs, format); + vfprintf(stream, format, vargs); + va_end(vargs); - fputs("\n", stream); - fflush(stream); + fputs("\n", stream); + fflush(stream); - fatal_error(fd, 0, NULL, NULL, -1); + fatal_error(fd, 0, NULL, NULL, -1); } - -void _Py_NO_RETURN -_Py_FatalRefcountErrorFunc(const char *func, const char *msg) -{ - _Py_FatalErrorFormat(func, - "%s: bug likely caused by a refcount error " - "in a C extension", - msg); +void _Py_NO_RETURN _Py_FatalRefcountErrorFunc(const char *func, + const char *msg) { + _Py_FatalErrorFormat(func, + "%s: bug likely caused by a refcount error " + "in a C extension", + msg); } - -void _Py_NO_RETURN -Py_ExitStatusException(PyStatus status) -{ - if (_PyStatus_IS_EXIT(status)) { - exit(status.exitcode); - } - else if (_PyStatus_IS_ERROR(status)) { - fatal_error(fileno(stderr), 1, status.func, status.err_msg, 1); - } - else { - Py_FatalError("Py_ExitStatusException() must not be called on success"); - } +void _Py_NO_RETURN Py_ExitStatusException(PyStatus status) { + if (_PyStatus_IS_EXIT(status)) { + exit(status.exitcode); + } else if (_PyStatus_IS_ERROR(status)) { + fatal_error(fileno(stderr), 1, status.func, status.err_msg, 1); + } else { + Py_FatalError("Py_ExitStatusException() must not be called on success"); + } } +static void handle_thread_shutdown_exception(PyThreadState *tstate) { + assert(tstate != NULL); + assert(_PyErr_Occurred(tstate)); + PyInterpreterState *interp = tstate->interp; + assert(interp->threads.head != NULL); + _PyEval_StopTheWorld(interp); -static void -handle_thread_shutdown_exception(PyThreadState *tstate) -{ - assert(tstate != NULL); - assert(_PyErr_Occurred(tstate)); - PyInterpreterState *interp = tstate->interp; - assert(interp->threads.head != NULL); - _PyEval_StopTheWorld(interp); - - // We don't have to worry about locking this because the - // world is stopped. - _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { - if (tstate->_whence == _PyThreadState_WHENCE_THREADING) { - tstate->_whence = _PyThreadState_WHENCE_THREADING_DAEMON; - } + // We don't have to worry about locking this because the + // world is stopped. + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + if (tstate->_whence == _PyThreadState_WHENCE_THREADING) { + tstate->_whence = _PyThreadState_WHENCE_THREADING_DAEMON; } + } - _PyEval_StartTheWorld(interp); - PyErr_FormatUnraisable("Exception ignored on threading shutdown"); + _PyEval_StartTheWorld(interp); + PyErr_FormatUnraisable("Exception ignored on threading shutdown"); } /* Wait until threading._shutdown completes, provided the threading module was imported in the first place. The shutdown routine will wait until all non-daemon "threading" threads have completed. */ -static void -wait_for_thread_shutdown(PyThreadState *tstate) -{ - PyObject *result; - PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); - if (threading == NULL) { - if (_PyErr_Occurred(tstate)) { - handle_thread_shutdown_exception(tstate); - } - /* else: threading not imported */ - return; - } - result = PyObject_CallMethodNoArgs(threading, &_Py_ID(_shutdown)); - if (result == NULL) { - handle_thread_shutdown_exception(tstate); - } - else { - Py_DECREF(result); - } - Py_DECREF(threading); -} - -int Py_AtExit(void (*func)(void)) -{ - struct _atexit_runtime_state *state = &_PyRuntime.atexit; - PyMutex_Lock(&state->mutex); - if (state->ncallbacks >= NEXITFUNCS) { - PyMutex_Unlock(&state->mutex); - return -1; - } - state->callbacks[state->ncallbacks++] = func; +static void wait_for_thread_shutdown(PyThreadState *tstate) { + PyObject *result; + PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); + if (threading == NULL) { + if (_PyErr_Occurred(tstate)) { + handle_thread_shutdown_exception(tstate); + } + /* else: threading not imported */ + return; + } + result = PyObject_CallMethodNoArgs(threading, &_Py_ID(_shutdown)); + if (result == NULL) { + handle_thread_shutdown_exception(tstate); + } else { + Py_DECREF(result); + } + Py_DECREF(threading); +} + +int Py_AtExit(void (*func)(void)) { + struct _atexit_runtime_state *state = &_PyRuntime.atexit; + PyMutex_Lock(&state->mutex); + if (state->ncallbacks >= NEXITFUNCS) { PyMutex_Unlock(&state->mutex); - return 0; + return -1; + } + state->callbacks[state->ncallbacks++] = func; + PyMutex_Unlock(&state->mutex); + return 0; } -static void -call_ll_exitfuncs(_PyRuntimeState *runtime) -{ - atexit_callbackfunc exitfunc; - struct _atexit_runtime_state *state = &runtime->atexit; +static void call_ll_exitfuncs(_PyRuntimeState *runtime) { + atexit_callbackfunc exitfunc; + struct _atexit_runtime_state *state = &runtime->atexit; - PyMutex_Lock(&state->mutex); - while (state->ncallbacks > 0) { - /* pop last function from the list */ - state->ncallbacks--; - exitfunc = state->callbacks[state->ncallbacks]; - state->callbacks[state->ncallbacks] = NULL; + PyMutex_Lock(&state->mutex); + while (state->ncallbacks > 0) { + /* pop last function from the list */ + state->ncallbacks--; + exitfunc = state->callbacks[state->ncallbacks]; + state->callbacks[state->ncallbacks] = NULL; - PyMutex_Unlock(&state->mutex); - exitfunc(); - PyMutex_Lock(&state->mutex); - } PyMutex_Unlock(&state->mutex); + exitfunc(); + PyMutex_Lock(&state->mutex); + } + PyMutex_Unlock(&state->mutex); - fflush(stdout); - fflush(stderr); + fflush(stdout); + fflush(stderr); } -void _Py_NO_RETURN -Py_Exit(int sts) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate != NULL && _PyThreadState_IsRunningMain(tstate)) { - _PyInterpreterState_SetNotRunningMain(tstate->interp); - } - if (_Py_Finalize(&_PyRuntime) < 0) { - sts = 120; - } +void _Py_NO_RETURN Py_Exit(int sts) { + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate != NULL && _PyThreadState_IsRunningMain(tstate)) { + _PyInterpreterState_SetNotRunningMain(tstate->interp); + } + if (_Py_Finalize(&_PyRuntime) < 0) { + sts = 120; + } - exit(sts); + exit(sts); } - /* * The file descriptor fd is considered ``interactive'' if either * a) isatty(fd) is TRUE, or * b) the -i flag was given, and the filename associated with * the descriptor is NULL or "" or "???". */ -int -Py_FdIsInteractive(FILE *fp, const char *filename) -{ - if (isatty(fileno(fp))) { - return 1; - } - if (!_Py_GetConfig()->interactive) { - return 0; - } - return ((filename == NULL) - || (strcmp(filename, "") == 0) - || (strcmp(filename, "???") == 0)); +int Py_FdIsInteractive(FILE *fp, const char *filename) { + if (isatty(fileno(fp))) { + return 1; + } + if (!_Py_GetConfig()->interactive) { + return 0; + } + return ((filename == NULL) || (strcmp(filename, "") == 0) || + (strcmp(filename, "???") == 0)); } - -int -_Py_FdIsInteractive(FILE *fp, PyObject *filename) -{ - if (isatty(fileno(fp))) { - return 1; - } - if (!_Py_GetConfig()->interactive) { - return 0; - } - return ((filename == NULL) - || (PyUnicode_CompareWithASCIIString(filename, "") == 0) - || (PyUnicode_CompareWithASCIIString(filename, "???") == 0)); +int _Py_FdIsInteractive(FILE *fp, PyObject *filename) { + if (isatty(fileno(fp))) { + return 1; + } + if (!_Py_GetConfig()->interactive) { + return 0; + } + return ((filename == NULL) || + (PyUnicode_CompareWithASCIIString(filename, "") == 0) || + (PyUnicode_CompareWithASCIIString(filename, "???") == 0)); } - /* Wrappers around sigaction() or signal(). */ -PyOS_sighandler_t -PyOS_getsig(int sig) -{ +PyOS_sighandler_t PyOS_getsig(int sig) { #ifdef HAVE_SIGACTION - struct sigaction context; - if (sigaction(sig, NULL, &context) == -1) - return SIG_ERR; - return context.sa_handler; + struct sigaction context; + if (sigaction(sig, NULL, &context) == -1) + return SIG_ERR; + return context.sa_handler; #else - PyOS_sighandler_t handler; + PyOS_sighandler_t handler; /* Special signal handling for the secure CRT in Visual Studio 2005 */ #if defined(_MSC_VER) && _MSC_VER >= 1400 - switch (sig) { - /* Only these signals are valid */ - case SIGINT: - case SIGILL: - case SIGFPE: - case SIGSEGV: - case SIGTERM: - case SIGBREAK: - case SIGABRT: - break; - /* Don't call signal() with other values or it will assert */ - default: - return SIG_ERR; - } + switch (sig) { + /* Only these signals are valid */ + case SIGINT: + case SIGILL: + case SIGFPE: + case SIGSEGV: + case SIGTERM: + case SIGBREAK: + case SIGABRT: + break; + /* Don't call signal() with other values or it will assert */ + default: + return SIG_ERR; + } #endif /* _MSC_VER && _MSC_VER >= 1400 */ - handler = signal(sig, SIG_IGN); - if (handler != SIG_ERR) - signal(sig, handler); - return handler; + handler = signal(sig, SIG_IGN); + if (handler != SIG_ERR) + signal(sig, handler); + return handler; #endif } @@ -3978,30 +3687,28 @@ PyOS_getsig(int sig) * listed at `man 7 signal-safety` or * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. */ -PyOS_sighandler_t -PyOS_setsig(int sig, PyOS_sighandler_t handler) -{ +PyOS_sighandler_t PyOS_setsig(int sig, PyOS_sighandler_t handler) { #ifdef HAVE_SIGACTION - /* Some code in Modules/signalmodule.c depends on sigaction() being - * used here if HAVE_SIGACTION is defined. Fix that if this code - * changes to invalidate that assumption. - */ - struct sigaction context, ocontext; - context.sa_handler = handler; - sigemptyset(&context.sa_mask); - /* Using SA_ONSTACK is friendlier to other C/C++/Golang-VM code that - * extension module or embedding code may use where tiny thread stacks - * are used. https://bugs.python.org/issue43390 */ - context.sa_flags = SA_ONSTACK; - if (sigaction(sig, &context, &ocontext) == -1) - return SIG_ERR; - return ocontext.sa_handler; + /* Some code in Modules/signalmodule.c depends on sigaction() being + * used here if HAVE_SIGACTION is defined. Fix that if this code + * changes to invalidate that assumption. + */ + struct sigaction context, ocontext; + context.sa_handler = handler; + sigemptyset(&context.sa_mask); + /* Using SA_ONSTACK is friendlier to other C/C++/Golang-VM code that + * extension module or embedding code may use where tiny thread stacks + * are used. https://bugs.python.org/issue43390 */ + context.sa_flags = SA_ONSTACK; + if (sigaction(sig, &context, &ocontext) == -1) + return SIG_ERR; + return ocontext.sa_handler; #else - PyOS_sighandler_t oldhandler; - oldhandler = signal(sig, handler); + PyOS_sighandler_t oldhandler; + oldhandler = signal(sig, handler); #ifdef HAVE_SIGINTERRUPT - siginterrupt(sig, 1); + siginterrupt(sig, 1); #endif - return oldhandler; + return oldhandler; #endif } From eff9b407eb504d42b65b6a34b82f60e76dbd2e52 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 29 Apr 2026 13:54:19 +0000 Subject: [PATCH 13/14] Improve new PyThreadState API docs. Co-authored-by: Petr Viktorin --- Doc/c-api/threads.rst | 140 ++++++------------------------------------ 1 file changed, 18 insertions(+), 122 deletions(-) diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index 08706a29f9a7f0..5b86e84e881570 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -282,46 +282,19 @@ Attaching/detaching thread states Otherwise, if both of the above cases fail, a new thread state is created for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. - This function will return ``NULL`` to indicate a memory allocation failure, and - otherwise return a pointer to the thread state that was previously attached - (which might have been ``NULL``, in which case an non-``NULL`` sentinel value is - returned instead to differentiate between failure -- this means that this function - will sometimes return an invalid ``PyThreadState`` pointer). - - To visualize, this function is roughly equivalent to the following: - - .. code-block:: c - - PyThreadState * - PyThreadState_Ensure(PyInterpreterGuard *guard) - { - assert(guard != NULL); - PyInterpreterState *interp = PyInterpreterGuard_GetInterpreter(guard); - assert(interp != NULL); - - PyThreadState *current_tstate = PyThreadState_GetUnchecked(); - if (current_tstate == NULL) { - PyThreadState *last_used = PyGILState_GetThisThreadState(); - if (last_used != NULL) { - ++last_used->ensure_counter; - PyThreadState_Swap(last_used); - return NO_TSTATE_SENTINEL; - } - } else if (current_tstate->interp == interp) { - ++current_tstate->ensure_counter; - return current_tstate; - } - - PyThreadState *new_tstate = PyThreadState_New(interp); - if (new_tstate == NULL) { - return NULL; - } - - ++new_tstate->ensure_counter; - new_tstate->owned_by_pythreadstate_ensure = true; - PyThreadState_Swap(new_tstate); - return current_tstate == NULL ? NO_TSTATE_SENTINEL : current_tstate; - } + The function's effect (if any) will be reversed by the matching call to + :c:func:`PyThreadState_Release`. + + On error, this function returns ``NULL`` *without* an exception set. + Do not call :c:func:`!PyThreadState_Release` in this case. + + On success, this function returns a pointer to the :term:`thread state` that + was previously attached, or, if no state was previously attached, a value that + is not a valid :c:type:`!PyThreadState` pointer. + + The returned value must be passed to the matching call to :c:func:`!PyThreadState_Release`, + and must not be passed to any other C API functions + (unless it matches a known valid :c:type:`!PyThreadState` pointer). .. versionadded:: next @@ -330,53 +303,17 @@ Attaching/detaching thread states Get an attached thread state for the interpreter referenced by *view*. - *view* must not be ``NULL``. If the interpreter referenced by *view* has been - finalized or is currently finalizing, then this function returns ``NULL`` without - setting an exception. This function may also return ``NULL`` on other errors, - such memory allocation failure. - - The interpreter referenced by *view* will be implicitly guarded. The - guard will be released upon the corresponding :c:func:`PyThreadState_Release` + On success, the interpreter referenced by *view* will be implicitly guarded; + the guard will be released upon the corresponding :c:func:`PyThreadState_Release` call. - - On success, this function will return the thread state that was previously attached. - If no thread state was previously attached, this returns a non-``NULL`` sentinel - value. The behavior of whether this function creates a thread state is - equivalent to that of :c:func:`PyThreadState_Ensure`. - - To visualize, function is roughly equivalent to the following: - - .. code-block:: c - - PyThreadState * - PyThreadState_EnsureFromView(PyInterpreterView *view) - { - assert(view != NULL); - PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); - if (guard == NULL) { - return NULL; - } - - PyThreadState *tstate = PyThreadState_Ensure(guard); - if (tstate == NULL) { - PyInterpreterGuard_Close(guard); - return NULL; - } - - if (tstate->guard == NULL) { - tstate->guard = guard; - } else { - PyInterpreterGuard_Close(guard); - } - - return tstate; - } + Otherwise, the behavior and return value are the same as for + :c:func:`PyThreadState_Ensure`. .. c:function:: void PyThreadState_Release(PyThreadState *tstate) Undo a :c:func:`PyThreadState_Ensure` call. This must be called exactly once - for each call to ``PyThreadState_Ensure``. + for each successful call to ``PyThreadState_Ensure``. This function will decrement an internal counter on the attached thread state. If this counter ever reaches below zero, this function emits a fatal error (via @@ -390,47 +327,6 @@ Attaching/detaching thread states If *tstate* indicates that no prior thread state was attached, there will be no attached thread state upon returning. - To visualize, this function is roughly equivalent to the following: - - .. code-block:: c - - void - PyThreadState_Release(PyThreadState *old_tstate) - { - PyThreadState *current_tstate = PyThreadState_Get(); - assert(old_tstate != NULL); - assert(current_tstate != NULL); - assert(current_tstate->ensure_counter > 0); - if (--current_tstate->ensure_counter > 0) { - // There are remaining PyThreadState_Ensure() calls - // for this thread state. - return; - } - - assert(current_tstate->ensure_counter == 0); - if (old_tstate == NO_TSTATE_SENTINEL) { - // No thread state was attached prior the PyThreadState_Ensure() - // call. So, we can just destroy the current thread state and return. - assert(current_tstate->owned_by_pythreadstate_ensure); - PyThreadState_Clear(current_tstate); - PyThreadState_DeleteCurrent(); - return; - } - - if (tstate->guard != NULL) { - PyInterpreterGuard_Close(tstate->guard); - return; - } - - if (tstate->owned_by_pythreadstate_ensure) { - // The attached thread state was created by the initial PyThreadState_Ensure() - // call. It's our job to destroy it. - PyThreadState_Clear(current_tstate); - PyThreadState_DeleteCurrent(); - } - - PyThreadState_Swap(old_tstate); - } .. _legacy-api: .. _gilstate: From bc78c10a472fdfecca9325b9e36b8c42cbb88da6 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 29 Apr 2026 14:01:49 +0000 Subject: [PATCH 14/14] Fix lint. --- Doc/c-api/interp-lifecycle.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index 8e8e2082f2057f..a6530ce760bbfa 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -628,7 +628,7 @@ it possible to avoid these issues by temporarily preventing finalization: Holding a guard for an interpreter is similar to holding a :term:`strong reference` to a Python object, except finalization does not happen - automatically after all guards are released: it requires an explicit + automatically after all guards are released: it requires an explicit :c:func:`Py_EndInterpreter` call. .. versionadded:: next