From 0fb826331b9223464f6ccf3aec0e98566baa709e Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 26 Oct 2026 08:22:37 +0000 Subject: [PATCH 1/5] Fix free-threaded build --- Include/internal/pycore_ceval.h | 3 +- Lib/test/test_free_threading/test_qsbr.py | 72 +++++++++++++++++++ ...-10-26-10-18-26.gh-issue-160697.qSbR1f.rst | 4 ++ Objects/obmalloc.c | 8 +-- Python/ceval_gil.c | 4 +- 5 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 Lib/test/test_free_threading/test_qsbr.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-10-18-26.gh-issue-160697.qSbR1f.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c0..c404fc6e67d1932 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -350,9 +350,10 @@ _PyEval_SpecialMethodCanSuggest(PyObject *self, int oparg); #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) #define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) +#define _PY_EVAL_PROCESS_DELAYED_BIT (1U << 8) /* Reserve a few bits for future use */ -#define _PY_EVAL_EVENTS_BITS 8 +#define _PY_EVAL_EVENTS_BITS 9 #define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) static inline void diff --git a/Lib/test/test_free_threading/test_qsbr.py b/Lib/test/test_free_threading/test_qsbr.py new file mode 100644 index 000000000000000..8ce37b73e80a047 --- /dev/null +++ b/Lib/test/test_free_threading/test_qsbr.py @@ -0,0 +1,72 @@ +import threading +import unittest + +from test.support import threading_helper + + +class Holder: + pass + + +def make_shared(func, *args): + t = threading.Thread(target=func, args=args) + t.start() + t.join() + + +@threading_helper.requires_working_threading() +class TestDelayedFreeReentrancy(unittest.TestCase): + ROUNDS = 1000 + + def queue_finalizers(self, callback): + class Finalizer: + def __del__(self): + callback() + + holders = [Holder() for _ in range(self.ROUNDS)] + for h in holders: + h.__dict__ = {"f": Finalizer()} + return holders + + def test_dict_resize(self): + d = {} + make_shared(d.get, 0) + holders = self.queue_finalizers(lambda: d.__setitem__(object(), 0)) + for n, h in enumerate(holders): + h.__dict__ = {} + if n % 2: + h.__dict__ = {} + for k in list(d): + del d[k] + for i in range(6): + d[i] = i + self.assertEqual(len(d), len(list(d))) + + def test_list_resize(self): + L = [] + make_shared(L.append, 0) + holders = self.queue_finalizers(L.clear) + for n, h in enumerate(holders): + h.__dict__ = {} + if n % 2: + h.__dict__ = {} + L.clear() + for i in range(9): + L.insert(len(L), i) + self.assertEqual(len(L), len(list(L))) + + def test_type_cache_invalidation(self): + class C: + x = None + + holders = self.queue_finalizers(lambda: C.x) + for h in holders: + h.__dict__ = {} + C.x + new = object() + C.x = new + self.assertIs(C.x, new) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-10-18-26.gh-issue-160697.qSbR1f.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-10-18-26.gh-issue-160697.qSbR1f.rst new file mode 100644 index 000000000000000..55649732e8f4b23 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-10-18-26.gh-issue-160697.qSbR1f.rst @@ -0,0 +1,4 @@ +Fix crashes and hangs in the :term:`free-threaded build` when a destructor +ran in the middle of a :class:`dict` resize, a :class:`list` resize or a type +cache update. The thread's queue of delayed frees is now processed only from +the eval breaker, instead of synchronously when the queue fills up. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 3ad485620ede41e..0f3436cd4eb0fbc 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,6 +1,7 @@ /* Python's malloc wrappers (see pymem.h) */ #include "Python.h" +#include "pycore_ceval.h" #include "pycore_interp.h" // _PyInterpreterState_HasFeature #include "pycore_mmap.h" // _PyAnnotateMemoryMap() #include "pycore_object.h" // _PyDebugAllocatorStats() definition @@ -1513,10 +1514,8 @@ free_delayed(uintptr_t ptr, size_t size) buf->wr_idx++; if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { - // Normally the processing of delayed items is done from the eval - // breaker. Processing here is a safety measure to ensure too much - // work does not accumulate. - _PyMem_ProcessDelayed((PyThreadState *)tstate); + _Py_set_eval_breaker_bit((PyThreadState *)tstate, + _PY_EVAL_PROCESS_DELAYED_BIT); } #endif } @@ -1626,6 +1625,7 @@ _PyMem_ProcessDelayed(PyThreadState *tstate) _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; tstate_impl->qsbr->should_process = false; + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PROCESS_DELAYED_BIT); // Process thread-local work process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, NULL, NULL); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 49b2ede4ed3e60f..bc86164f5ceab88 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1389,7 +1389,9 @@ _Py_HandlePending(PyThreadState *tstate) _Py_brc_merge_refcounts(tstate); } /* Process deferred memory frees held by QSBR */ - if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) { + if ((breaker & _PY_EVAL_PROCESS_DELAYED_BIT) != 0 || + _Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) + { _PyMem_ProcessDelayed(tstate); } #endif From 50ffb1047efa561f385834d65766690c9f72e0fc Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 7 Nov 2026 15:09:39 +0000 Subject: [PATCH 2/5] Remove should_process --- Include/internal/pycore_qsbr.h | 11 ----------- Objects/obmalloc.c | 10 ++-------- Python/ceval_gil.c | 4 +--- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index 91d3415614b154d..98c23384221d9ca 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -61,9 +61,6 @@ struct _qsbr_thread_state { // and in a different thread. size_t deferred_page_memory; - // True if the deferred memory frees should be processed. - bool should_process; - // Is this thread state allocated? bool allocated; struct _qsbr_thread_state *freelist_next; @@ -142,14 +139,6 @@ _Py_qsbr_advance(struct _qsbr_shared *shared); extern uint64_t _Py_qsbr_shared_next(struct _qsbr_shared *shared); -// Return true if deferred memory frees held by QSBR should be processed to -// determine if they can be safely freed. -static inline bool -_Py_qsbr_should_process(struct _qsbr_thread_state *qsbr) -{ - return qsbr->should_process; -} - // Have the read sequences advanced to the given goal? If this returns true, // it safe to reclaim any memory tagged with the goal (or earlier goal). extern bool diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 0f3436cd4eb0fbc..66931dc396fa95a 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1425,7 +1425,7 @@ should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size) if (size > QSBR_FREE_MEM_LIMIT) { qsbr->deferred_count = 0; qsbr->deferred_memory = 0; - qsbr->should_process = true; + _Py_set_eval_breaker_bit(qsbr->tstate, _PY_EVAL_PROCESS_DELAYED_BIT); return true; } qsbr->deferred_count++; @@ -1434,7 +1434,7 @@ should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size) qsbr->deferred_memory > QSBR_FREE_MEM_LIMIT) { qsbr->deferred_count = 0; qsbr->deferred_memory = 0; - qsbr->should_process = true; + _Py_set_eval_breaker_bit(qsbr->tstate, _PY_EVAL_PROCESS_DELAYED_BIT); return true; } return false; @@ -1512,11 +1512,6 @@ free_delayed(uintptr_t ptr, size_t size) buf->array[buf->wr_idx].ptr = ptr; buf->array[buf->wr_idx].qsbr_goal = seq; buf->wr_idx++; - - if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { - _Py_set_eval_breaker_bit((PyThreadState *)tstate, - _PY_EVAL_PROCESS_DELAYED_BIT); - } #endif } @@ -1624,7 +1619,6 @@ _PyMem_ProcessDelayed(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; - tstate_impl->qsbr->should_process = false; _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PROCESS_DELAYED_BIT); // Process thread-local work diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index bc86164f5ceab88..db6d06edf632309 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1389,9 +1389,7 @@ _Py_HandlePending(PyThreadState *tstate) _Py_brc_merge_refcounts(tstate); } /* Process deferred memory frees held by QSBR */ - if ((breaker & _PY_EVAL_PROCESS_DELAYED_BIT) != 0 || - _Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) - { + if ((breaker & _PY_EVAL_PROCESS_DELAYED_BIT) != 0) { _PyMem_ProcessDelayed(tstate); } #endif From b1abedbb8e785300c47f930ce578f3825256e08e Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 7 Nov 2026 15:30:42 +0000 Subject: [PATCH 3/5] Update comment near the MONITOR_VERSION_INCREMENT --- Python/instrumentation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3c9b2afcde100f3..48daa91259d3c9e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -2041,7 +2041,7 @@ check_tool(PyInterpreterState *interp, int tool_id) } /* We share the eval-breaker with flags, so the monitoring - * version goes in the top 24 bits */ + * version goes in the bits above _PY_EVAL_EVENTS_BITS */ #define MONITORING_VERSION_INCREMENT (1 << _PY_EVAL_EVENTS_BITS) int From 783feab236c4c347108c52f6da3fc5bdc3df594f Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 7 Nov 2026 15:32:30 +0000 Subject: [PATCH 4/5] Update InternalDocs/qsbr.md --- InternalDocs/qsbr.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/InternalDocs/qsbr.md b/InternalDocs/qsbr.md index d511396fdab6452..58532a6045d3fec 100644 --- a/InternalDocs/qsbr.md +++ b/InternalDocs/qsbr.md @@ -116,8 +116,9 @@ its last advance: is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT` (127, sized so a chunk of work items is processed before it overflows). - Crossing any of these thresholds also sets a per-thread `should_process` - flag, signalling that the deferred-free list should be drained. + Crossing any of these thresholds also sets the thread's + `_PY_EVAL_PROCESS_DELAYED_BIT` eval breaker bit, signalling that the + deferred-free list should be drained. * For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory` and advances `wr_seq` when either the individual page or the accumulated @@ -125,11 +126,12 @@ its last advance: promptly here matters because a held page cannot be reused for a different size class or by a different thread. -Processing of the deferred-free list normally happens from the eval breaker -(rather than from inside `_PyMem_FreeDelayed`), which gives the global -`rd_seq` a better chance to have advanced far enough that items can actually -be freed. `_PyMem_ProcessDelayed` is still called from the free path as a -safety valve when a work-item chunk fills up. +Processing of the deferred-free list happens from the eval breaker rather +than from inside `_PyMem_FreeDelayed`. This gives the global `rd_seq` a better +chance to have advanced far enough that items can actually be freed. It is +also required for correctness: items queued by `_PyObject_XDecRefDelayed` may +run arbitrary code when they are released, and the callers of +`_PyMem_FreeDelayed` may be in the middle of mutating a dict, list or type. This optimization improves runtime speed but may increase peak memory usage by slightly delaying when memory can be reclaimed; the size-based thresholds From 87ca9ba4f4228ab44f37981af38d13555767e8c9 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 9 Nov 2026 15:06:52 +0000 Subject: [PATCH 5/5] Simplification --- Lib/test/test_free_threading/test_qsbr.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_free_threading/test_qsbr.py b/Lib/test/test_free_threading/test_qsbr.py index 8ce37b73e80a047..ec2a7454ca29079 100644 --- a/Lib/test/test_free_threading/test_qsbr.py +++ b/Lib/test/test_free_threading/test_qsbr.py @@ -18,12 +18,12 @@ def make_shared(func, *args): class TestDelayedFreeReentrancy(unittest.TestCase): ROUNDS = 1000 - def queue_finalizers(self, callback): + def queue_finalizers(self, callback, count=ROUNDS): class Finalizer: def __del__(self): callback() - holders = [Holder() for _ in range(self.ROUNDS)] + holders = [Holder() for _ in range(count)] for h in holders: h.__dict__ = {"f": Finalizer()} return holders @@ -31,16 +31,16 @@ def __del__(self): def test_dict_resize(self): d = {} make_shared(d.get, 0) - holders = self.queue_finalizers(lambda: d.__setitem__(object(), 0)) - for n, h in enumerate(holders): + holders = self.queue_finalizers(lambda: d.__setitem__(object(), 0), + count=100) + for h in holders: h.__dict__ = {} - if n % 2: - h.__dict__ = {} + for _ in range(self.ROUNDS): for k in list(d): del d[k] for i in range(6): d[i] = i - self.assertEqual(len(d), len(list(d))) + self.assertEqual(len(d), sum(1 for _ in d)) def test_list_resize(self): L = []