Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Include/internal/pycore_ceval.h
Original file line number Diff line number Diff line change
Expand Up @@ -350,9 +350,10 @@ _PyEval_SpecialMethodCanSuggest(PyObject *self, int oparg);
#define _PY_EVAL_PLEASE_STOP_BIT (1U << 5)
#define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6)
#define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7)
#define _PY_EVAL_PROCESS_DELAYED_BIT (1U << 8)

/* Reserve a few bits for future use */
#define _PY_EVAL_EVENTS_BITS 8
#define _PY_EVAL_EVENTS_BITS 9

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

/* We share the eval-breaker with flags, so the monitoring
* version goes in the top 24 bits */
#define MONITORING_VERSION_INCREMENT (1 << _PY_EVAL_EVENTS_BITS)

The set_events overflows sooner now, is that fine? Above comment needs updating anyway.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, yes, that's true. A tight set_events() toggle loop now hits OverflowError after 2^23 - 1 calls instead of 2^24 - 1.
Unfortunately, all 8 event bits were already in use, so the new flag needs a ninth. Since only the free-threaded build uses this flag, the alternative would be to define it (and the extra bit) only under free-threaded build. I'd rather not have the bit layout differ between builds, so I've just updated the comment to drop the reference to 24 bits.

#define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1)

static inline void
Expand Down
11 changes: 0 additions & 11 deletions Include/internal/pycore_qsbr.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,6 @@ struct _qsbr_thread_state {
// and in a different thread.
size_t deferred_page_memory;

// True if the deferred memory frees should be processed.
bool should_process;

// Is this thread state allocated?
bool allocated;
struct _qsbr_thread_state *freelist_next;
Expand Down Expand Up @@ -142,14 +139,6 @@ _Py_qsbr_advance(struct _qsbr_shared *shared);
extern uint64_t
_Py_qsbr_shared_next(struct _qsbr_shared *shared);

// Return true if deferred memory frees held by QSBR should be processed to
// determine if they can be safely freed.
static inline bool
_Py_qsbr_should_process(struct _qsbr_thread_state *qsbr)
{
return qsbr->should_process;
}

// Have the read sequences advanced to the given goal? If this returns true,
// it safe to reclaim any memory tagged with the goal (or earlier goal).
extern bool
Expand Down
16 changes: 9 additions & 7 deletions InternalDocs/qsbr.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,20 +116,22 @@ its last advance:
is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred
memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT`
(127, sized so a chunk of work items is processed before it overflows).
Crossing any of these thresholds also sets a per-thread `should_process`
flag, signalling that the deferred-free list should be drained.
Crossing any of these thresholds also sets the thread's
`_PY_EVAL_PROCESS_DELAYED_BIT` eval breaker bit, signalling that the
deferred-free list should be drained.

* For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory`
and advances `wr_seq` when either the individual page or the accumulated
page memory exceeds `QSBR_PAGE_MEM_LIMIT` (4096 * 20 bytes). Advancing
promptly here matters because a held page cannot be reused for a different
size class or by a different thread.

Processing of the deferred-free list normally happens from the eval breaker
(rather than from inside `_PyMem_FreeDelayed`), which gives the global
`rd_seq` a better chance to have advanced far enough that items can actually
be freed. `_PyMem_ProcessDelayed` is still called from the free path as a
safety valve when a work-item chunk fills up.
Processing of the deferred-free list happens from the eval breaker rather
than from inside `_PyMem_FreeDelayed`. This gives the global `rd_seq` a better
chance to have advanced far enough that items can actually be freed. It is
also required for correctness: items queued by `_PyObject_XDecRefDelayed` may
run arbitrary code when they are released, and the callers of
`_PyMem_FreeDelayed` may be in the middle of mutating a dict, list or type.

This optimization improves runtime speed but may increase peak memory usage
by slightly delaying when memory can be reclaimed; the size-based thresholds
Expand Down
72 changes: 72 additions & 0 deletions Lib/test/test_free_threading/test_qsbr.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import threading
import unittest

from test.support import threading_helper


class Holder:
pass


def make_shared(func, *args):
t = threading.Thread(target=func, args=args)
t.start()
t.join()


@threading_helper.requires_working_threading()
class TestDelayedFreeReentrancy(unittest.TestCase):
ROUNDS = 1000

def queue_finalizers(self, callback, count=ROUNDS):
class Finalizer:
def __del__(self):
callback()

holders = [Holder() for _ in range(count)]
for h in holders:
h.__dict__ = {"f": Finalizer()}
return holders

def test_dict_resize(self):
d = {}
make_shared(d.get, 0)
holders = self.queue_finalizers(lambda: d.__setitem__(object(), 0),
count=100)
for h in holders:
h.__dict__ = {}
for _ in range(self.ROUNDS):
for k in list(d):
del d[k]
for i in range(6):
d[i] = i
self.assertEqual(len(d), sum(1 for _ in d))

def test_list_resize(self):
L = []
make_shared(L.append, 0)
holders = self.queue_finalizers(L.clear)
for n, h in enumerate(holders):
h.__dict__ = {}
if n % 2:
h.__dict__ = {}
L.clear()
for i in range(9):
L.insert(len(L), i)
self.assertEqual(len(L), len(list(L)))

def test_type_cache_invalidation(self):
class C:
x = None

holders = self.queue_finalizers(lambda: C.x)
for h in holders:
h.__dict__ = {}
C.x
new = object()
C.x = new
self.assertIs(C.x, new)
Comment on lines +58 to +68

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The type cache is unfortunately affected too.



if __name__ == "__main__":
unittest.main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix crashes and hangs in the :term:`free-threaded build` when a destructor
ran in the middle of a :class:`dict` resize, a :class:`list` resize or a type
cache update. The thread's queue of delayed frees is now processed only from
the eval breaker, instead of synchronously when the queue fills up.
14 changes: 4 additions & 10 deletions Objects/obmalloc.c
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/* Python's malloc wrappers (see pymem.h) */

#include "Python.h"
#include "pycore_ceval.h"
#include "pycore_interp.h" // _PyInterpreterState_HasFeature
#include "pycore_mmap.h" // _PyAnnotateMemoryMap()
#include "pycore_object.h" // _PyDebugAllocatorStats() definition
Expand Down Expand Up @@ -1424,7 +1425,7 @@ should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size)
if (size > QSBR_FREE_MEM_LIMIT) {
qsbr->deferred_count = 0;
qsbr->deferred_memory = 0;
qsbr->should_process = true;
_Py_set_eval_breaker_bit(qsbr->tstate, _PY_EVAL_PROCESS_DELAYED_BIT);
return true;
}
qsbr->deferred_count++;
Expand All @@ -1433,7 +1434,7 @@ should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size)
qsbr->deferred_memory > QSBR_FREE_MEM_LIMIT) {
qsbr->deferred_count = 0;
qsbr->deferred_memory = 0;
qsbr->should_process = true;
_Py_set_eval_breaker_bit(qsbr->tstate, _PY_EVAL_PROCESS_DELAYED_BIT);
return true;
}
return false;
Expand Down Expand Up @@ -1511,13 +1512,6 @@ free_delayed(uintptr_t ptr, size_t size)
buf->array[buf->wr_idx].ptr = ptr;
buf->array[buf->wr_idx].qsbr_goal = seq;
buf->wr_idx++;

if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) {
// Normally the processing of delayed items is done from the eval
// breaker. Processing here is a safety measure to ensure too much
// work does not accumulate.
_PyMem_ProcessDelayed((PyThreadState *)tstate);
}
#endif
}

Expand Down Expand Up @@ -1625,7 +1619,7 @@ _PyMem_ProcessDelayed(PyThreadState *tstate)
PyInterpreterState *interp = tstate->interp;
_PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;

tstate_impl->qsbr->should_process = false;
_Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PROCESS_DELAYED_BIT);

// Process thread-local work
process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, NULL, NULL);
Expand Down
2 changes: 1 addition & 1 deletion Python/ceval_gil.c
Original file line number Diff line number Diff line change
Expand Up @@ -1389,7 +1389,7 @@ _Py_HandlePending(PyThreadState *tstate)
_Py_brc_merge_refcounts(tstate);
}
/* Process deferred memory frees held by QSBR */
if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) {
if ((breaker & _PY_EVAL_PROCESS_DELAYED_BIT) != 0) {
_PyMem_ProcessDelayed(tstate);
}
#endif
Expand Down
2 changes: 1 addition & 1 deletion Python/instrumentation.c
Original file line number Diff line number Diff line change
Expand Up @@ -2041,7 +2041,7 @@ check_tool(PyInterpreterState *interp, int tool_id)
}

/* We share the eval-breaker with flags, so the monitoring
* version goes in the top 24 bits */
* version goes in the bits above _PY_EVAL_EVENTS_BITS */
#define MONITORING_VERSION_INCREMENT (1 << _PY_EVAL_EVENTS_BITS)

int
Expand Down
Loading