3f685e8824
Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
54 lines
1.3 KiB
C
54 lines
1.3 KiB
C
#ifndef JEMALLOC_INTERNAL_RTREE_STRUCTS_H
|
|
#define JEMALLOC_INTERNAL_RTREE_STRUCTS_H
|
|
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/mutex_pool_structs.h"
|
|
|
|
struct rtree_node_elm_s {
|
|
atomic_p_t child; /* (rtree_{node,leaf}_elm_t *) */
|
|
};
|
|
|
|
struct rtree_leaf_elm_s {
|
|
#ifdef RTREE_LEAF_COMPACT
|
|
/*
|
|
* Single pointer-width field containing all three leaf element fields.
|
|
* For example, on a 64-bit x64 system with 48 significant virtual
|
|
* memory address bits, the index, extent, and slab fields are packed as
|
|
* such:
|
|
*
|
|
* x: index
|
|
* e: extent
|
|
* b: slab
|
|
*
|
|
* 00000000 xxxxxxxx eeeeeeee [...] eeeeeeee eeee000b
|
|
*/
|
|
atomic_p_t le_bits;
|
|
#else
|
|
atomic_p_t le_extent; /* (extent_t *) */
|
|
atomic_u_t le_szind; /* (szind_t) */
|
|
atomic_b_t le_slab; /* (bool) */
|
|
#endif
|
|
};
|
|
|
|
struct rtree_level_s {
|
|
/* Number of key bits distinguished by this level. */
|
|
unsigned bits;
|
|
/*
|
|
* Cumulative number of key bits distinguished by traversing to
|
|
* corresponding tree level.
|
|
*/
|
|
unsigned cumbits;
|
|
};
|
|
|
|
struct rtree_s {
|
|
malloc_mutex_t init_lock;
|
|
/* Number of elements based on rtree_levels[0].bits. */
|
|
#if RTREE_HEIGHT > 1
|
|
rtree_node_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)];
|
|
#else
|
|
rtree_leaf_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)];
|
|
#endif
|
|
};
|
|
|
|
#endif /* JEMALLOC_INTERNAL_RTREE_STRUCTS_H */
|