project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
faec7219b2
server-skynet-source-3rd-je.../include/jemalloc/internal/rtree.h

525 lines
17 KiB
C
Raw Normal View History

Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
#ifndef JEMALLOC_INTERNAL_RTREE_H
#define JEMALLOC_INTERNAL_RTREE_H
Port to Mac OS X. Add Mac OS X support, based in large part on the OS X support in Mozilla's version of jemalloc.
2010-09-06 01:35:13 +08:00
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/rtree_tsd.h"
Hide size class computation behind a layer of indirection. This class removes almost all the dependencies on size_classes.h, accessing the data there only via the new module sc.h, which does not depend on any configuration options. In a subsequent commit, we'll remove the configure-time size class computations, doing them at boot time, instead.
2017-12-15 04:46:39 +08:00
#include "jemalloc/internal/sc.h"
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
#include "jemalloc/internal/tsd.h"
/*
* This radix tree implementation is tailored to the singular purpose of
* associating metadata with extents that are currently owned by jemalloc.
*
*******************************************************************************
*/
/* Number of high insignificant bits. */
#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR)
/* Number of low insigificant bits. */
#define RTREE_NLIB LG_PAGE
/* Number of significant bits. */
#define RTREE_NSB (LG_VADDR - RTREE_NLIB)
/* Number of levels in radix tree. */
#if RTREE_NSB <= 10
# define RTREE_HEIGHT 1
#elif RTREE_NSB <= 36
# define RTREE_HEIGHT 2
#elif RTREE_NSB <= 52
# define RTREE_HEIGHT 3
#else
# error Unsupported number of significant virtual address bits
#endif
/* Use compact leaf representation if virtual address encoding allows. */
Hide size class computation behind a layer of indirection. This class removes almost all the dependencies on size_classes.h, accessing the data there only via the new module sc.h, which does not depend on any configuration options. In a subsequent commit, we'll remove the configure-time size class computations, doing them at boot time, instead.
2017-12-15 04:46:39 +08:00
#if RTREE_NHIB >= LG_CEIL(SC_NSIZES)
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
# define RTREE_LEAF_COMPACT
#endif
/* Needed for initialization only. */
#define RTREE_LEAFKEY_INVALID ((uintptr_t)1)
typedef struct rtree_node_elm_s rtree_node_elm_t;
struct rtree_node_elm_s {
atomic_p_t child; /* (rtree_{node,leaf}_elm_t *) */
};
struct rtree_leaf_elm_s {
#ifdef RTREE_LEAF_COMPACT
/*
* Single pointer-width field containing all three leaf element fields.
* For example, on a 64-bit x64 system with 48 significant virtual
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
* memory address bits, the index, edata, and slab fields are packed as
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
* such:
*
* x: index
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
* e: edata
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
* b: slab
*
* 00000000 xxxxxxxx eeeeeeee [...] eeeeeeee eeee000b
*/
atomic_p_t le_bits;
#else
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
atomic_p_t le_edata; /* (edata_t *) */
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
atomic_u_t le_szind; /* (szind_t) */
atomic_b_t le_slab; /* (bool) */
#endif
};
typedef struct rtree_level_s rtree_level_t;
struct rtree_level_s {
/* Number of key bits distinguished by this level. */
unsigned bits;
/*
* Cumulative number of key bits distinguished by traversing to
* corresponding tree level.
*/
unsigned cumbits;
};
typedef struct rtree_s rtree_t;
struct rtree_s {
Rtree: take the base allocator as a parameter. This facilitates better testing by avoiding mixing of the "real" base with the base used by the rtree under test.
2020-02-18 05:11:10 +08:00
base_t *base;
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
malloc_mutex_t init_lock;
/* Number of elements based on rtree_levels[0].bits. */
#if RTREE_HEIGHT > 1
rtree_node_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)];
#else
rtree_leaf_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)];
#endif
};
/*
* Split the bits into one to three partitions depending on number of
* significant bits. It the number of bits does not divide evenly into the
* number of levels, place one remainder bit per level starting at the leaf
* level.
*/
static const rtree_level_t rtree_levels[] = {
#if RTREE_HEIGHT == 1
{RTREE_NSB, RTREE_NHIB + RTREE_NSB}
#elif RTREE_HEIGHT == 2
{RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2},
{RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB}
#elif RTREE_HEIGHT == 3
{RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3},
{RTREE_NSB/3 + RTREE_NSB%3/2,
RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2},
{RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB}
#else
# error Unsupported rtree height
#endif
};
Rtree: take the base allocator as a parameter. This facilitates better testing by avoiding mixing of the "real" base with the base used by the rtree under test.
2020-02-18 05:11:10 +08:00
bool rtree_new(rtree_t *rtree, base_t *base, bool zeroed);
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,
rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
Header refactoring: unify spin.h and move it out of the catch-all.
2017-04-18 07:35:04 +08:00
Replace rtree path cache with LRU cache. Rework rtree_ctx_t to encapsulate an rtree leaf LRU lookup cache rather than a single-path element lookup cache. The replacement is logically much simpler, as well as slightly faster in the fast path case and less prone to degraded performance during non-trivial sequences of lookups.
2017-02-04 12:21:56 +08:00
JEMALLOC_ALWAYS_INLINE uintptr_t
Determine rtree levels at compile time. Rather than dynamically building a table to aid per level computations, define a constant table at compile time. Omit both high and low insignificant bits. Use one to three tree levels, depending on the number of significant bits.
2017-02-07 05:17:12 +08:00
rtree_leafkey(uintptr_t key) {
Replace rtree path cache with LRU cache. Rework rtree_ctx_t to encapsulate an rtree leaf LRU lookup cache rather than a single-path element lookup cache. The replacement is logically much simpler, as well as slightly faster in the fast path case and less prone to degraded performance during non-trivial sequences of lookups.
2017-02-04 12:21:56 +08:00
unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3);
Determine rtree levels at compile time. Rather than dynamically building a table to aid per level computations, define a constant table at compile time. Omit both high and low insignificant bits. Use one to three tree levels, depending on the number of significant bits.
2017-02-07 05:17:12 +08:00
unsigned cumbits = (rtree_levels[RTREE_HEIGHT-1].cumbits -
rtree_levels[RTREE_HEIGHT-1].bits);
Replace rtree path cache with LRU cache. Rework rtree_ctx_t to encapsulate an rtree leaf LRU lookup cache rather than a single-path element lookup cache. The replacement is logically much simpler, as well as slightly faster in the fast path case and less prone to degraded performance during non-trivial sequences of lookups.
2017-02-04 12:21:56 +08:00
unsigned maskbits = ptrbits - cumbits;
uintptr_t mask = ~((ZU(1) << maskbits) - 1);
return (key & mask);
Port to Mac OS X. Add Mac OS X support, based in large part on the OS X support in Mozilla's version of jemalloc.
2010-09-06 01:35:13 +08:00
}
Improve rtree cache with a two-level cache design. Two levels of rcache is implemented: a direct mapped cache as L1, combined with a LRU cache as L2. The L1 cache offers low cost on cache hit, but could suffer collision under circumstances. This is complemented by the L2 LRU cache, which is slower on cache access (overhead from linear search + reordering), but solves collison of L1 rather well.
2017-04-15 02:05:38 +08:00
JEMALLOC_ALWAYS_INLINE size_t
rtree_cache_direct_map(uintptr_t key) {
unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3);
unsigned cumbits = (rtree_levels[RTREE_HEIGHT-1].cumbits -
rtree_levels[RTREE_HEIGHT-1].bits);
unsigned maskbits = ptrbits - cumbits;
return (size_t)((key >> maskbits) & (RTREE_CTX_NCACHE - 1));
}
Always inline performance-critical rtree operations.
2016-03-24 07:14:41 +08:00
JEMALLOC_ALWAYS_INLINE uintptr_t
Determine rtree levels at compile time. Rather than dynamically building a table to aid per level computations, define a constant table at compile time. Omit both high and low insignificant bits. Use one to three tree levels, depending on the number of significant bits.
2017-02-07 05:17:12 +08:00
rtree_subkey(uintptr_t key, unsigned level) {
Replace rtree path cache with LRU cache. Rework rtree_ctx_t to encapsulate an rtree leaf LRU lookup cache rather than a single-path element lookup cache. The replacement is logically much simpler, as well as slightly faster in the fast path case and less prone to degraded performance during non-trivial sequences of lookups.
2017-02-04 12:21:56 +08:00
unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3);
Determine rtree levels at compile time. Rather than dynamically building a table to aid per level computations, define a constant table at compile time. Omit both high and low insignificant bits. Use one to three tree levels, depending on the number of significant bits.
2017-02-07 05:17:12 +08:00
unsigned cumbits = rtree_levels[level].cumbits;
Replace rtree path cache with LRU cache. Rework rtree_ctx_t to encapsulate an rtree leaf LRU lookup cache rather than a single-path element lookup cache. The replacement is logically much simpler, as well as slightly faster in the fast path case and less prone to degraded performance during non-trivial sequences of lookups.
2017-02-04 12:21:56 +08:00
unsigned shiftbits = ptrbits - cumbits;
Determine rtree levels at compile time. Rather than dynamically building a table to aid per level computations, define a constant table at compile time. Omit both high and low insignificant bits. Use one to three tree levels, depending on the number of significant bits.
2017-02-07 05:17:12 +08:00
unsigned maskbits = rtree_levels[level].bits;
Fix rtree_subkey() regression. Fix rtree_subkey() to use uintptr_t rather than unsigned for key bitmasking. This regression was introduced by 4a346f55939af4f200121cc4454089592d952f18 (Replace rtree path cache with LRU cache.).
2017-02-10 04:31:11 +08:00
uintptr_t mask = (ZU(1) << maskbits) - 1;
Replace rtree path cache with LRU cache. Rework rtree_ctx_t to encapsulate an rtree leaf LRU lookup cache rather than a single-path element lookup cache. The replacement is logically much simpler, as well as slightly faster in the fast path case and less prone to degraded performance during non-trivial sequences of lookups.
2017-02-04 12:21:56 +08:00
return ((key >> shiftbits) & mask);
Refactor rtree to be lock-free. Recent huge allocation refactoring associates huge allocations with arenas, but it remains necessary to quickly look up huge allocation metadata during reallocation/deallocation. A global radix tree remains a good solution to this problem, but locking would have become the primary bottleneck after (upcoming) migration of chunk management from global to per arena data structures. This lock-free implementation uses double-checked reads to traverse the tree, so that in the steady state, each read or write requires only a single atomic operation. This implementation also assures that no more than two tree levels actually exist, through a combination of careful virtual memory allocation which makes large sparse nodes cheap, and skipping the root node on x64 (possible because the top 16 bits are all 0 in practice).
2015-01-31 14:54:08 +08:00
}
Port to Mac OS X. Add Mac OS X support, based in large part on the OS X support in Mozilla's version of jemalloc.
2010-09-06 01:35:13 +08:00
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
/*
* Atomic getters.
*
* dependent: Reading a value on behalf of a pointer to a valid allocation
* is guaranteed to be a clean read even without synchronization,
* because the rtree update became visible in memory before the
* pointer came into existence.
* !dependent: An arbitrary read, e.g. on behalf of ivsalloc(), may not be
* dependent on a previous rtree write, which means a stale read
* could result if synchronization were omitted here.
*/
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
# ifdef RTREE_LEAF_COMPACT
JEMALLOC_ALWAYS_INLINE uintptr_t
Clean compilation -Wextra Before this commit jemalloc produced many warnings when compiled with -Wextra with both Clang and GCC. This commit fixes the issues raised by these warnings or suppresses them if they were spurious at least for the Clang and GCC versions covered by CI. This commit: * adds `JEMALLOC_DIAGNOSTIC` macros: `JEMALLOC_DIAGNOSTIC_{PUSH,POP}` are used to modify the stack of enabled diagnostics. The `JEMALLOC_DIAGNOSTIC_IGNORE_...` macros are used to ignore a concrete diagnostic. * adds `JEMALLOC_FALLTHROUGH` macro to explicitly state that falling through `case` labels in a `switch` statement is intended * Removes all UNUSED annotations on function parameters. The warning -Wunused-parameter is now disabled globally in `jemalloc_internal_macros.h` for all translation units that include that header. It is never re-enabled since that header cannot be included by users. * locally suppresses some -Wextra diagnostics: * `-Wmissing-field-initializer` is buggy in older Clang and GCC versions, where it does not understanding that, in C, `= {0}` is a common C idiom to initialize a struct to zero * `-Wtype-bounds` is suppressed in a particular situation where a generic macro, used in multiple different places, compares an unsigned integer for smaller than zero, which is always true. * `-Walloc-larger-than-size=` diagnostics warn when an allocation function is called with a size that is too large (out-of-range). These are suppressed in the parts of the tests where `jemalloc` explicitly does this to test that the allocation functions fail properly. * adds a new CI build bot that runs the log unit test on CI. Closes #1196 .
2018-05-03 17:40:53 +08:00
rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree,
rtree_leaf_elm_t *elm, bool dependent) {
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
return (uintptr_t)atomic_load_p(&elm->le_bits, dependent
rtree_leaf_elm_extent_write() --> rtree_leaf_elm_extent_lock_write() Refactor rtree_leaf_elm_extent_write() as rtree_leaf_elm_extent_lock_write(), so that whether the leaf element is currently acquired is separate from what lock state to write. This allows for a relaxed atomic read when releasing the lock.
2017-05-03 12:42:33 +08:00
? ATOMIC_RELAXED : ATOMIC_ACQUIRE);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
}
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
JEMALLOC_ALWAYS_INLINE edata_t *
rtree_leaf_elm_bits_edata_get(uintptr_t bits) {
ARM: Don't extend bit LG_VADDR to compute high address bits. In userspace ARM on Linux, zero-ing the high bits is the correct way to do this. This doesn't fix the fact that we currently set LG_VADDR to 48 on ARM, when in fact larger virtual address sizes are coming soon. We'll cross that bridge when we come to it.
2017-09-30 04:54:08 +08:00
# ifdef __aarch64__
/*
* aarch64 doesn't sign extend the highest virtual address bit to set
* the higher ones. Instead, the high bits gets zeroed.
*/
uintptr_t high_bit_mask = ((uintptr_t)1 << LG_VADDR) - 1;
/* Mask off the slab bit. */
uintptr_t low_bit_mask = ~(uintptr_t)1;
uintptr_t mask = high_bit_mask & low_bit_mask;
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
return (edata_t *)(bits & mask);
ARM: Don't extend bit LG_VADDR to compute high address bits. In userspace ARM on Linux, zero-ing the high bits is the correct way to do this. This doesn't fix the fact that we currently set LG_VADDR to 48 on ARM, when in fact larger virtual address sizes are coming soon. We'll cross that bridge when we come to it.
2017-09-30 04:54:08 +08:00
# else
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
/* Restore sign-extended high bits, mask slab bit. */
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
return (edata_t *)((uintptr_t)((intptr_t)(bits << RTREE_NHIB) >>
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
RTREE_NHIB) & ~((uintptr_t)0x1));
ARM: Don't extend bit LG_VADDR to compute high address bits. In userspace ARM on Linux, zero-ing the high bits is the correct way to do this. This doesn't fix the fact that we currently set LG_VADDR to 48 on ARM, when in fact larger virtual address sizes are coming soon. We'll cross that bridge when we come to it.
2017-09-30 04:54:08 +08:00
# endif
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
}
JEMALLOC_ALWAYS_INLINE szind_t
rtree_leaf_elm_bits_szind_get(uintptr_t bits) {
return (szind_t)(bits >> LG_VADDR);
}
JEMALLOC_ALWAYS_INLINE bool
rtree_leaf_elm_bits_slab_get(uintptr_t bits) {
return (bool)(bits & (uintptr_t)0x1);
}
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
# endif
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
JEMALLOC_ALWAYS_INLINE edata_t *
rtree_leaf_elm_edata_read(tsdn_t *tsdn, rtree_t *rtree,
Add UNUSED to avoid compiler warnings.
2018-04-17 03:08:27 +08:00
rtree_leaf_elm_t *elm, bool dependent) {
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#ifdef RTREE_LEAF_COMPACT
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
return rtree_leaf_elm_bits_edata_get(bits);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#else
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
edata_t *edata = (edata_t *)atomic_load_p(&elm->le_edata, dependent
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
? ATOMIC_RELAXED : ATOMIC_ACQUIRE);
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
return edata;
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#endif
Refactor rtree to be lock-free. Recent huge allocation refactoring associates huge allocations with arenas, but it remains necessary to quickly look up huge allocation metadata during reallocation/deallocation. A global radix tree remains a good solution to this problem, but locking would have become the primary bottleneck after (upcoming) migration of chunk management from global to per arena data structures. This lock-free implementation uses double-checked reads to traverse the tree, so that in the steady state, each read or write requires only a single atomic operation. This implementation also assures that no more than two tree levels actually exist, through a combination of careful virtual memory allocation which makes large sparse nodes cheap, and skipping the root node on x64 (possible because the top 16 bits are all 0 in practice).
2015-01-31 14:54:08 +08:00
}
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
JEMALLOC_ALWAYS_INLINE szind_t
Clean compilation -Wextra Before this commit jemalloc produced many warnings when compiled with -Wextra with both Clang and GCC. This commit fixes the issues raised by these warnings or suppresses them if they were spurious at least for the Clang and GCC versions covered by CI. This commit: * adds `JEMALLOC_DIAGNOSTIC` macros: `JEMALLOC_DIAGNOSTIC_{PUSH,POP}` are used to modify the stack of enabled diagnostics. The `JEMALLOC_DIAGNOSTIC_IGNORE_...` macros are used to ignore a concrete diagnostic. * adds `JEMALLOC_FALLTHROUGH` macro to explicitly state that falling through `case` labels in a `switch` statement is intended * Removes all UNUSED annotations on function parameters. The warning -Wunused-parameter is now disabled globally in `jemalloc_internal_macros.h` for all translation units that include that header. It is never re-enabled since that header cannot be included by users. * locally suppresses some -Wextra diagnostics: * `-Wmissing-field-initializer` is buggy in older Clang and GCC versions, where it does not understanding that, in C, `= {0}` is a common C idiom to initialize a struct to zero * `-Wtype-bounds` is suppressed in a particular situation where a generic macro, used in multiple different places, compares an unsigned integer for smaller than zero, which is always true. * `-Walloc-larger-than-size=` diagnostics warn when an allocation function is called with a size that is too large (out-of-range). These are suppressed in the parts of the tests where `jemalloc` explicitly does this to test that the allocation functions fail properly. * adds a new CI build bot that runs the log unit test on CI. Closes #1196 .
2018-05-03 17:40:53 +08:00
rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree,
Add UNUSED to avoid compiler warnings.
2018-04-17 03:08:27 +08:00
rtree_leaf_elm_t *elm, bool dependent) {
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#ifdef RTREE_LEAF_COMPACT
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
return rtree_leaf_elm_bits_szind_get(bits);
#else
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
return (szind_t)atomic_load_u(&elm->le_szind, dependent ? ATOMIC_RELAXED
: ATOMIC_ACQUIRE);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#endif
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
}
JEMALLOC_ALWAYS_INLINE bool
Clean compilation -Wextra Before this commit jemalloc produced many warnings when compiled with -Wextra with both Clang and GCC. This commit fixes the issues raised by these warnings or suppresses them if they were spurious at least for the Clang and GCC versions covered by CI. This commit: * adds `JEMALLOC_DIAGNOSTIC` macros: `JEMALLOC_DIAGNOSTIC_{PUSH,POP}` are used to modify the stack of enabled diagnostics. The `JEMALLOC_DIAGNOSTIC_IGNORE_...` macros are used to ignore a concrete diagnostic. * adds `JEMALLOC_FALLTHROUGH` macro to explicitly state that falling through `case` labels in a `switch` statement is intended * Removes all UNUSED annotations on function parameters. The warning -Wunused-parameter is now disabled globally in `jemalloc_internal_macros.h` for all translation units that include that header. It is never re-enabled since that header cannot be included by users. * locally suppresses some -Wextra diagnostics: * `-Wmissing-field-initializer` is buggy in older Clang and GCC versions, where it does not understanding that, in C, `= {0}` is a common C idiom to initialize a struct to zero * `-Wtype-bounds` is suppressed in a particular situation where a generic macro, used in multiple different places, compares an unsigned integer for smaller than zero, which is always true. * `-Walloc-larger-than-size=` diagnostics warn when an allocation function is called with a size that is too large (out-of-range). These are suppressed in the parts of the tests where `jemalloc` explicitly does this to test that the allocation functions fail properly. * adds a new CI build bot that runs the log unit test on CI. Closes #1196 .
2018-05-03 17:40:53 +08:00
rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree,
Add UNUSED to avoid compiler warnings.
2018-04-17 03:08:27 +08:00
rtree_leaf_elm_t *elm, bool dependent) {
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#ifdef RTREE_LEAF_COMPACT
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
return rtree_leaf_elm_bits_slab_get(bits);
#else
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
return atomic_load_b(&elm->le_slab, dependent ? ATOMIC_RELAXED :
ATOMIC_ACQUIRE);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#endif
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
}
Get rid of most of the various inline macros.
2017-04-22 00:37:34 +08:00
static inline void
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
rtree_leaf_elm_edata_write(tsdn_t *tsdn, rtree_t *rtree,
rtree_leaf_elm_t *elm, edata_t *edata) {
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#ifdef RTREE_LEAF_COMPACT
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, true);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
uintptr_t bits = ((uintptr_t)rtree_leaf_elm_bits_szind_get(old_bits) <<
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
LG_VADDR) | ((uintptr_t)edata & (((uintptr_t)0x1 << LG_VADDR) - 1))
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
| ((uintptr_t)rtree_leaf_elm_bits_slab_get(old_bits));
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
atomic_store_p(&elm->le_bits, (void *)bits, ATOMIC_RELEASE);
#else
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
atomic_store_p(&elm->le_edata, edata, ATOMIC_RELEASE);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#endif
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
}
Get rid of most of the various inline macros.
2017-04-22 00:37:34 +08:00
static inline void
Clean compilation -Wextra Before this commit jemalloc produced many warnings when compiled with -Wextra with both Clang and GCC. This commit fixes the issues raised by these warnings or suppresses them if they were spurious at least for the Clang and GCC versions covered by CI. This commit: * adds `JEMALLOC_DIAGNOSTIC` macros: `JEMALLOC_DIAGNOSTIC_{PUSH,POP}` are used to modify the stack of enabled diagnostics. The `JEMALLOC_DIAGNOSTIC_IGNORE_...` macros are used to ignore a concrete diagnostic. * adds `JEMALLOC_FALLTHROUGH` macro to explicitly state that falling through `case` labels in a `switch` statement is intended * Removes all UNUSED annotations on function parameters. The warning -Wunused-parameter is now disabled globally in `jemalloc_internal_macros.h` for all translation units that include that header. It is never re-enabled since that header cannot be included by users. * locally suppresses some -Wextra diagnostics: * `-Wmissing-field-initializer` is buggy in older Clang and GCC versions, where it does not understanding that, in C, `= {0}` is a common C idiom to initialize a struct to zero * `-Wtype-bounds` is suppressed in a particular situation where a generic macro, used in multiple different places, compares an unsigned integer for smaller than zero, which is always true. * `-Walloc-larger-than-size=` diagnostics warn when an allocation function is called with a size that is too large (out-of-range). These are suppressed in the parts of the tests where `jemalloc` explicitly does this to test that the allocation functions fail properly. * adds a new CI build bot that runs the log unit test on CI. Closes #1196 .
2018-05-03 17:40:53 +08:00
rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree,
Add UNUSED to avoid compiler warnings.
2018-04-17 03:08:27 +08:00
rtree_leaf_elm_t *elm, szind_t szind) {
Hide size class computation behind a layer of indirection. This class removes almost all the dependencies on size_classes.h, accessing the data there only via the new module sc.h, which does not depend on any configuration options. In a subsequent commit, we'll remove the configure-time size class computations, doing them at boot time, instead.
2017-12-15 04:46:39 +08:00
assert(szind <= SC_NSIZES);
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#ifdef RTREE_LEAF_COMPACT
uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm,
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
true);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
uintptr_t bits = ((uintptr_t)szind << LG_VADDR) |
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
((uintptr_t)rtree_leaf_elm_bits_edata_get(old_bits) &
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
(((uintptr_t)0x1 << LG_VADDR) - 1)) |
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
((uintptr_t)rtree_leaf_elm_bits_slab_get(old_bits));
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
atomic_store_p(&elm->le_bits, (void *)bits, ATOMIC_RELEASE);
#else
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
atomic_store_u(&elm->le_szind, szind, ATOMIC_RELEASE);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#endif
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
}
Get rid of most of the various inline macros.
2017-04-22 00:37:34 +08:00
static inline void
Clean compilation -Wextra Before this commit jemalloc produced many warnings when compiled with -Wextra with both Clang and GCC. This commit fixes the issues raised by these warnings or suppresses them if they were spurious at least for the Clang and GCC versions covered by CI. This commit: * adds `JEMALLOC_DIAGNOSTIC` macros: `JEMALLOC_DIAGNOSTIC_{PUSH,POP}` are used to modify the stack of enabled diagnostics. The `JEMALLOC_DIAGNOSTIC_IGNORE_...` macros are used to ignore a concrete diagnostic. * adds `JEMALLOC_FALLTHROUGH` macro to explicitly state that falling through `case` labels in a `switch` statement is intended * Removes all UNUSED annotations on function parameters. The warning -Wunused-parameter is now disabled globally in `jemalloc_internal_macros.h` for all translation units that include that header. It is never re-enabled since that header cannot be included by users. * locally suppresses some -Wextra diagnostics: * `-Wmissing-field-initializer` is buggy in older Clang and GCC versions, where it does not understanding that, in C, `= {0}` is a common C idiom to initialize a struct to zero * `-Wtype-bounds` is suppressed in a particular situation where a generic macro, used in multiple different places, compares an unsigned integer for smaller than zero, which is always true. * `-Walloc-larger-than-size=` diagnostics warn when an allocation function is called with a size that is too large (out-of-range). These are suppressed in the parts of the tests where `jemalloc` explicitly does this to test that the allocation functions fail properly. * adds a new CI build bot that runs the log unit test on CI. Closes #1196 .
2018-05-03 17:40:53 +08:00
rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree,
Add UNUSED to avoid compiler warnings.
2018-04-17 03:08:27 +08:00
rtree_leaf_elm_t *elm, bool slab) {
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#ifdef RTREE_LEAF_COMPACT
uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm,
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
true);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
uintptr_t bits = ((uintptr_t)rtree_leaf_elm_bits_szind_get(old_bits) <<
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
LG_VADDR) | ((uintptr_t)rtree_leaf_elm_bits_edata_get(old_bits) &
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
(((uintptr_t)0x1 << LG_VADDR) - 1)) | ((uintptr_t)slab);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
atomic_store_p(&elm->le_bits, (void *)bits, ATOMIC_RELEASE);
#else
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
atomic_store_b(&elm->le_slab, slab, ATOMIC_RELEASE);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#endif
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
}
Get rid of most of the various inline macros.
2017-04-22 00:37:34 +08:00
static inline void
Clean compilation -Wextra Before this commit jemalloc produced many warnings when compiled with -Wextra with both Clang and GCC. This commit fixes the issues raised by these warnings or suppresses them if they were spurious at least for the Clang and GCC versions covered by CI. This commit: * adds `JEMALLOC_DIAGNOSTIC` macros: `JEMALLOC_DIAGNOSTIC_{PUSH,POP}` are used to modify the stack of enabled diagnostics. The `JEMALLOC_DIAGNOSTIC_IGNORE_...` macros are used to ignore a concrete diagnostic. * adds `JEMALLOC_FALLTHROUGH` macro to explicitly state that falling through `case` labels in a `switch` statement is intended * Removes all UNUSED annotations on function parameters. The warning -Wunused-parameter is now disabled globally in `jemalloc_internal_macros.h` for all translation units that include that header. It is never re-enabled since that header cannot be included by users. * locally suppresses some -Wextra diagnostics: * `-Wmissing-field-initializer` is buggy in older Clang and GCC versions, where it does not understanding that, in C, `= {0}` is a common C idiom to initialize a struct to zero * `-Wtype-bounds` is suppressed in a particular situation where a generic macro, used in multiple different places, compares an unsigned integer for smaller than zero, which is always true. * `-Walloc-larger-than-size=` diagnostics warn when an allocation function is called with a size that is too large (out-of-range). These are suppressed in the parts of the tests where `jemalloc` explicitly does this to test that the allocation functions fail properly. * adds a new CI build bot that runs the log unit test on CI. Closes #1196 .
2018-05-03 17:40:53 +08:00
rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree,
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
rtree_leaf_elm_t *elm, edata_t *edata, szind_t szind, bool slab) {
Fix a race in rtree_szind_slab_update() for RTREE_LEAF_COMPACT.
2017-03-27 19:08:51 +08:00
#ifdef RTREE_LEAF_COMPACT
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
uintptr_t bits = ((uintptr_t)szind << LG_VADDR) |
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
((uintptr_t)edata & (((uintptr_t)0x1 << LG_VADDR) - 1)) |
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
((uintptr_t)slab);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
atomic_store_p(&elm->le_bits, (void *)bits, ATOMIC_RELEASE);
#else
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
rtree_leaf_elm_slab_write(tsdn, rtree, elm, slab);
rtree_leaf_elm_szind_write(tsdn, rtree, elm, szind);
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
/*
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
* Write edata last, since the element is atomically considered valid
* as soon as the edata field is non-NULL.
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
*/
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
rtree_leaf_elm_edata_write(tsdn, rtree, elm, edata);
Implement compact rtree leaf element representation. If a single virtual adddress pointer has enough unused bits to pack {szind_t, extent_t *, bool, bool}, use a single pointer-sized field in each rtree leaf element, rather than using three separate fields. This has little impact on access speed (fewer loads/stores, but more bit twiddling), except that denser representation increases TLB effectiveness.
2017-03-21 07:38:21 +08:00
#endif
Refactor rtree to be lock-free. Recent huge allocation refactoring associates huge allocations with arenas, but it remains necessary to quickly look up huge allocation metadata during reallocation/deallocation. A global radix tree remains a good solution to this problem, but locking would have become the primary bottleneck after (upcoming) migration of chunk management from global to per arena data structures. This lock-free implementation uses double-checked reads to traverse the tree, so that in the steady state, each read or write requires only a single atomic operation. This implementation also assures that no more than two tree levels actually exist, through a combination of careful virtual memory allocation which makes large sparse nodes cheap, and skipping the root node on x64 (possible because the top 16 bits are all 0 in practice).
2015-01-31 14:54:08 +08:00
}
Get rid of most of the various inline macros.
2017-04-22 00:37:34 +08:00
static inline void
Fix a race in rtree_szind_slab_update() for RTREE_LEAF_COMPACT.
2017-03-27 19:08:51 +08:00
rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree,
rtree_leaf_elm_t *elm, szind_t szind, bool slab) {
Hide size class computation behind a layer of indirection. This class removes almost all the dependencies on size_classes.h, accessing the data there only via the new module sc.h, which does not depend on any configuration options. In a subsequent commit, we'll remove the configure-time size class computations, doing them at boot time, instead.
2017-12-15 04:46:39 +08:00
assert(!slab || szind < SC_NBINS);
Fix a race in rtree_szind_slab_update() for RTREE_LEAF_COMPACT.
2017-03-27 19:08:51 +08:00
/*
* The caller implicitly assures that it is the only writer to the szind
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
* and slab fields, and that the edata field cannot currently change.
Fix a race in rtree_szind_slab_update() for RTREE_LEAF_COMPACT.
2017-03-27 19:08:51 +08:00
*/
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
rtree_leaf_elm_slab_write(tsdn, rtree, elm, slab);
rtree_leaf_elm_szind_write(tsdn, rtree, elm, szind);
Fix a race in rtree_szind_slab_update() for RTREE_LEAF_COMPACT.
2017-03-27 19:08:51 +08:00
}
Split rtree_elm_t into rtree_{node,leaf}_elm_t. This allows leaf elements to differ in size from internal node elements. In principle it would be more correct to use a different type for each level of the tree, but due to implementation details related to atomic operations, we use casts anyway, thus counteracting the value of additional type correctness. Furthermore, such a scheme would require function code generation (via cpp macros), as well as either unwieldy type names for leaves or type aliases, e.g. typedef struct rtree_elm_d2_s rtree_leaf_elm_t; This alternate strategy would be more correct, and with less code duplication, but probably not worth the complexity.
2017-03-17 00:46:42 +08:00
JEMALLOC_ALWAYS_INLINE rtree_leaf_elm_t *
rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
Update brace style. Add braces around single-line blocks, and remove line breaks before function-opening braces. This resolves #537.
2017-01-16 08:56:30 +08:00
uintptr_t key, bool dependent, bool init_missing) {
Remove rtree support for 0 (NULL) keys. NULL can never actually be inserted in practice, and removing support allows a branch to be removed from the fast path.
2017-02-05 18:50:59 +08:00
assert(key != 0);
Add element acquire/release capabilities to rtree. This makes it possible to acquire short-term "ownership" of rtree elements so that it is possible to read an extent pointer *and* read the extent's contents with a guarantee that the element will not be modified until the ownership is released. This is intended as a mechanism for resolving rtree read/write races rather than as a way to lock extents.
2016-03-28 18:06:35 +08:00
assert(!dependent || !init_missing);
Refactor rtree to be lock-free. Recent huge allocation refactoring associates huge allocations with arenas, but it remains necessary to quickly look up huge allocation metadata during reallocation/deallocation. A global radix tree remains a good solution to this problem, but locking would have become the primary bottleneck after (upcoming) migration of chunk management from global to per arena data structures. This lock-free implementation uses double-checked reads to traverse the tree, so that in the steady state, each read or write requires only a single atomic operation. This implementation also assures that no more than two tree levels actually exist, through a combination of careful virtual memory allocation which makes large sparse nodes cheap, and skipping the root node on x64 (possible because the top 16 bits are all 0 in practice).
2015-01-31 14:54:08 +08:00
Improve rtree cache with a two-level cache design. Two levels of rcache is implemented: a direct mapped cache as L1, combined with a LRU cache as L2. The L1 cache offers low cost on cache hit, but could suffer collision under circumstances. This is complemented by the L2 LRU cache, which is slower on cache access (overhead from linear search + reordering), but solves collison of L1 rather well.
2017-04-15 02:05:38 +08:00
size_t slot = rtree_cache_direct_map(key);
Remove rtree support for 0 (NULL) keys. NULL can never actually be inserted in practice, and removing support allows a branch to be removed from the fast path.
2017-02-05 18:50:59 +08:00
uintptr_t leafkey = rtree_leafkey(key);
Integrate auto tcache into TSD. The embedded tcache is initialized upon tsd initialization. The avail arrays for the tbins will be allocated / deallocated accordingly during init / cleanup. With this change, the pointer to the auto tcache will always be available, as long as we have access to the TSD. tcache_available() (called in tcache_get()) is provided to check if we should use tcache.
2017-03-28 12:50:38 +08:00
assert(leafkey != RTREE_LEAFKEY_INVALID);
Improve rtree cache with a two-level cache design. Two levels of rcache is implemented: a direct mapped cache as L1, combined with a LRU cache as L2. The L1 cache offers low cost on cache hit, but could suffer collision under circumstances. This is complemented by the L2 LRU cache, which is slower on cache access (overhead from linear search + reordering), but solves collison of L1 rather well.
2017-04-15 02:05:38 +08:00
/* Fast path: L1 direct mapped cache. */
if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) {
rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf;
assert(leaf != NULL);
uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1);
return &leaf[subkey];
}
/*
* Search the L2 LRU cache. On hit, swap the matching element into the
* slot in L1 cache, and move the position in L2 up by 1.
*/
#define RTREE_CACHE_CHECK_L2(i) do { \
if (likely(rtree_ctx->l2_cache[i].leafkey == leafkey)) { \
rtree_leaf_elm_t *leaf = rtree_ctx->l2_cache[i].leaf; \
Remove the leafkey NULL check in leaf_elm_lookup.
2017-03-29 08:14:43 +08:00
assert(leaf != NULL); \
if (i > 0) { \
Simplify rtree cache replacement policy. To avoid memmove on free() fast path, simplify the cache replacement policy to only bubble up the cache hit element by 1.
2017-03-28 02:48:39 +08:00
/* Bubble up by one. */ \
Improve rtree cache with a two-level cache design. Two levels of rcache is implemented: a direct mapped cache as L1, combined with a LRU cache as L2. The L1 cache offers low cost on cache hit, but could suffer collision under circumstances. This is complemented by the L2 LRU cache, which is slower on cache access (overhead from linear search + reordering), but solves collison of L1 rather well.
2017-04-15 02:05:38 +08:00
rtree_ctx->l2_cache[i].leafkey = \
rtree_ctx->l2_cache[i - 1].leafkey; \
rtree_ctx->l2_cache[i].leaf = \
rtree_ctx->l2_cache[i - 1].leaf; \
rtree_ctx->l2_cache[i - 1].leafkey = \
rtree_ctx->cache[slot].leafkey; \
rtree_ctx->l2_cache[i - 1].leaf = \
rtree_ctx->cache[slot].leaf; \
} else { \
rtree_ctx->l2_cache[0].leafkey = \
rtree_ctx->cache[slot].leafkey; \
rtree_ctx->l2_cache[0].leaf = \
rtree_ctx->cache[slot].leaf; \
Replace rtree path cache with LRU cache. Rework rtree_ctx_t to encapsulate an rtree leaf LRU lookup cache rather than a single-path element lookup cache. The replacement is logically much simpler, as well as slightly faster in the fast path case and less prone to degraded performance during non-trivial sequences of lookups.
2017-02-04 12:21:56 +08:00
} \
Improve rtree cache with a two-level cache design. Two levels of rcache is implemented: a direct mapped cache as L1, combined with a LRU cache as L2. The L1 cache offers low cost on cache hit, but could suffer collision under circumstances. This is complemented by the L2 LRU cache, which is slower on cache access (overhead from linear search + reordering), but solves collison of L1 rather well.
2017-04-15 02:05:38 +08:00
rtree_ctx->cache[slot].leafkey = leafkey; \
rtree_ctx->cache[slot].leaf = leaf; \
Remove the leafkey NULL check in leaf_elm_lookup.
2017-03-29 08:14:43 +08:00
uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); \
return &leaf[subkey]; \
Remove rtree support for 0 (NULL) keys. NULL can never actually be inserted in practice, and removing support allows a branch to be removed from the fast path.
2017-02-05 18:50:59 +08:00
} \
Replace rtree path cache with LRU cache. Rework rtree_ctx_t to encapsulate an rtree leaf LRU lookup cache rather than a single-path element lookup cache. The replacement is logically much simpler, as well as slightly faster in the fast path case and less prone to degraded performance during non-trivial sequences of lookups.
2017-02-04 12:21:56 +08:00
} while (0)
Simplify rtree cache replacement policy. To avoid memmove on free() fast path, simplify the cache replacement policy to only bubble up the cache hit element by 1.
2017-03-28 02:48:39 +08:00
/* Check the first cache entry. */
Improve rtree cache with a two-level cache design. Two levels of rcache is implemented: a direct mapped cache as L1, combined with a LRU cache as L2. The L1 cache offers low cost on cache hit, but could suffer collision under circumstances. This is complemented by the L2 LRU cache, which is slower on cache access (overhead from linear search + reordering), but solves collison of L1 rather well.
2017-04-15 02:05:38 +08:00
RTREE_CACHE_CHECK_L2(0);
/* Search the remaining cache elements. */
for (unsigned i = 1; i < RTREE_CTX_NCACHE_L2; i++) {
RTREE_CACHE_CHECK_L2(i);
Remove rtree support for 0 (NULL) keys. NULL can never actually be inserted in practice, and removing support allows a branch to be removed from the fast path.
2017-02-05 18:50:59 +08:00
}
Improve rtree cache with a two-level cache design. Two levels of rcache is implemented: a direct mapped cache as L1, combined with a LRU cache as L2. The L1 cache offers low cost on cache hit, but could suffer collision under circumstances. This is complemented by the L2 LRU cache, which is slower on cache access (overhead from linear search + reordering), but solves collison of L1 rather well.
2017-04-15 02:05:38 +08:00
#undef RTREE_CACHE_CHECK_L2
Refactor rtree to be lock-free. Recent huge allocation refactoring associates huge allocations with arenas, but it remains necessary to quickly look up huge allocation metadata during reallocation/deallocation. A global radix tree remains a good solution to this problem, but locking would have become the primary bottleneck after (upcoming) migration of chunk management from global to per arena data structures. This lock-free implementation uses double-checked reads to traverse the tree, so that in the steady state, each read or write requires only a single atomic operation. This implementation also assures that no more than two tree levels actually exist, through a combination of careful virtual memory allocation which makes large sparse nodes cheap, and skipping the root node on x64 (possible because the top 16 bits are all 0 in practice).
2015-01-31 14:54:08 +08:00
Split rtree_elm_t into rtree_{node,leaf}_elm_t. This allows leaf elements to differ in size from internal node elements. In principle it would be more correct to use a different type for each level of the tree, but due to implementation details related to atomic operations, we use casts anyway, thus counteracting the value of additional type correctness. Furthermore, such a scheme would require function code generation (via cpp macros), as well as either unwieldy type names for leaves or type aliases, e.g. typedef struct rtree_elm_d2_s rtree_leaf_elm_t; This alternate strategy would be more correct, and with less code duplication, but probably not worth the complexity.
2017-03-17 00:46:42 +08:00
return rtree_leaf_elm_lookup_hard(tsdn, rtree, rtree_ctx, key,
dependent, init_missing);
Refactor rtree to be lock-free. Recent huge allocation refactoring associates huge allocations with arenas, but it remains necessary to quickly look up huge allocation metadata during reallocation/deallocation. A global radix tree remains a good solution to this problem, but locking would have become the primary bottleneck after (upcoming) migration of chunk management from global to per arena data structures. This lock-free implementation uses double-checked reads to traverse the tree, so that in the steady state, each read or write requires only a single atomic operation. This implementation also assures that no more than two tree levels actually exist, through a combination of careful virtual memory allocation which makes large sparse nodes cheap, and skipping the root node on x64 (possible because the top 16 bits are all 0 in practice).
2015-01-31 14:54:08 +08:00
}
Port to Mac OS X. Add Mac OS X support, based in large part on the OS X support in Mozilla's version of jemalloc.
2010-09-06 01:35:13 +08:00
Get rid of most of the various inline macros.
2017-04-22 00:37:34 +08:00
static inline bool
Add rtree lookup path caching. rtree-based extent lookups remain more expensive than chunk-based run lookups, but with this optimization the fast path slowdown is ~3 CPU cycles per metadata lookup (on Intel Core i7-4980HQ), versus ~11 cycles prior. The path caching speedup tends to degrade gracefully unless allocated memory is spread far apart (as is the case when using a mixture of sbrk() and mmap()).
2016-06-03 09:43:10 +08:00
rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
edata_t *edata, szind_t szind, bool slab) {
/* Use rtree_clear() to set the edata to NULL. */
assert(edata != NULL);
Port to Mac OS X. Add Mac OS X support, based in large part on the OS X support in Mozilla's version of jemalloc.
2010-09-06 01:35:13 +08:00
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, false, true);
Update brace style. Add braces around single-line blocks, and remove line breaks before function-opening braces. This resolves #537.
2017-01-16 08:56:30 +08:00
if (elm == NULL) {
Remove extraneous parens around return arguments. This resolves #540.
2017-01-20 10:15:45 +08:00
return true;
Update brace style. Add braces around single-line blocks, and remove line breaks before function-opening braces. This resolves #537.
2017-01-16 08:56:30 +08:00
}
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
assert(rtree_leaf_elm_edata_read(tsdn, rtree, elm, false) == NULL);
rtree_leaf_elm_write(tsdn, rtree, elm, edata, szind, slab);
Add element acquire/release capabilities to rtree. This makes it possible to acquire short-term "ownership" of rtree elements so that it is possible to read an extent pointer *and* read the extent's contents with a guarantee that the element will not be modified until the ownership is released. This is intended as a mechanism for resolving rtree read/write races rather than as a way to lock extents.
2016-03-28 18:06:35 +08:00
Remove extraneous parens around return arguments. This resolves #540.
2017-01-20 10:15:45 +08:00
return false;
Add element acquire/release capabilities to rtree. This makes it possible to acquire short-term "ownership" of rtree elements so that it is possible to read an extent pointer *and* read the extent's contents with a guarantee that the element will not be modified until the ownership is released. This is intended as a mechanism for resolving rtree read/write races rather than as a way to lock extents.
2016-03-28 18:06:35 +08:00
}
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
JEMALLOC_ALWAYS_INLINE rtree_leaf_elm_t *
Add rtree lookup path caching. rtree-based extent lookups remain more expensive than chunk-based run lookups, but with this optimization the fast path slowdown is ~3 CPU cycles per metadata lookup (on Intel Core i7-4980HQ), versus ~11 cycles prior. The path caching speedup tends to degrade gracefully unless allocated memory is spread far apart (as is the case when using a mixture of sbrk() and mmap()).
2016-06-03 09:43:10 +08:00
rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
Update brace style. Add braces around single-line blocks, and remove line breaks before function-opening braces. This resolves #537.
2017-01-16 08:56:30 +08:00
bool dependent) {
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, dependent, false);
if (!dependent && elm == NULL) {
return NULL;
}
assert(elm != NULL);
return elm;
}
Add element acquire/release capabilities to rtree. This makes it possible to acquire short-term "ownership" of rtree elements so that it is possible to read an extent pointer *and* read the extent's contents with a guarantee that the element will not be modified until the ownership is released. This is intended as a mechanism for resolving rtree read/write races rather than as a way to lock extents.
2016-03-28 18:06:35 +08:00
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
JEMALLOC_ALWAYS_INLINE edata_t *
rtree_edata_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
uintptr_t key, bool dependent) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
dependent);
Optimize a branch out of rtree_read() if !dependent.
2017-02-04 12:17:47 +08:00
if (!dependent && elm == NULL) {
Remove extraneous parens around return arguments. This resolves #540.
2017-01-20 10:15:45 +08:00
return NULL;
Update brace style. Add braces around single-line blocks, and remove line breaks before function-opening braces. This resolves #537.
2017-01-16 08:56:30 +08:00
}
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
return rtree_leaf_elm_edata_read(tsdn, rtree, elm, dependent);
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
}
JEMALLOC_ALWAYS_INLINE szind_t
rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, bool dependent) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
dependent);
if (!dependent && elm == NULL) {
Hide size class computation behind a layer of indirection. This class removes almost all the dependencies on size_classes.h, accessing the data there only via the new module sc.h, which does not depend on any configuration options. In a subsequent commit, we'll remove the configure-time size class computations, doing them at boot time, instead.
2017-12-15 04:46:39 +08:00
return SC_NSIZES;
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
}
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
return rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent);
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
}
/*
* rtree_slab_read() is intentionally omitted because slab is always read in
* conjunction with szind, which makes rtree_szind_slab_read() a better choice.
*/
JEMALLOC_ALWAYS_INLINE bool
Emap: Remove direct access to emap internals. In the process, we do a few local cleanups and optimizations. In particular, the size safety check on tcache flush no longer does a redundant load.
2020-02-07 05:16:07 +08:00
rtree_edata_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree,
rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, edata_t **r_edata,
szind_t *r_szind, bool *r_slab) {
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
dependent);
if (!dependent && elm == NULL) {
return true;
}
Emap: Remove direct access to emap internals. In the process, we do a few local cleanups and optimizations. In particular, the size safety check on tcache flush no longer does a redundant load.
2020-02-07 05:16:07 +08:00
#ifdef RTREE_LEAF_COMPACT
uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);
*r_edata = rtree_leaf_elm_bits_edata_get(bits);
*r_szind = rtree_leaf_elm_bits_szind_get(bits);
*r_slab = rtree_leaf_elm_bits_slab_get(bits);
#else
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
*r_edata = rtree_leaf_elm_edata_read(tsdn, rtree, elm, dependent);
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
*r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent);
Emap: Remove direct access to emap internals. In the process, we do a few local cleanups and optimizations. In particular, the size safety check on tcache flush no longer does a redundant load.
2020-02-07 05:16:07 +08:00
*r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, dependent);
#endif
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
return false;
}
Add element acquire/release capabilities to rtree. This makes it possible to acquire short-term "ownership" of rtree elements so that it is possible to read an extent pointer *and* read the extent's contents with a guarantee that the element will not be modified until the ownership is released. This is intended as a mechanism for resolving rtree read/write races rather than as a way to lock extents.
2016-03-28 18:06:35 +08:00
rtree: add rtree_szind_slab_read_fast For a free fastpath, we want something that will not make additional calls. Assume most free() calls will hit the L1 cache, and use a custom rtree function for this. Additionally, roll the ptr=NULL check in to the rtree cache check.
2018-10-19 03:51:54 +08:00
/*
* Try to read szind_slab from the L1 cache. Returns true on a hit,
* and fills in r_szind and r_slab. Otherwise returns false.
*
* Key is allowed to be NULL in order to save an extra branch on the
* fastpath. returns false in this case.
*/
JEMALLOC_ALWAYS_INLINE bool
rtree_szind_slab_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, szind_t *r_szind, bool *r_slab) {
rtree_leaf_elm_t *elm;
size_t slot = rtree_cache_direct_map(key);
uintptr_t leafkey = rtree_leafkey(key);
assert(leafkey != RTREE_LEAFKEY_INVALID);
if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) {
rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf;
assert(leaf != NULL);
uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1);
elm = &leaf[subkey];
#ifdef RTREE_LEAF_COMPACT
uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree,
elm, true);
*r_szind = rtree_leaf_elm_bits_szind_get(bits);
*r_slab = rtree_leaf_elm_bits_slab_get(bits);
#else
*r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, true);
*r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, true);
#endif
return true;
} else {
return false;
}
}
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
JEMALLOC_ALWAYS_INLINE bool
rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
dependent);
if (!dependent && elm == NULL) {
return true;
}
Consolidate the two memory loads in rtree_szind_slab_read(). szind and slab bits are read on fast path, where compiler generated two memory loads separately for them before this diff. Manually operate on the bits to avoid the extra memory load.
2018-04-06 05:37:17 +08:00
#ifdef RTREE_LEAF_COMPACT
uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);
*r_szind = rtree_leaf_elm_bits_szind_get(bits);
*r_slab = rtree_leaf_elm_bits_slab_get(bits);
#else
Protect the rtree/extent interactions with a mutex pool. Instead of embedding a lock bit in rtree leaf elements, we associate extents with a small set of mutexes. This gets us two things: - We can use the system mutexes. This (hypothetically) protects us from priority inversion, and lets us stop doing a backoff/sleep loop, instead opting for precise wakeups from the mutex. - Cuts down on the number of mutex acquisitions we have to do (from 4 in the worst case to two). We end up simplifying most of the rtree code (which no longer has to deal with locking or concurrency at all), at the cost of additional complexity in the extent code: since the mutex protecting the rtree leaf elements is determined by reading the extent out of those elements, the initial read is racy, so that we may acquire an out of date mutex. We re-check the extent in the leaf after acquiring the mutex to protect us from this race.
2017-05-16 05:23:51 +08:00
*r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent);
*r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, dependent);
Consolidate the two memory loads in rtree_szind_slab_read(). szind and slab bits are read on fast path, where compiler generated two memory loads separately for them before this diff. Manually operate on the bits to avoid the extra memory load.
2018-04-06 05:37:17 +08:00
#endif
Incorporate szind/slab into rtree leaves. Expand and restructure the rtree API such that all common operations can be achieved with minimal work, regardless of whether the rtree leaf fields are independent versus packed into a single atomic pointer.
2017-03-17 08:57:52 +08:00
return false;
}
Get rid of most of the various inline macros.
2017-04-22 00:37:34 +08:00
static inline void
Fix a race in rtree_szind_slab_update() for RTREE_LEAF_COMPACT.
2017-03-27 19:08:51 +08:00
rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, szind_t szind, bool slab) {
Hide size class computation behind a layer of indirection. This class removes almost all the dependencies on size_classes.h, accessing the data there only via the new module sc.h, which does not depend on any configuration options. In a subsequent commit, we'll remove the configure-time size class computations, doing them at boot time, instead.
2017-12-15 04:46:39 +08:00
assert(!slab || szind < SC_NBINS);
Fix a race in rtree_szind_slab_update() for RTREE_LEAF_COMPACT.
2017-03-27 19:08:51 +08:00
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true);
rtree_leaf_elm_szind_slab_update(tsdn, rtree, elm, szind, slab);
}
Get rid of most of the various inline macros.
2017-04-22 00:37:34 +08:00
static inline void
Update brace style. Add braces around single-line blocks, and remove line breaks before function-opening braces. This resolves #537.
2017-01-16 08:56:30 +08:00
rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key) {
Simplify rtree_clear() to avoid locking.
2017-03-27 19:10:11 +08:00
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true);
Rename extent_t to edata_t. This frees us up from the unfortunate extent/extent2 naming collision.
2019-12-10 06:36:45 +08:00
assert(rtree_leaf_elm_edata_read(tsdn, rtree, elm, false) !=
Simplify rtree_clear() to avoid locking.
2017-03-27 19:10:11 +08:00
NULL);
Hide size class computation behind a layer of indirection. This class removes almost all the dependencies on size_classes.h, accessing the data there only via the new module sc.h, which does not depend on any configuration options. In a subsequent commit, we'll remove the configure-time size class computations, doing them at boot time, instead.
2017-12-15 04:46:39 +08:00
rtree_leaf_elm_write(tsdn, rtree, elm, NULL, SC_NSIZES, false);
Port to Mac OS X. Add Mac OS X support, based in large part on the OS X support in Mozilla's version of jemalloc.
2010-09-06 01:35:13 +08:00
}
Header refactoring: unify and de-catchall rtree module.
2017-05-24 05:26:31 +08:00
#endif /* JEMALLOC_INTERNAL_RTREE_H */
Reference in New Issue Copy Permalink