Use a bitmap in extents_t to speed up search.
Rather than iteratively checking all sufficiently large heaps during search, maintain and use a bitmap in order to skip empty heaps.
This commit is contained in:
parent
57e353163f
commit
5d33233a5e
@ -2,7 +2,13 @@
|
||||
#define JEMALLOC_INTERNAL_BITMAP_TYPES_H
|
||||
|
||||
/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
|
||||
#define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS
|
||||
#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES
|
||||
/* Maximum bitmap bit count is determined by maximum regions per slab. */
|
||||
# define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS
|
||||
#else
|
||||
/* Maximum bitmap bit count is determined by number of extent size classes. */
|
||||
# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES
|
||||
#endif
|
||||
#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
|
||||
|
||||
typedef struct bitmap_level_s bitmap_level_t;
|
||||
|
@ -102,6 +102,13 @@ struct extents_s {
|
||||
*/
|
||||
extent_heap_t heaps[NPSIZES+1];
|
||||
|
||||
/*
|
||||
* Bitmap for which set bits correspond to non-empty heaps.
|
||||
*
|
||||
* Synchronization: mtx.
|
||||
*/
|
||||
bitmap_t bitmap[BITMAP_GROUPS(NPSIZES+1)];
|
||||
|
||||
/*
|
||||
* LRU of all extents in heaps.
|
||||
*
|
||||
|
41
src/extent.c
41
src/extent.c
@ -6,6 +6,9 @@
|
||||
|
||||
rtree_t extents_rtree;
|
||||
|
||||
static const bitmap_info_t extents_bitmap_info =
|
||||
BITMAP_INFO_INITIALIZER(NPSIZES+1);
|
||||
|
||||
static void *extent_alloc_default(extent_hooks_t *extent_hooks,
|
||||
void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
|
||||
unsigned arena_ind);
|
||||
@ -189,6 +192,7 @@ extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
|
||||
for (unsigned i = 0; i < NPSIZES+1; i++) {
|
||||
extent_heap_new(&extents->heaps[i]);
|
||||
}
|
||||
bitmap_init(extents->bitmap, &extents_bitmap_info, true);
|
||||
extent_list_init(&extents->lru);
|
||||
atomic_store_zu(&extents->npages, 0, ATOMIC_RELAXED);
|
||||
extents->state = state;
|
||||
@ -215,6 +219,10 @@ extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent,
|
||||
size_t size = extent_size_get(extent);
|
||||
size_t psz = extent_size_quantize_floor(size);
|
||||
pszind_t pind = psz2ind(psz);
|
||||
if (extent_heap_empty(&extents->heaps[pind])) {
|
||||
bitmap_unset(extents->bitmap, &extents_bitmap_info,
|
||||
(size_t)pind);
|
||||
}
|
||||
extent_heap_insert(&extents->heaps[pind], extent);
|
||||
if (!preserve_lru) {
|
||||
extent_list_append(&extents->lru, extent);
|
||||
@ -241,6 +249,10 @@ extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent,
|
||||
size_t psz = extent_size_quantize_floor(size);
|
||||
pszind_t pind = psz2ind(psz);
|
||||
extent_heap_remove(&extents->heaps[pind], extent);
|
||||
if (extent_heap_empty(&extents->heaps[pind])) {
|
||||
bitmap_set(extents->bitmap, &extents_bitmap_info,
|
||||
(size_t)pind);
|
||||
}
|
||||
if (!preserve_lru) {
|
||||
extent_list_remove(&extents->lru, extent);
|
||||
}
|
||||
@ -261,12 +273,13 @@ static extent_t *
|
||||
extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
|
||||
size_t size) {
|
||||
pszind_t pind = psz2ind(extent_size_quantize_ceil(size));
|
||||
for (pszind_t i = pind; i < NPSIZES+1; i++) {
|
||||
pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
|
||||
(size_t)pind);
|
||||
if (i < NPSIZES+1) {
|
||||
assert(!extent_heap_empty(&extents->heaps[i]));
|
||||
extent_t *extent = extent_heap_any(&extents->heaps[i]);
|
||||
if (extent != NULL) {
|
||||
assert(extent_size_get(extent) >= size);
|
||||
return extent;
|
||||
}
|
||||
assert(extent_size_get(extent) >= size);
|
||||
return extent;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
@ -282,14 +295,20 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
|
||||
extent_t *ret = NULL;
|
||||
|
||||
pszind_t pind = psz2ind(extent_size_quantize_ceil(size));
|
||||
for (pszind_t i = pind; i < NPSIZES+1; i++) {
|
||||
for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap,
|
||||
&extents_bitmap_info, (size_t)pind); i < NPSIZES+1; i =
|
||||
(pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
|
||||
(size_t)i+1)) {
|
||||
assert(!extent_heap_empty(&extents->heaps[i]));
|
||||
extent_t *extent = extent_heap_first(&extents->heaps[i]);
|
||||
if (extent != NULL) {
|
||||
assert(extent_size_get(extent) >= size);
|
||||
if (ret == NULL || extent_snad_comp(extent, ret) < 0) {
|
||||
ret = extent;
|
||||
}
|
||||
assert(extent_size_get(extent) >= size);
|
||||
if (ret == NULL || extent_snad_comp(extent, ret) < 0) {
|
||||
ret = extent;
|
||||
}
|
||||
if (i == NPSIZES) {
|
||||
break;
|
||||
}
|
||||
assert(i < NPSIZES);
|
||||
}
|
||||
|
||||
return ret;
|
||||
|
Loading…
Reference in New Issue
Block a user