project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use a bitmap in extents_t to speed up search.

Browse Source 
Rather than iteratively checking all sufficiently large heaps during
search, maintain and use a bitmap in order to skip empty heaps.
This commit is contained in:
Jason Evans 2017-03-23 23:45:11 -07:00
parent 57e353163f
commit 5d33233a5e
3 changed files with 44 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/jemalloc/internal/bitmap_types.h
View File

@ -2,7 +2,13 @@
#define JEMALLOC_INTERNAL_BITMAP_TYPES_H #define JEMALLOC_INTERNAL_BITMAP_TYPES_H
/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
#define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS #if LG_SLAB_MAXREGS > LG_CEIL_NSIZES
/* Maximum bitmap bit count is determined by maximum regions per slab. */
# define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS
#else
/* Maximum bitmap bit count is determined by number of extent size classes. */
# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES
#endif
#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) #define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
typedef struct bitmap_level_s bitmap_level_t; typedef struct bitmap_level_s bitmap_level_t;

7
include/jemalloc/internal/extent_structs.h
View File

@ -102,6 +102,13 @@ struct extents_s {
*/ */
extent_heap_t heaps[NPSIZES+1]; extent_heap_t heaps[NPSIZES+1];
/*
* Bitmap for which set bits correspond to non-empty heaps.
*
* Synchronization: mtx.
*/
bitmap_t bitmap[BITMAP_GROUPS(NPSIZES+1)];
/* /*
* LRU of all extents in heaps. * LRU of all extents in heaps.
* *

41
src/extent.c
View File

@ -6,6 +6,9 @@
rtree_t extents_rtree; rtree_t extents_rtree;
static const bitmap_info_t extents_bitmap_info =
BITMAP_INFO_INITIALIZER(NPSIZES+1);
static void *extent_alloc_default(extent_hooks_t *extent_hooks, static void *extent_alloc_default(extent_hooks_t *extent_hooks,
void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
unsigned arena_ind); unsigned arena_ind);
@ -189,6 +192,7 @@ extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
for (unsigned i = 0; i < NPSIZES+1; i++) { for (unsigned i = 0; i < NPSIZES+1; i++) {
extent_heap_new(&extents->heaps[i]); extent_heap_new(&extents->heaps[i]);
} }
bitmap_init(extents->bitmap, &extents_bitmap_info, true);
extent_list_init(&extents->lru); extent_list_init(&extents->lru);
atomic_store_zu(&extents->npages, 0, ATOMIC_RELAXED); atomic_store_zu(&extents->npages, 0, ATOMIC_RELAXED);
extents->state = state; extents->state = state;
@ -215,6 +219,10 @@ extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent,
size_t size = extent_size_get(extent); size_t size = extent_size_get(extent);
size_t psz = extent_size_quantize_floor(size); size_t psz = extent_size_quantize_floor(size);
pszind_t pind = psz2ind(psz); pszind_t pind = psz2ind(psz);
if (extent_heap_empty(&extents->heaps[pind])) {
bitmap_unset(extents->bitmap, &extents_bitmap_info,
(size_t)pind);
}
extent_heap_insert(&extents->heaps[pind], extent); extent_heap_insert(&extents->heaps[pind], extent);
if (!preserve_lru) { if (!preserve_lru) {
extent_list_append(&extents->lru, extent); extent_list_append(&extents->lru, extent);
@ -241,6 +249,10 @@ extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent,
size_t psz = extent_size_quantize_floor(size); size_t psz = extent_size_quantize_floor(size);
pszind_t pind = psz2ind(psz); pszind_t pind = psz2ind(psz);
extent_heap_remove(&extents->heaps[pind], extent); extent_heap_remove(&extents->heaps[pind], extent);
if (extent_heap_empty(&extents->heaps[pind])) {
bitmap_set(extents->bitmap, &extents_bitmap_info,
(size_t)pind);
}
if (!preserve_lru) { if (!preserve_lru) {
extent_list_remove(&extents->lru, extent); extent_list_remove(&extents->lru, extent);
} }
@ -261,12 +273,13 @@ static extent_t *
extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
size_t size) { size_t size) {
pszind_t pind = psz2ind(extent_size_quantize_ceil(size)); pszind_t pind = psz2ind(extent_size_quantize_ceil(size));
for (pszind_t i = pind; i < NPSIZES+1; i++) { pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
(size_t)pind);
if (i < NPSIZES+1) {
assert(!extent_heap_empty(&extents->heaps[i]));
extent_t *extent = extent_heap_any(&extents->heaps[i]); extent_t *extent = extent_heap_any(&extents->heaps[i]);
if (extent != NULL) { assert(extent_size_get(extent) >= size);
assert(extent_size_get(extent) >= size); return extent;
return extent;
}
} }
return NULL; return NULL;
@ -282,14 +295,20 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
extent_t *ret = NULL; extent_t *ret = NULL;
pszind_t pind = psz2ind(extent_size_quantize_ceil(size)); pszind_t pind = psz2ind(extent_size_quantize_ceil(size));
for (pszind_t i = pind; i < NPSIZES+1; i++) { for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap,
&extents_bitmap_info, (size_t)pind); i < NPSIZES+1; i =
(pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
(size_t)i+1)) {
assert(!extent_heap_empty(&extents->heaps[i]));
extent_t *extent = extent_heap_first(&extents->heaps[i]); extent_t *extent = extent_heap_first(&extents->heaps[i]);
if (extent != NULL) { assert(extent_size_get(extent) >= size);
assert(extent_size_get(extent) >= size); if (ret == NULL || extent_snad_comp(extent, ret) < 0) {
if (ret == NULL || extent_snad_comp(extent, ret) < 0) { ret = extent;
ret = extent;
}
} }
if (i == NPSIZES) {
break;
}
assert(i < NPSIZES);
} }
return ret; return ret;