project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix and refactor runs_dirty-based purging.

Browse Source 
Fix runs_dirty-based purging to also purge dirty pages in the spare
chunk.

Refactor runs_dirty manipulation into arena_dirty_{insert,remove}(), and
move the arena->ndirty accounting into those functions.

Remove the u.ql_link field from arena_chunk_map_t, and get rid of the
enclosing union for u.rb_link, since only rb_link remains.

Remove the ndirty field from arena_chunk_t.
This commit is contained in:
Jason Evans 2014-08-14 14:45:58 -07:00
parent e8a2fd83a2
commit 070b3c3fbd
2 changed files with 91 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
include/jemalloc/internal/arena.h
View File

@ -65,23 +65,14 @@ struct arena_chunk_map_s {
*/ */
union { union {
#endif #endif
union { /*
/* * Linkage for run trees. There are two disjoint uses:
* Linkage for run trees. There are two disjoint uses: *
* * 1) arena_t's runs_avail tree.
* 1) arena_t's runs_avail tree. * 2) arena_run_t conceptually uses this linkage for in-use non-full
* 2) arena_run_t conceptually uses this linkage for in-use * runs, rather than directly embedding linkage.
* non-full runs, rather than directly embedding linkage. */
*/ rb_node(arena_chunk_map_t) rb_link;
rb_node(arena_chunk_map_t) rb_link;
/*
* List of runs currently in purgatory. arena_chunk_purge()
* temporarily allocates runs that contain dirty pages while
* purging, so that other threads cannot use the runs while the
* purging thread is operating without the arena lock held.
*/
ql_elm(arena_chunk_map_t) ql_link;
} u;
/* Profile counters, used for large object runs. */ /* Profile counters, used for large object runs. */
prof_ctx_t *prof_ctx; prof_ctx_t *prof_ctx;
@ -167,9 +158,6 @@ struct arena_chunk_s {
/* Arena that owns the chunk. */ /* Arena that owns the chunk. */
arena_t *arena; arena_t *arena;
/* Number of dirty pages. */
size_t ndirty;
/* /*
* Map of pages within chunk that keeps track of free/large/small. The * Map of pages within chunk that keeps track of free/large/small. The
* first map_bias entries are omitted, since the chunk header does not * first map_bias entries are omitted, since the chunk header does not
@ -317,9 +305,6 @@ struct arena_s {
dss_prec_t dss_prec; dss_prec_t dss_prec;
/* List of dirty runs this arena manages. */
arena_chunk_mapelms_t runs_dirty;
/* /*
* In order to avoid rapid chunk allocation/deallocation when an arena * In order to avoid rapid chunk allocation/deallocation when an arena
* oscillates right on the cusp of needing a new chunk, cache the most * oscillates right on the cusp of needing a new chunk, cache the most
@ -349,6 +334,9 @@ struct arena_s {
*/ */
arena_avail_tree_t runs_avail; arena_avail_tree_t runs_avail;
/* List of dirty runs this arena manages. */
arena_chunk_mapelms_t runs_dirty;
/* /*
* user-configureable chunk allocation and deallocation functions. * user-configureable chunk allocation and deallocation functions.
*/ */

184
src/arena.c
View File

@ -90,7 +90,7 @@ arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
/* Generate red-black tree functions. */ /* Generate red-black tree functions. */
rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t,
u.rb_link, arena_run_comp) rb_link, arena_run_comp)
static inline int static inline int
arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
@ -123,7 +123,7 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
/* Generate red-black tree functions. */ /* Generate red-black tree functions. */
rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t,
u.rb_link, arena_avail_comp) rb_link, arena_avail_comp)
static void static void
arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
@ -132,12 +132,6 @@ arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
LG_PAGE)); LG_PAGE));
if (arena_mapbits_dirty_get(chunk, pageind) != 0) {
arena->ndirty += npages;
chunk->ndirty += npages;
}
arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk,
pageind)); pageind));
} }
@ -149,16 +143,39 @@ arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
LG_PAGE)); LG_PAGE));
if (arena_mapbits_dirty_get(chunk, pageind) != 0) {
arena->ndirty -= npages;
chunk->ndirty -= npages;
}
arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk,
pageind)); pageind));
} }
static void
arena_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
size_t npages)
{
arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
LG_PAGE));
assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY);
assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) ==
CHUNK_MAP_DIRTY);
ql_elm_new(mapelm, dr_link);
ql_tail_insert(&arena->runs_dirty, mapelm, dr_link);
arena->ndirty += npages;
}
static void
arena_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
size_t npages)
{
arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
LG_PAGE));
assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY);
assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) ==
CHUNK_MAP_DIRTY);
ql_remove(&arena->runs_dirty, mapelm, dr_link);
arena->ndirty -= npages;
}
static inline void * static inline void *
arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info)
{ {
@ -252,7 +269,6 @@ static void
arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,
size_t flag_dirty, size_t need_pages) size_t flag_dirty, size_t need_pages)
{ {
arena_chunk_map_t *mapelm;
size_t total_pages, rem_pages; size_t total_pages, rem_pages;
total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >>
@ -263,11 +279,8 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,
rem_pages = total_pages - need_pages; rem_pages = total_pages - need_pages;
arena_avail_remove(arena, chunk, run_ind, total_pages); arena_avail_remove(arena, chunk, run_ind, total_pages);
if (flag_dirty != 0) { if (flag_dirty != 0)
/* If the run is dirty, it must be in the dirty list. */ arena_dirty_remove(arena, chunk, run_ind, total_pages);
mapelm = arena_mapp_get(chunk, run_ind);
ql_remove(&arena->runs_dirty, mapelm, dr_link);
}
arena_cactive_update(arena, need_pages, 0); arena_cactive_update(arena, need_pages, 0);
arena->nactive += need_pages; arena->nactive += need_pages;
@ -280,14 +293,8 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,
arena_mapbits_unallocated_set(chunk, arena_mapbits_unallocated_set(chunk,
run_ind+total_pages-1, (rem_pages << LG_PAGE), run_ind+total_pages-1, (rem_pages << LG_PAGE),
flag_dirty); flag_dirty);
mapelm = arena_mapp_get(chunk, run_ind+need_pages); arena_dirty_insert(arena, chunk, run_ind+need_pages,
/* rem_pages);
* Append the trailing run at the end of the dirty list.
* We could also insert the run at the original place.
* Let us consider this later.
*/
ql_elm_new(mapelm, dr_link);
ql_tail_insert(&arena->runs_dirty, mapelm, dr_link);
} else { } else {
arena_mapbits_unallocated_set(chunk, run_ind+need_pages, arena_mapbits_unallocated_set(chunk, run_ind+need_pages,
(rem_pages << LG_PAGE), (rem_pages << LG_PAGE),
@ -512,11 +519,6 @@ arena_chunk_init_hard(arena_t *arena)
chunk->arena = arena; chunk->arena = arena;
/*
* Claim that no pages are in use, since the header is merely overhead.
*/
chunk->ndirty = 0;
/* /*
* Initialize the map to contain one maximal free untouched run. Mark * Initialize the map to contain one maximal free untouched run. Mark
* the pages as zeroed iff chunk_alloc() returned a zeroed chunk. * the pages as zeroed iff chunk_alloc() returned a zeroed chunk.
@ -568,11 +570,6 @@ arena_chunk_alloc(arena_t *arena)
/* Insert the run into the runs_avail tree. */ /* Insert the run into the runs_avail tree. */
arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias);
if (arena_mapbits_dirty_get(chunk, map_bias) != 0) {
arena_chunk_map_t *mapelm = arena_mapp_get(chunk, map_bias);
ql_elm_new(mapelm, dr_link);
ql_tail_insert(&arena->runs_dirty, mapelm, dr_link);
}
return (chunk); return (chunk);
} }
@ -626,15 +623,15 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
* it. * it.
*/ */
arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias);
if (arena_mapbits_dirty_get(chunk, map_bias) != 0) {
arena_chunk_map_t *mapelm = arena_mapp_get(chunk, map_bias);
ql_remove(&arena->runs_dirty, mapelm, dr_link);
}
if (arena->spare != NULL) { if (arena->spare != NULL) {
arena_chunk_t *spare = arena->spare; arena_chunk_t *spare = arena->spare;
arena->spare = chunk; arena->spare = chunk;
if (arena_mapbits_dirty_get(spare, map_bias) != 0) {
arena_dirty_remove(arena, spare, map_bias,
chunk_npages-map_bias);
}
arena_chunk_dalloc_internal(arena, spare); arena_chunk_dalloc_internal(arena, spare);
} else } else
arena->spare = chunk; arena->spare = chunk;
@ -752,18 +749,17 @@ arena_run_alloc_small(arena_t *arena, size_t size, size_t binind)
static inline void static inline void
arena_maybe_purge(arena_t *arena) arena_maybe_purge(arena_t *arena)
{ {
size_t npurgeable, threshold; size_t threshold;
/* Don't purge if the option is disabled. */ /* Don't purge if the option is disabled. */
if (opt_lg_dirty_mult < 0) if (opt_lg_dirty_mult < 0)
return; return;
npurgeable = arena->ndirty;
threshold = (arena->nactive >> opt_lg_dirty_mult); threshold = (arena->nactive >> opt_lg_dirty_mult);
/* /*
* Don't purge unless the number of purgeable pages exceeds the * Don't purge unless the number of purgeable pages exceeds the
* threshold. * threshold.
*/ */
if (npurgeable <= threshold) if (arena->ndirty <= threshold)
return; return;
arena_purge(arena, false); arena_purge(arena, false);
@ -792,50 +788,53 @@ arena_dirty_count(arena_t *arena)
} }
static size_t static size_t
arena_compute_npurgatory(arena_t *arena, bool all) arena_compute_npurge(arena_t *arena, bool all)
{ {
size_t npurgatory, npurgeable; size_t npurge;
/* /*
* Compute the minimum number of pages that this thread should try to * Compute the minimum number of pages that this thread should try to
* purge. * purge.
*/ */
npurgeable = arena->ndirty;
if (all == false) { if (all == false) {
size_t threshold = (arena->nactive >> opt_lg_dirty_mult); size_t threshold = (arena->nactive >> opt_lg_dirty_mult);
npurgatory = npurgeable - threshold; npurge = arena->ndirty - threshold;
} else } else
npurgatory = npurgeable; npurge = arena->ndirty;
return (npurgatory); return (npurge);
} }
static size_t static size_t
arena_stash_dirty(arena_t *arena, bool all, size_t npurgatory, arena_stash_dirty(arena_t *arena, bool all, size_t npurge,
arena_chunk_mapelms_t *mapelms) arena_chunk_mapelms_t *mapelms)
{ {
arena_chunk_map_t *mapelm; arena_chunk_map_t *mapelm;
size_t nstashed = 0; size_t nstashed = 0;
arena_chunk_t *chunk;
size_t pageind, npages, run_size;
arena_run_t *run;
/* Add at least npurgatory pages to purge_list. */ /* Add at least npurge pages to purge_list. */
for (mapelm = ql_first(&arena->runs_dirty); mapelm != NULL; for (mapelm = ql_first(&arena->runs_dirty); mapelm != NULL;
mapelm = ql_first(&arena->runs_dirty)) { mapelm = ql_first(&arena->runs_dirty)) {
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm);
pageind = arena_mapelm_to_pageind(mapelm); size_t pageind = arena_mapelm_to_pageind(mapelm);
run_size = arena_mapbits_unallocated_size_get(chunk, pageind); size_t run_size = arena_mapbits_unallocated_size_get(chunk,
npages = run_size >> LG_PAGE; pageind);
run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << size_t npages = run_size >> LG_PAGE;
LG_PAGE)); arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
(uintptr_t)(pageind << LG_PAGE));
assert(pageind + npages <= chunk_npages); assert(pageind + npages <= chunk_npages);
assert(arena_mapbits_dirty_get(chunk, pageind) == assert(arena_mapbits_dirty_get(chunk, pageind) ==
arena_mapbits_dirty_get(chunk, pageind+npages-1)); arena_mapbits_dirty_get(chunk, pageind+npages-1));
/*
* If purging the spare chunk's run, make it available prior to
* allocation.
*/
if (chunk == arena->spare)
arena_chunk_alloc(arena);
/* Temporarily allocate the free dirty run. */ /* Temporarily allocate the free dirty run. */
arena_run_split_large(arena, run, run_size, false); arena_run_split_large(arena, run, run_size, false);
/* Append to purge_list for later processing. */ /* Append to purge_list for later processing. */
@ -844,7 +843,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurgatory,
nstashed += npages; nstashed += npages;
if (all == false && nstashed >= npurgatory) if (all == false && nstashed >= npurge)
break; break;
} }
@ -856,8 +855,6 @@ arena_purge_stashed(arena_t *arena, arena_chunk_mapelms_t *mapelms)
{ {
size_t npurged, nmadvise; size_t npurged, nmadvise;
arena_chunk_map_t *mapelm; arena_chunk_map_t *mapelm;
arena_chunk_t *chunk;
size_t pageind, npages, run_size;
if (config_stats) if (config_stats)
nmadvise = 0; nmadvise = 0;
@ -866,8 +863,9 @@ arena_purge_stashed(arena_t *arena, arena_chunk_mapelms_t *mapelms)
malloc_mutex_unlock(&arena->lock); malloc_mutex_unlock(&arena->lock);
ql_foreach(mapelm, mapelms, dr_link) { ql_foreach(mapelm, mapelms, dr_link) {
arena_chunk_t *chunk;
size_t pageind, run_size, npages, flag_unzeroed, i;
bool unzeroed; bool unzeroed;
size_t flag_unzeroed, i;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm);
pageind = arena_mapelm_to_pageind(mapelm); pageind = arena_mapelm_to_pageind(mapelm);
@ -913,17 +911,14 @@ static void
arena_unstash_purged(arena_t *arena, arena_chunk_mapelms_t *mapelms) arena_unstash_purged(arena_t *arena, arena_chunk_mapelms_t *mapelms)
{ {
arena_chunk_map_t *mapelm; arena_chunk_map_t *mapelm;
arena_chunk_t *chunk;
arena_run_t *run;
size_t pageind;
/* Deallocate runs. */ /* Deallocate runs. */
for (mapelm = ql_first(mapelms); mapelm != NULL; for (mapelm = ql_first(mapelms); mapelm != NULL;
mapelm = ql_first(mapelms)) { mapelm = ql_first(mapelms)) {
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm);
pageind = arena_mapelm_to_pageind(mapelm); size_t pageind = arena_mapelm_to_pageind(mapelm);
run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
LG_PAGE)); (uintptr_t)(pageind << LG_PAGE));
ql_remove(mapelms, mapelm, dr_link); ql_remove(mapelms, mapelm, dr_link);
arena_run_dalloc(arena, run, false, true); arena_run_dalloc(arena, run, false, true);
} }
@ -932,7 +927,7 @@ arena_unstash_purged(arena_t *arena, arena_chunk_mapelms_t *mapelms)
void void
arena_purge(arena_t *arena, bool all) arena_purge(arena_t *arena, bool all)
{ {
size_t npurgatory, npurgeable, npurged; size_t npurge, npurgeable, npurged;
arena_chunk_mapelms_t purge_list; arena_chunk_mapelms_t purge_list;
if (config_debug) { if (config_debug) {
@ -944,21 +939,12 @@ arena_purge(arena_t *arena, bool all)
if (config_stats) if (config_stats)
arena->stats.npurge++; arena->stats.npurge++;
/* npurge = arena_compute_npurge(arena, all);
* Add the minimum number of pages this thread should try to purge to
* arena->npurgatory. This will keep multiple threads from racing to
* reduce ndirty below the threshold.
*/
npurgatory = arena_compute_npurgatory(arena, all);
ql_new(&purge_list); ql_new(&purge_list);
npurgeable = arena_stash_dirty(arena, all, npurge, &purge_list);
npurgeable = arena_stash_dirty(arena, all, npurgatory, &purge_list); assert(npurgeable >= npurge);
assert(npurgeable >= npurgatory);
npurged = arena_purge_stashed(arena, &purge_list); npurged = arena_purge_stashed(arena, &purge_list);
assert(npurged == npurgeable); assert(npurged == npurgeable);
arena_unstash_purged(arena, &purge_list); arena_unstash_purged(arena, &purge_list);
} }
@ -999,9 +985,8 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size,
/* If the successor is dirty, remove it from runs_dirty. */ /* If the successor is dirty, remove it from runs_dirty. */
if (flag_dirty != 0) { if (flag_dirty != 0) {
arena_chunk_map_t *mapelm = arena_mapp_get(chunk, arena_dirty_remove(arena, chunk, run_ind+run_pages,
run_ind+run_pages); nrun_pages);
ql_remove(&arena->runs_dirty, mapelm, dr_link);
} }
size += nrun_size; size += nrun_size;
@ -1032,11 +1017,8 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size,
arena_avail_remove(arena, chunk, run_ind, prun_pages); arena_avail_remove(arena, chunk, run_ind, prun_pages);
/* If the predecessor is dirty, remove it from runs_dirty. */ /* If the predecessor is dirty, remove it from runs_dirty. */
if (flag_dirty != 0) { if (flag_dirty != 0)
arena_chunk_map_t *mapelm = arena_mapp_get(chunk, arena_dirty_remove(arena, chunk, run_ind, prun_pages);
run_ind);
ql_remove(&arena->runs_dirty, mapelm, dr_link);
}
size += prun_size; size += prun_size;
run_pages += prun_pages; run_pages += prun_pages;
@ -1055,7 +1037,6 @@ static void
arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned)
{ {
arena_chunk_t *chunk; arena_chunk_t *chunk;
arena_chunk_map_t *mapelm;
size_t size, run_ind, run_pages, flag_dirty; size_t size, run_ind, run_pages, flag_dirty;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
@ -1110,12 +1091,8 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned)
arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
arena_avail_insert(arena, chunk, run_ind, run_pages); arena_avail_insert(arena, chunk, run_ind, run_pages);
if (dirty) { if (dirty)
/* Insert into runs_dirty list. */ arena_dirty_insert(arena, chunk, run_ind, run_pages);
mapelm = arena_mapp_get(chunk, run_ind);
ql_elm_new(mapelm, dr_link);
ql_tail_insert(&arena->runs_dirty, mapelm, dr_link);
}
/* Deallocate chunk if it is now completely unused. */ /* Deallocate chunk if it is now completely unused. */
if (size == arena_maxclass) { if (size == arena_maxclass) {
@ -2237,14 +2214,13 @@ arena_new(arena_t *arena, unsigned ind)
arena->dss_prec = chunk_dss_prec_get(); arena->dss_prec = chunk_dss_prec_get();
/* Initialize chunks. */
ql_new(&arena->runs_dirty);
arena->spare = NULL; arena->spare = NULL;
arena->nactive = 0; arena->nactive = 0;
arena->ndirty = 0; arena->ndirty = 0;
arena_avail_tree_new(&arena->runs_avail); arena_avail_tree_new(&arena->runs_avail);
ql_new(&arena->runs_dirty);
/* Initialize bins. */ /* Initialize bins. */
for (i = 0; i < NBINS; i++) { for (i = 0; i < NBINS; i++) {