project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor dirty run linkage to reduce sizeof(extent_node_t).

Browse Source
This commit is contained in:
Jason Evans 2015-03-10 18:15:40 -07:00
parent 54673fd8d7
commit 38e42d311c
4 changed files with 95 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
include/jemalloc/internal/arena.h
View File

@ -23,6 +23,7 @@
*/ */
#define LG_DIRTY_MULT_DEFAULT 3 #define LG_DIRTY_MULT_DEFAULT 3
typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t;
typedef struct arena_run_s arena_run_t; typedef struct arena_run_s arena_run_t;
typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t;
typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t;
@ -120,6 +121,10 @@ struct arena_chunk_map_bits_s {
#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED #define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED
}; };
struct arena_runs_dirty_link_s {
qr(arena_runs_dirty_link_t) rd_link;
};
/* /*
* Each arena_chunk_map_misc_t corresponds to one page within the chunk, just * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just
* like arena_chunk_map_bits_t. Two separate arrays are stored within each * like arena_chunk_map_bits_t. Two separate arrays are stored within each
@ -131,13 +136,13 @@ struct arena_chunk_map_misc_s {
* *
* 1) arena_t's runs_avail tree. * 1) arena_t's runs_avail tree.
* 2) arena_run_t conceptually uses this linkage for in-use non-full * 2) arena_run_t conceptually uses this linkage for in-use non-full
* runs, rather than directly embedding linkage. * runs, rather than directly embedding linkage.
*/ */
rb_node(arena_chunk_map_misc_t) rb_link; rb_node(arena_chunk_map_misc_t) rb_link;
union { union {
/* Linkage for list of dirty runs. */ /* Linkage for list of dirty runs. */
qr(arena_chunk_map_misc_t) rd_link; arena_runs_dirty_link_t rd;
/* Profile counters, used for large object runs. */ /* Profile counters, used for large object runs. */
prof_tctx_t *prof_tctx; prof_tctx_t *prof_tctx;
@ -324,15 +329,27 @@ struct arena_s {
* *
* LRU-----------------------------------------------------------MRU * LRU-----------------------------------------------------------MRU
* *
* ______________ ___ ___ * /------------------\
* ...-->|chunks_cache|<--------->|c|<-------------------->|c|<--... * | arena |
* -------------- |h| |h| * | |
* ____________ _____ |u| _____ _____ |u| * | /------------\ | /-----------\
* ...-->|runs_dirty|<-->|run|<-->|n|<-->|run|<-->|run|<-->|n|<--... * ...---->|chunks_cache|<---------------------->| chunk |<--...
* ------------ ----- |k| ----- ----- |k| * | \------------/ | | |
* --- --- * | | | |
* | | /---\ /---\ | |
* | | |run| |run| | |
* | | | | | | | |
* | /----------\ | |---| |---| | /-----\ |
* ...----->|runs_dirty|<---->|rd |<---->|rd |<---->|rdelm|<-----...
* | \----------/ | |---| |---| | \-----/ |
* | | | | | | | |
* | | | | | | | |
* | | \---/ \---/ | |
* | | | |
* | | | |
* \------------------/ \-----------/
*/ */
arena_chunk_map_misc_t runs_dirty; arena_runs_dirty_link_t runs_dirty;
extent_node_t chunks_cache; extent_node_t chunks_cache;
/* Extant huge allocations. */ /* Extant huge allocations. */
@ -465,6 +482,7 @@ arena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk,
size_t pageind); size_t pageind);
size_t arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm); size_t arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm);
void *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm); void *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm);
arena_chunk_map_misc_t *arena_rd_to_miscelm(arena_runs_dirty_link_t *rd);
arena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); arena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run);
size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbitsp_read(size_t *mapbitsp); size_t arena_mapbitsp_read(size_t *mapbitsp);
@ -556,6 +574,18 @@ arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm)
return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE)));
} }
JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t *
arena_rd_to_miscelm(arena_runs_dirty_link_t *rd)
{
arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t
*)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd));
assert(arena_miscelm_to_pageind(miscelm) >= map_bias);
assert(arena_miscelm_to_pageind(miscelm) < chunk_npages);
return (miscelm);
}
JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t *
arena_run_to_miscelm(arena_run_t *run) arena_run_to_miscelm(arena_run_t *run)
{ {

12
include/jemalloc/internal/extent.h
View File

@ -34,7 +34,7 @@ struct extent_node_s {
prof_tctx_t *en_prof_tctx; prof_tctx_t *en_prof_tctx;
/* Linkage for arena's runs_dirty and chunks_cache rings. */ /* Linkage for arena's runs_dirty and chunks_cache rings. */
arena_chunk_map_misc_t runs_dirty; arena_runs_dirty_link_t rdelm;
qr(extent_node_t) cc_link; qr(extent_node_t) cc_link;
union { union {
@ -79,7 +79,7 @@ void extent_node_init(extent_node_t *node, arena_t *arena, void *addr,
size_t size, bool zeroed); size_t size, bool zeroed);
void extent_node_dirty_linkage_init(extent_node_t *node); void extent_node_dirty_linkage_init(extent_node_t *node);
void extent_node_dirty_insert(extent_node_t *node, void extent_node_dirty_insert(extent_node_t *node,
arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty); arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty);
void extent_node_dirty_remove(extent_node_t *node); void extent_node_dirty_remove(extent_node_t *node);
#endif #endif
@ -186,16 +186,16 @@ JEMALLOC_INLINE void
extent_node_dirty_linkage_init(extent_node_t *node) extent_node_dirty_linkage_init(extent_node_t *node)
{ {
qr_new(&node->runs_dirty, rd_link); qr_new(&node->rdelm, rd_link);
qr_new(node, cc_link); qr_new(node, cc_link);
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
extent_node_dirty_insert(extent_node_t *node, extent_node_dirty_insert(extent_node_t *node,
arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty) arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty)
{ {
qr_meld(runs_dirty, &node->runs_dirty, rd_link); qr_meld(runs_dirty, &node->rdelm, rd_link);
qr_meld(chunks_dirty, node, cc_link); qr_meld(chunks_dirty, node, cc_link);
} }
@ -203,7 +203,7 @@ JEMALLOC_INLINE void
extent_node_dirty_remove(extent_node_t *node) extent_node_dirty_remove(extent_node_t *node)
{ {
qr_remove(&node->runs_dirty, rd_link); qr_remove(&node->rdelm, rd_link);
qr_remove(node, cc_link); qr_remove(node, cc_link);
} }

1
include/jemalloc/internal/private_symbols.txt
View File

@ -81,6 +81,7 @@ arena_quarantine_junk_small
arena_ralloc arena_ralloc
arena_ralloc_junk_large arena_ralloc_junk_large
arena_ralloc_no_move arena_ralloc_no_move
arena_rd_to_miscelm
arena_redzone_corruption arena_redzone_corruption
arena_run_regind arena_run_regind
arena_run_to_miscelm arena_run_to_miscelm

89
src/arena.c
View File

@ -136,8 +136,8 @@ arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) ==
CHUNK_MAP_DIRTY); CHUNK_MAP_DIRTY);
qr_new(miscelm, rd_link); qr_new(&miscelm->rd, rd_link);
qr_meld(&arena->runs_dirty, miscelm, rd_link); qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link);
arena->ndirty += npages; arena->ndirty += npages;
} }
@ -153,7 +153,7 @@ arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) ==
CHUNK_MAP_DIRTY); CHUNK_MAP_DIRTY);
qr_remove(miscelm, rd_link); qr_remove(&miscelm->rd, rd_link);
assert(arena->ndirty >= npages); assert(arena->ndirty >= npages);
arena->ndirty -= npages; arena->ndirty -= npages;
} }
@ -1056,22 +1056,23 @@ static size_t
arena_dirty_count(arena_t *arena) arena_dirty_count(arena_t *arena)
{ {
size_t ndirty = 0; size_t ndirty = 0;
arena_chunk_map_misc_t *runselm; arena_runs_dirty_link_t *rdelm;
extent_node_t *chunkselm; extent_node_t *chunkselm;
for (runselm = qr_next(&arena->runs_dirty, rd_link), for (rdelm = qr_next(&arena->runs_dirty, rd_link),
chunkselm = qr_next(&arena->chunks_cache, cc_link); chunkselm = qr_next(&arena->chunks_cache, cc_link);
runselm != &arena->runs_dirty; runselm = qr_next(runselm, rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) {
rd_link)) {
size_t npages; size_t npages;
if (runselm == &chunkselm->runs_dirty) { if (rdelm == &chunkselm->rdelm) {
npages = extent_node_size_get(chunkselm) >> LG_PAGE; npages = extent_node_size_get(chunkselm) >> LG_PAGE;
chunkselm = qr_next(chunkselm, cc_link); chunkselm = qr_next(chunkselm, cc_link);
} else { } else {
arena_chunk_t *chunk = (arena_chunk_t arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
*)CHUNK_ADDR2BASE(runselm); rdelm);
size_t pageind = arena_miscelm_to_pageind(runselm); arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm);
assert(arena_mapbits_allocated_get(chunk, pageind) == assert(arena_mapbits_allocated_get(chunk, pageind) ==
0); 0);
assert(arena_mapbits_large_get(chunk, pageind) == 0); assert(arena_mapbits_large_get(chunk, pageind) == 0);
@ -1107,21 +1108,21 @@ arena_compute_npurge(arena_t *arena, bool all)
static size_t static size_t
arena_stash_dirty(arena_t *arena, bool all, size_t npurge, arena_stash_dirty(arena_t *arena, bool all, size_t npurge,
arena_chunk_map_misc_t *purge_runs_sentinel, arena_runs_dirty_link_t *purge_runs_sentinel,
extent_node_t *purge_chunks_sentinel) extent_node_t *purge_chunks_sentinel)
{ {
arena_chunk_map_misc_t *runselm, *runselm_next; arena_runs_dirty_link_t *rdelm, *rdelm_next;
extent_node_t *chunkselm; extent_node_t *chunkselm;
size_t nstashed = 0; size_t nstashed = 0;
/* Stash at least npurge pages. */ /* Stash at least npurge pages. */
for (runselm = qr_next(&arena->runs_dirty, rd_link), for (rdelm = qr_next(&arena->runs_dirty, rd_link),
chunkselm = qr_next(&arena->chunks_cache, cc_link); chunkselm = qr_next(&arena->chunks_cache, cc_link);
runselm != &arena->runs_dirty; runselm = runselm_next) { rdelm != &arena->runs_dirty; rdelm = rdelm_next) {
size_t npages; size_t npages;
runselm_next = qr_next(runselm, rd_link); rdelm_next = qr_next(rdelm, rd_link);
if (runselm == &chunkselm->runs_dirty) { if (rdelm == &chunkselm->rdelm) {
extent_node_t *chunkselm_next; extent_node_t *chunkselm_next;
bool zero; bool zero;
UNUSED void *chunk; UNUSED void *chunk;
@ -1144,9 +1145,11 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge,
chunkselm = chunkselm_next; chunkselm = chunkselm_next;
} else { } else {
arena_chunk_t *chunk = arena_chunk_t *chunk =
(arena_chunk_t *)CHUNK_ADDR2BASE(runselm); (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
size_t pageind = arena_miscelm_to_pageind(runselm); arena_chunk_map_misc_t *miscelm =
arena_run_t *run = &runselm->run; arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm);
arena_run_t *run = &miscelm->run;
size_t run_size = size_t run_size =
arena_mapbits_unallocated_size_get(chunk, pageind); arena_mapbits_unallocated_size_get(chunk, pageind);
@ -1167,12 +1170,12 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge,
arena_run_split_large(arena, run, run_size, false); arena_run_split_large(arena, run, run_size, false);
/* Stash. */ /* Stash. */
if (false) if (false)
qr_new(runselm, rd_link); /* Redundant. */ qr_new(rdelm, rd_link); /* Redundant. */
else { else {
assert(qr_next(runselm, rd_link) == runselm); assert(qr_next(rdelm, rd_link) == rdelm);
assert(qr_prev(runselm, rd_link) == runselm); assert(qr_prev(rdelm, rd_link) == rdelm);
} }
qr_meld(purge_runs_sentinel, runselm, rd_link); qr_meld(purge_runs_sentinel, rdelm, rd_link);
} }
nstashed += npages; nstashed += npages;
@ -1184,11 +1187,12 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge,
} }
static size_t static size_t
arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, arena_purge_stashed(arena_t *arena,
arena_runs_dirty_link_t *purge_runs_sentinel,
extent_node_t *purge_chunks_sentinel) extent_node_t *purge_chunks_sentinel)
{ {
size_t npurged, nmadvise; size_t npurged, nmadvise;
arena_chunk_map_misc_t *runselm; arena_runs_dirty_link_t *rdelm;
extent_node_t *chunkselm; extent_node_t *chunkselm;
if (config_stats) if (config_stats)
@ -1196,13 +1200,12 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel,
npurged = 0; npurged = 0;
malloc_mutex_unlock(&arena->lock); malloc_mutex_unlock(&arena->lock);
for (runselm = qr_next(purge_runs_sentinel, rd_link), for (rdelm = qr_next(purge_runs_sentinel, rd_link),
chunkselm = qr_next(purge_chunks_sentinel, cc_link); chunkselm = qr_next(purge_chunks_sentinel, cc_link);
runselm != purge_runs_sentinel; runselm = qr_next(runselm, rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) {
rd_link)) {
size_t npages; size_t npages;
if (runselm == &chunkselm->runs_dirty) { if (rdelm == &chunkselm->rdelm) {
size_t size = extent_node_size_get(chunkselm); size_t size = extent_node_size_get(chunkselm);
bool unzeroed; bool unzeroed;
@ -1216,8 +1219,10 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel,
size_t pageind, run_size, flag_unzeroed, i; size_t pageind, run_size, flag_unzeroed, i;
bool unzeroed; bool unzeroed;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
pageind = arena_miscelm_to_pageind(runselm); arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm);
pageind = arena_miscelm_to_pageind(miscelm);
run_size = arena_mapbits_large_size_get(chunk, pageind); run_size = arena_mapbits_large_size_get(chunk, pageind);
npages = run_size >> LG_PAGE; npages = run_size >> LG_PAGE;
@ -1259,18 +1264,18 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel,
static void static void
arena_unstash_purged(arena_t *arena, arena_unstash_purged(arena_t *arena,
arena_chunk_map_misc_t *purge_runs_sentinel, arena_runs_dirty_link_t *purge_runs_sentinel,
extent_node_t *purge_chunks_sentinel) extent_node_t *purge_chunks_sentinel)
{ {
arena_chunk_map_misc_t *runselm, *runselm_next; arena_runs_dirty_link_t *rdelm, *rdelm_next;
extent_node_t *chunkselm; extent_node_t *chunkselm;
/* Deallocate runs. */ /* Deallocate runs. */
for (runselm = qr_next(purge_runs_sentinel, rd_link), for (rdelm = qr_next(purge_runs_sentinel, rd_link),
chunkselm = qr_next(purge_chunks_sentinel, cc_link); chunkselm = qr_next(purge_chunks_sentinel, cc_link);
runselm != purge_runs_sentinel; runselm = runselm_next) { rdelm != purge_runs_sentinel; rdelm = rdelm_next) {
runselm_next = qr_next(runselm, rd_link); rdelm_next = qr_next(rdelm, rd_link);
if (runselm == &chunkselm->runs_dirty) { if (rdelm == &chunkselm->rdelm) {
extent_node_t *chunkselm_next = qr_next(chunkselm, extent_node_t *chunkselm_next = qr_next(chunkselm,
cc_link); cc_link);
void *addr = extent_node_addr_get(chunkselm); void *addr = extent_node_addr_get(chunkselm);
@ -1281,8 +1286,10 @@ arena_unstash_purged(arena_t *arena,
chunkselm = chunkselm_next; chunkselm = chunkselm_next;
chunk_dalloc_arena(arena, addr, size, zeroed); chunk_dalloc_arena(arena, addr, size, zeroed);
} else { } else {
arena_run_t *run = &runselm->run; arena_chunk_map_misc_t *miscelm =
qr_remove(runselm, rd_link); arena_rd_to_miscelm(rdelm);
arena_run_t *run = &miscelm->run;
qr_remove(rdelm, rd_link);
arena_run_dalloc(arena, run, false, true); arena_run_dalloc(arena, run, false, true);
} }
} }
@ -1292,7 +1299,7 @@ void
arena_purge(arena_t *arena, bool all) arena_purge(arena_t *arena, bool all)
{ {
size_t npurge, npurgeable, npurged; size_t npurge, npurgeable, npurged;
arena_chunk_map_misc_t purge_runs_sentinel; arena_runs_dirty_link_t purge_runs_sentinel;
extent_node_t purge_chunks_sentinel; extent_node_t purge_chunks_sentinel;
/* /*