project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrate whole chunks into unused dirty page purging machinery.

Browse Source 
Extend per arena unused dirty page purging to manage unused dirty chunks
in aaddtion to unused dirty runs.  Rather than immediately unmapping
deallocated chunks (or purging them in the --disable-munmap case), store
them in a separate set of trees, chunks_[sz]ad_dirty.  Preferrentially
allocate dirty chunks.  When excessive unused dirty pages accumulate,
purge runs and chunks in ingegrated LRU order (and unmap chunks in the
--enable-munmap case).

Refactor extent_node_t to provide accessor functions.
This commit is contained in:
Jason Evans
2015-02-15 18:04:46 -08:00
parent 40ab8f98e4
commit ee41ad409a
12 changed files with 631 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
include/jemalloc/internal/arena.h
View File

@@ -35,6 +35,7 @@ typedef struct arena_s arena_t;
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#ifdef JEMALLOC_ARENA_STRUCTS_A
struct arena_run_s {
/* Index of bin this run is associated with. */
index_t binind;
@@ -136,7 +137,7 @@ struct arena_chunk_map_misc_s {
union {
/* Linkage for list of dirty runs. */
ql_elm(arena_chunk_map_misc_t) dr_link;
qr(arena_chunk_map_misc_t) rd_link;
/* Profile counters, used for large object runs. */
prof_tctx_t *prof_tctx;
@@ -147,14 +148,16 @@ struct arena_chunk_map_misc_s {
};
typedef rb_tree(arena_chunk_map_misc_t) arena_avail_tree_t;
typedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t;
typedef ql_head(arena_chunk_map_misc_t) arena_chunk_miscelms_t;
typedef qr(arena_chunk_map_misc_t) arena_chunk_miscelms_t;
#endif /* JEMALLOC_ARENA_STRUCTS_A */
#ifdef JEMALLOC_ARENA_STRUCTS_B
/* Arena chunk header. */
struct arena_chunk_s {
/*
* The arena that owns the chunk is node.arena. This field as a whole
* is used by chunks_rtree to support both ivsalloc() and core-based
* debugging.
* A pointer to the arena that owns the chunk is stored within the node.
* This field as a whole is used by chunks_rtree to support both
* ivsalloc() and core-based debugging.
*/
extent_node_t node;
@@ -309,13 +312,29 @@ struct arena_s {
size_t ndirty;
/*
* Size/address-ordered trees of this arena's available runs. The trees
* are used for first-best-fit run allocation.
* Size/address-ordered tree of this arena's available runs. The tree
* is used for first-best-fit run allocation.
*/
arena_avail_tree_t runs_avail;
/* List of dirty runs this arena manages. */
arena_chunk_miscelms_t runs_dirty;
/*
* Unused dirty memory this arena manages. Dirty memory is conceptually
* tracked as an arbitrarily interleaved LRU of runs and chunks, but the
* list linkage is actually semi-duplicated in order to avoid extra
* arena_chunk_map_misc_t space overhead.
*
* LRU-----------------------------------------------------------MRU
*
* ______________ ___ ___
* ...-->|chunks_dirty|<--------->|c|<-------------------->|c|<--...
* -------------- |h| |h|
* ____________ _____ |u| _____ _____ |u|
* ...-->|runs_dirty|<-->|run|<-->|n|<-->|run|<-->|run|<-->|n|<--...
* ------------ ----- |k| ----- ----- |k|
* --- ---
*/
arena_chunk_map_misc_t runs_dirty;
extent_node_t chunks_dirty;
/* Extant huge allocations. */
ql_head(extent_node_t) huge;
@@ -329,6 +348,8 @@ struct arena_s {
* orderings are needed, which is why there are two trees with the same
* contents.
*/
extent_tree_t chunks_szad_dirty;
extent_tree_t chunks_ad_dirty;
extent_tree_t chunks_szad_mmap;
extent_tree_t chunks_ad_mmap;
extent_tree_t chunks_szad_dss;
@@ -347,6 +368,7 @@ struct arena_s {
/* bins is used to store trees of free regions. */
arena_bin_t bins[NBINS];
};
#endif /* JEMALLOC_ARENA_STRUCTS_B */
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
@@ -363,6 +385,10 @@ extern size_t arena_maxclass; /* Max size class for arenas. */
extern unsigned nlclasses; /* Number of large size classes. */
extern unsigned nhclasses; /* Number of huge size classes. */
void arena_chunk_dirty_maybe_insert(arena_t *arena, extent_node_t *node,
bool dirty);
void arena_chunk_dirty_maybe_remove(arena_t *arena, extent_node_t *node,
bool dirty);
extent_node_t *arena_node_alloc(arena_t *arena);
void arena_node_dalloc(arena_t *arena, extent_node_t *node);
void *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment,
@@ -818,7 +844,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
assert(binind != BININD_INVALID);
assert(binind < NBINS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
arena = chunk->node.arena;
arena = extent_node_arena_get(&chunk->node);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
actual_mapbits = arena_mapbits_get(chunk, pageind);
assert(mapbits == actual_mapbits);
@@ -1013,7 +1039,7 @@ arena_aalloc(const void *ptr)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (likely(chunk != ptr))
return (chunk->node.arena);
return (extent_node_arena_get(&chunk->node));
else
return (huge_aalloc(ptr));
}
@@ -1085,8 +1111,8 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
mapbits);
tcache_dalloc_small(tsd, tcache, ptr, binind);
} else {
arena_dalloc_small(chunk->node.arena, chunk,
ptr, pageind);
arena_dalloc_small(extent_node_arena_get(
&chunk->node), chunk, ptr, pageind);
}
} else {
size_t size = arena_mapbits_large_size_get(chunk,
@@ -1097,8 +1123,8 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
if (likely(tcache != NULL) && size <= tcache_maxclass)
tcache_dalloc_large(tsd, tcache, ptr, size);
else {
arena_dalloc_large(chunk->node.arena, chunk,
ptr);
arena_dalloc_large(extent_node_arena_get(
&chunk->node), chunk, ptr);
}
}
} else
@@ -1136,8 +1162,8 @@ arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
} else {
size_t pageind = ((uintptr_t)ptr -
(uintptr_t)chunk) >> LG_PAGE;
arena_dalloc_small(chunk->node.arena, chunk,
ptr, pageind);
arena_dalloc_small(extent_node_arena_get(
&chunk->node), chunk, ptr, pageind);
}
} else {
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
@@ -1145,8 +1171,8 @@ arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
if (likely(tcache != NULL) && size <= tcache_maxclass)
tcache_dalloc_large(tsd, tcache, ptr, size);
else {
arena_dalloc_large(chunk->node.arena, chunk,
ptr);
arena_dalloc_large(extent_node_arena_get(
&chunk->node), chunk, ptr);
}
}
} else

4
include/jemalloc/internal/chunk.h
View File

@@ -44,8 +44,10 @@ void *chunk_alloc_arena(chunk_alloc_t *chunk_alloc,
size_t size, size_t alignment, bool *zero);
void *chunk_alloc_default(void *new_addr, size_t size, size_t alignment,
bool *zero, unsigned arena_ind);
void chunk_unmap(arena_t *arena, void *chunk, size_t size);
void chunk_record(arena_t *arena, extent_tree_t *chunks_szad,
extent_tree_t *chunks_ad, bool dirty, void *chunk, size_t size);
bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind);
void chunk_unmap(arena_t *arena, void *chunk, size_t size);
bool chunk_boot(void);
void chunk_prefork(void);
void chunk_postfork_parent(void);

135
include/jemalloc/internal/extent.h
View File

@@ -7,36 +7,48 @@ typedef struct extent_node_s extent_node_t;
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
/* Tree of extents. */
/* Tree of extents. Use accessor functions for en_* fields. */
struct extent_node_s {
/* Arena from which this extent came, if any. */
arena_t *arena;
arena_t *en_arena;
/* Pointer to the extent that this tree node is responsible for. */
void *addr;
void *en_addr;
/* Total region size. */
size_t en_size;
/*
* Total region size, or 0 if this node corresponds to an arena chunk.
* The zeroed flag is used by chunk recycling code to track whether
* memory is zero-filled.
*/
size_t size;
bool en_zeroed;
/*
* 'prof_tctx' and 'zeroed' are never needed at the same time, so
* overlay them in order to fit extent_node_t in one cache line.
* The achunk flag is used to validate that huge allocation lookups
* don't return arena chunks.
*/
bool en_achunk;
union {
/* Profile counters, used for huge objects. */
prof_tctx_t *prof_tctx;
prof_tctx_t *en_prof_tctx;
/* True if zero-filled; used by chunk recycling code. */
bool zeroed;
struct {
/*
* Linkage for arena's runs_dirty and chunks_dirty
* rings.
*/
qr(extent_node_t) cd_link;
arena_chunk_map_misc_t runs_dirty;
};
};
union {
/* Linkage for the size/address-ordered tree. */
rb_node(extent_node_t) szad_link;
/* Linkage for huge allocations and cached chunks nodes. */
/* Linkage for arena's huge and node_cache lists. */
ql_elm(extent_node_t) ql_link;
};
@@ -57,6 +69,107 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
arena_t *extent_node_arena_get(const extent_node_t *node);
void *extent_node_addr_get(const extent_node_t *node);
size_t extent_node_size_get(const extent_node_t *node);
bool extent_node_zeroed_get(const extent_node_t *node);
bool extent_node_achunk_get(const extent_node_t *node);
prof_tctx_t *extent_node_prof_tctx_get(const extent_node_t *node);
void extent_node_arena_set(extent_node_t *node, arena_t *arena);
void extent_node_addr_set(extent_node_t *node, void *addr);
void extent_node_size_set(extent_node_t *node, size_t size);
void extent_node_zeroed_set(extent_node_t *node, bool zeroed);
void extent_node_achunk_set(extent_node_t *node, bool achunk);
void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
JEMALLOC_INLINE arena_t *
extent_node_arena_get(const extent_node_t *node)
{
return (node->en_arena);
}
JEMALLOC_INLINE void *
extent_node_addr_get(const extent_node_t *node)
{
return (node->en_addr);
}
JEMALLOC_INLINE size_t
extent_node_size_get(const extent_node_t *node)
{
return (node->en_size);
}
JEMALLOC_INLINE bool
extent_node_zeroed_get(const extent_node_t *node)
{
return (node->en_zeroed);
}
JEMALLOC_INLINE bool
extent_node_achunk_get(const extent_node_t *node)
{
return (node->en_achunk);
}
JEMALLOC_INLINE prof_tctx_t *
extent_node_prof_tctx_get(const extent_node_t *node)
{
return (node->en_prof_tctx);
}
JEMALLOC_INLINE void
extent_node_arena_set(extent_node_t *node, arena_t *arena)
{
node->en_arena = arena;
}
JEMALLOC_INLINE void
extent_node_addr_set(extent_node_t *node, void *addr)
{
node->en_addr = addr;
}
JEMALLOC_INLINE void
extent_node_size_set(extent_node_t *node, size_t size)
{
node->en_size = size;
}
JEMALLOC_INLINE void
extent_node_zeroed_set(extent_node_t *node, bool zeroed)
{
node->en_zeroed = zeroed;
}
JEMALLOC_INLINE void
extent_node_achunk_set(extent_node_t *node, bool achunk)
{
node->en_achunk = achunk;
}
JEMALLOC_INLINE void
extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx)
{
node->en_prof_tctx = tctx;
}
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/

10
include/jemalloc/internal/jemalloc_internal.h.in
View File

@@ -368,8 +368,13 @@ typedef unsigned index_t;
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
#define JEMALLOC_ARENA_STRUCTS_A
#include "jemalloc/internal/arena.h"
#undef JEMALLOC_ARENA_STRUCTS_A
#include "jemalloc/internal/extent.h"
#define JEMALLOC_ARENA_STRUCTS_B
#include "jemalloc/internal/arena.h"
#undef JEMALLOC_ARENA_STRUCTS_B
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/chunk.h"
@@ -933,7 +938,8 @@ ivsalloc(const void *ptr, bool demote)
if (node == NULL)
return (0);
/* Only arena chunks should be looked up via interior pointers. */
assert(node->addr == ptr || node->size == 0);
assert(extent_node_addr_get(node) == ptr ||
extent_node_achunk_get(node));
return (isalloc(ptr, demote));
}

15
include/jemalloc/internal/private_symbols.txt
View File

@@ -13,6 +13,8 @@ arena_choose
arena_choose_hard
arena_chunk_alloc_huge
arena_chunk_dalloc_huge
arena_chunk_dirty_maybe_insert
arena_chunk_dirty_maybe_remove
arena_chunk_ralloc_huge_expand
arena_chunk_ralloc_huge_shrink
arena_chunk_ralloc_huge_similar
@@ -143,6 +145,7 @@ chunk_npages
chunk_postfork_child
chunk_postfork_parent
chunk_prefork
chunk_record
chunk_register
chunk_unmap
chunks_rtree
@@ -173,6 +176,18 @@ ctl_postfork_child
ctl_postfork_parent
ctl_prefork
dss_prec_names
extent_node_achunk_get
extent_node_achunk_set
extent_node_addr_get
extent_node_addr_set
extent_node_arena_get
extent_node_arena_set
extent_node_prof_tctx_get
extent_node_prof_tctx_set
extent_node_size_get
extent_node_size_set
extent_node_zeroed_get
extent_node_zeroed_set
extent_tree_ad_empty
extent_tree_ad_first
extent_tree_ad_insert