project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move slabs out of chunks.

Browse Source
This commit is contained in:
Jason Evans
2016-05-29 18:34:50 -07:00
parent d28e5a6696
commit 498856f44a
21 changed files with 596 additions and 2332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

795
include/jemalloc/internal/arena.h
View File

@@ -3,9 +3,9 @@
#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS)
/* Maximum number of regions in one run. */
#define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN)
#define RUN_MAXREGS (1U << LG_RUN_MAXREGS)
/* Maximum number of regions in one slab. */
#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN)
#define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS)
/*
* The minimum ratio of active:dirty pages per arena is computed as:
@@ -29,12 +29,7 @@ typedef enum {
/* Number of event ticks between time checks. */
#define DECAY_NTICKS_PER_UPDATE 1000
typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t;
typedef struct arena_avail_links_s arena_avail_links_t;
typedef struct arena_run_s arena_run_t;
typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t;
typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t;
typedef struct arena_chunk_s arena_chunk_t;
typedef struct arena_slab_data_s arena_slab_data_t;
typedef struct arena_bin_info_s arena_bin_info_t;
typedef struct arena_bin_s arena_bin_t;
typedef struct arena_s arena_t;
@@ -45,152 +40,25 @@ typedef struct arena_tdata_s arena_tdata_t;
#ifdef JEMALLOC_H_STRUCTS
#ifdef JEMALLOC_ARENA_STRUCTS_A
struct arena_run_s {
/* Index of bin this run is associated with. */
struct arena_slab_data_s {
/* Index of bin this slab is associated with. */
szind_t binind;
/* Number of free regions in run. */
/* Number of free regions in slab. */
unsigned nfree;
/* Per region allocated/deallocated bitmap. */
bitmap_t bitmap[BITMAP_GROUPS_MAX];
};
/* Each element of the chunk map corresponds to one page within the chunk. */
struct arena_chunk_map_bits_s {
/*
* Run address (or size) and various flags are stored together. The bit
* layout looks like (assuming 32-bit system):
*
* ???????? ???????? ???nnnnn nnndumla
*
* ? : Unallocated: Run address for first/last pages, unset for internal
* pages.
* Small: Run page offset.
* Large: Run page count for first page, unset for trailing pages.
* n : binind for small size class, BININD_INVALID for large size class.
* d : dirty?
* u : unzeroed?
* m : decommitted?
* l : large?
* a : allocated?
*
* Following are example bit patterns for the three types of runs.
*
* p : run page offset
* s : run size
* n : binind for size class; large objects set these to BININD_INVALID
* x : don't care
* - : 0
* + : 1
* [DUMLA] : bit set
* [dumla] : bit unset
*
* Unallocated (clean):
* ssssssss ssssssss sss+++++ +++dum-a
* xxxxxxxx xxxxxxxx xxxxxxxx xxx-Uxxx
* ssssssss ssssssss sss+++++ +++dUm-a
*
* Unallocated (dirty):
* ssssssss ssssssss sss+++++ +++D-m-a
* xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
* ssssssss ssssssss sss+++++ +++D-m-a
*
* Small:
* pppppppp pppppppp pppnnnnn nnnd---A
* pppppppp pppppppp pppnnnnn nnn----A
* pppppppp pppppppp pppnnnnn nnnd---A
*
* Large:
* ssssssss ssssssss sss+++++ +++D--LA
* xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
* -------- -------- ---+++++ +++D--LA
*
* Large (sampled, size <= LARGE_MINCLASS):
* ssssssss ssssssss sssnnnnn nnnD--LA
* xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
* -------- -------- ---+++++ +++D--LA
*
* Large (not sampled, size == LARGE_MINCLASS):
* ssssssss ssssssss sss+++++ +++D--LA
* xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
* -------- -------- ---+++++ +++D--LA
*/
size_t bits;
#define CHUNK_MAP_ALLOCATED ((size_t)0x01U)
#define CHUNK_MAP_LARGE ((size_t)0x02U)
#define CHUNK_MAP_STATE_MASK ((size_t)0x3U)
#define CHUNK_MAP_DECOMMITTED ((size_t)0x04U)
#define CHUNK_MAP_UNZEROED ((size_t)0x08U)
#define CHUNK_MAP_DIRTY ((size_t)0x10U)
#define CHUNK_MAP_FLAGS_MASK ((size_t)0x1cU)
#define CHUNK_MAP_BININD_SHIFT 5
#define BININD_INVALID ((size_t)0xffU)
#define CHUNK_MAP_BININD_MASK (BININD_INVALID << CHUNK_MAP_BININD_SHIFT)
#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK
#define CHUNK_MAP_RUNIND_SHIFT (CHUNK_MAP_BININD_SHIFT + 8)
#define CHUNK_MAP_SIZE_SHIFT (CHUNK_MAP_RUNIND_SHIFT - LG_PAGE)
#define CHUNK_MAP_SIZE_MASK \
(~(CHUNK_MAP_BININD_MASK | CHUNK_MAP_FLAGS_MASK | CHUNK_MAP_STATE_MASK))
};
struct arena_runs_dirty_link_s {
qr(arena_runs_dirty_link_t) rd_link;
};
/*
* Each arena_chunk_map_misc_t corresponds to one page within the chunk, just
* like arena_chunk_map_bits_t. Two separate arrays are stored within each
* chunk header in order to improve cache locality.
*/
struct arena_chunk_map_misc_s {
/*
* Linkage for run heaps. There are two disjoint uses:
*
* 1) arena_t's runs_avail heaps.
* 2) arena_run_t conceptually uses this linkage for in-use non-full
* runs, rather than directly embedding linkage.
*/
phn(arena_chunk_map_misc_t) ph_link;
union {
/* Linkage for list of dirty runs. */
arena_runs_dirty_link_t rd;
/* Profile counters, used for large object runs. */
union {
void *prof_tctx_pun;
prof_tctx_t *prof_tctx;
};
/* Small region run metadata. */
arena_run_t run;
};
};
typedef ph(arena_chunk_map_misc_t) arena_run_heap_t;
#endif /* JEMALLOC_ARENA_STRUCTS_A */
#ifdef JEMALLOC_ARENA_STRUCTS_B
/* Arena chunk header. */
struct arena_chunk_s {
/*
* Map of pages within chunk that keeps track of free/large/small. The
* first map_bias entries are omitted, since the chunk header does not
* need to be tracked in the map. This omission saves a header page
* for common chunk sizes (e.g. 4 MiB).
*/
arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */
};
/*
* Read-only information associated with each element of arena_t's bins array
* is stored separately, partly to reduce memory usage (only one copy, rather
* than one per arena), but mainly to avoid false cacheline sharing.
*
* Each run has the following layout:
* Each slab has the following layout:
*
* /--------------------\
* | region 0 |
@@ -205,45 +73,42 @@ struct arena_chunk_s {
* \--------------------/
*/
struct arena_bin_info_s {
/* Size of regions in a run for this bin's size class. */
/* Size of regions in a slab for this bin's size class. */
size_t reg_size;
/* Total size of a run for this bin's size class. */
size_t run_size;
/* Total size of a slab for this bin's size class. */
size_t slab_size;
/* Total number of regions in a run for this bin's size class. */
/* Total number of regions in a slab for this bin's size class. */
uint32_t nregs;
/*
* Metadata used to manipulate bitmaps for runs associated with this
* Metadata used to manipulate bitmaps for slabs associated with this
* bin.
*/
bitmap_info_t bitmap_info;
};
struct arena_bin_s {
/*
* All operations on runcur, runs, and stats require that lock be
* locked. Run allocation/deallocation are protected by the arena lock,
* which may be acquired while holding one or more bin locks, but not
* vise versa.
*/
/* All operations on arena_bin_t fields require lock ownership. */
malloc_mutex_t lock;
/*
* Current run being used to service allocations of this bin's size
* class.
* Current slab being used to service allocations of this bin's size
* class. slabcur is independent of slabs_{nonfull,full}; whenever
* slabcur is reassigned, the previous slab must be deallocated or
* inserted into slabs_{nonfull,full}.
*/
arena_run_t *runcur;
extent_t *slabcur;
/*
* Heap of non-full runs. This heap is used when looking for an
* existing run when runcur is no longer usable. We choose the
* non-full run that is lowest in memory; this policy tends to keep
* objects packed well, and it can also help reduce the number of
* almost-empty chunks.
* Heap of non-full slabs. This heap is used to assure that new
* allocations come from the non-full slab that is lowest in memory.
*/
arena_run_heap_t runs;
extent_heap_t slabs_nonfull;
/* Ring sentinel used to track full slabs. */
extent_t slabs_full;
/* Bin statistics. */
malloc_bin_stats_t stats;
@@ -272,7 +137,7 @@ struct arena_s {
* perspective:
* 1) Thread assignment (modifies nthreads) is synchronized via atomics.
* 2) Bin-related operations are protected by bin locks.
* 3) Chunk- and run-related operations are protected by this mutex.
* 3) Chunk-related operations are protected by this mutex.
*/
malloc_mutex_t lock;
@@ -294,32 +159,17 @@ struct arena_s {
dss_prec_t dss_prec;
/* Extant arena chunks. */
ql_head(extent_t) achunks;
/*
* In order to avoid rapid chunk allocation/deallocation when an arena
* oscillates right on the cusp of needing a new chunk, cache the most
* recently freed chunk. The spare is left in the arena's chunk trees
* until it is deleted.
*
* There is one spare chunk per arena, rather than one spare total, in
* order to avoid interactions between multiple threads that could make
* a single spare inadequate.
*/
extent_t *spare;
/* Minimum ratio (log base 2) of nactive:ndirty. */
ssize_t lg_dirty_mult;
/* True if a thread is currently executing arena_purge_to_limit(). */
bool purging;
/* Number of pages in active runs and huge regions. */
/* Number of pages in active extents. */
size_t nactive;
/*
* Current count of pages within unused runs that are potentially
* Current count of pages within unused extents that are potentially
* dirty, and for which madvise(... MADV_DONTNEED) has not been called.
* By tracking this, we can institute a limit on how much dirty unused
* memory is mapped for each arena.
@@ -327,35 +177,10 @@ struct arena_s {
size_t ndirty;
/*
* Unused dirty memory this arena manages. Dirty memory is conceptually
* tracked as an arbitrarily interleaved LRU of dirty runs and cached
* chunks, but the list linkage is actually semi-duplicated in order to
* avoid extra arena_chunk_map_misc_t space overhead.
*
* LRU-----------------------------------------------------------MRU
*
* /-- arena ---\
* | |
* | |
* |------------| /-- chunk --\
* ...->|chunks_cache|<--------------------------->| /------\ |<--...
* |------------| | |extent| |
* | | | | | |
* | | /- run -\ /- run -\ | | | |
* | | | | | | | | | |
* | | | | | | | | | |
* |------------| |-------| |-------| | |------| |
* ...->|runs_dirty |<-->|rd |<-->|rd |<---->|rd |<----...
* |------------| |-------| |-------| | |------| |
* | | | | | | | | | |
* | | | | | | | \------/ |
* | | \-------/ \-------/ | |
* | | | |
* | | | |
* \------------/ \-----------/
* Ring sentinel used to track unused dirty memory. Dirty memory is
* managed as an LRU of cached extents.
*/
arena_runs_dirty_link_t runs_dirty;
extent_t chunks_cache;
extent_t extents_dirty;
/*
* Approximate time in seconds from the creation of a set of unused
@@ -424,16 +249,8 @@ struct arena_s {
/* User-configurable chunk hook functions. */
chunk_hooks_t chunk_hooks;
/* bins is used to store trees of free regions. */
/* bins is used to store heaps of free regions. */
arena_bin_t bins[NBINS];
/*
* Size-segregated address-ordered heaps of this arena's available runs,
* used for first-best-fit run allocation. Runs are quantized, i.e.
* they reside in the last heap which corresponds to a size class less
* than or equal to the run size.
*/
arena_run_heap_t runs_avail[NPSIZES];
};
/* Used in conjunction with tsd for fast arena-related context lookup. */
@@ -461,15 +278,6 @@ extern ssize_t opt_decay_time;
extern const arena_bin_info_t arena_bin_info[NBINS];
extern size_t map_bias; /* Number of arena chunk header pages. */
extern size_t map_misc_offset;
extern size_t arena_maxrun; /* Max run size for arenas. */
#ifdef JEMALLOC_JET
typedef size_t (run_quantize_t)(size_t);
extern run_quantize_t *run_quantize_floor;
extern run_quantize_t *run_quantize_ceil;
#endif
extent_t *arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero);
@@ -514,10 +322,9 @@ void arena_prof_promote(tsdn_t *tsdn, extent_t *extent, const void *ptr,
void arena_dalloc_promoted(tsdn_t *tsdn, extent_t *extent, void *ptr,
tcache_t *tcache, bool slow_path);
void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, extent_t *extent, void *ptr,
arena_chunk_map_bits_t *bitselm);
void arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
extent_t *extent, void *ptr, size_t pageind);
extent_t *extent, void *ptr);
void arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
void *ptr);
bool arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t oldsize, size_t size, size_t extra, bool zero);
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
@@ -552,70 +359,19 @@ void arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
arena_chunk_map_bits_t *arena_bitselm_get_mutable(arena_chunk_t *chunk,
size_t pageind);
const arena_chunk_map_bits_t *arena_bitselm_get_const(
const arena_chunk_t *chunk, size_t pageind);
arena_chunk_map_misc_t *arena_miscelm_get_mutable(arena_chunk_t *chunk,
size_t pageind);
const arena_chunk_map_misc_t *arena_miscelm_get_const(
const arena_chunk_t *chunk, size_t pageind);
size_t arena_miscelm_to_pageind(const extent_t *extent,
const arena_chunk_map_misc_t *miscelm);
void *arena_miscelm_to_rpages(const extent_t *extent,
const arena_chunk_map_misc_t *miscelm);
arena_chunk_map_misc_t *arena_rd_to_miscelm(const extent_t *extent,
arena_runs_dirty_link_t *rd);
arena_chunk_map_misc_t *arena_run_to_miscelm(const extent_t *extent,
arena_run_t *run);
size_t *arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind);
const size_t *arena_mapbitsp_get_const(const arena_chunk_t *chunk,
size_t pageind);
size_t arena_mapbitsp_read(const size_t *mapbitsp);
size_t arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_size_decode(size_t mapbits);
size_t arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk,
size_t pageind);
size_t arena_mapbits_large_size_get(const arena_chunk_t *chunk,
size_t pageind);
size_t arena_mapbits_small_runind_get(const arena_chunk_t *chunk,
size_t pageind);
szind_t arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_decommitted_get(const arena_chunk_t *chunk,
size_t pageind);
size_t arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind);
void arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits);
size_t arena_mapbits_size_encode(size_t size);
void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind,
size_t size, size_t flags);
void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
size_t size);
void arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind,
size_t flags);
void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind,
size_t size, size_t flags);
void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
szind_t binind);
void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind,
size_t runind, szind_t binind, size_t flags);
void arena_metadata_allocated_add(arena_t *arena, size_t size);
void arena_metadata_allocated_sub(arena_t *arena, size_t size);
size_t arena_metadata_allocated_get(arena_t *arena);
bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes);
bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes);
bool arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes);
szind_t arena_ptr_small_binind_get(tsdn_t *tsdn, const void *ptr,
size_t mapbits);
szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
const void *ptr);
void arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx);
void arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, const void *old_ptr, prof_tctx_t *old_tctx);
prof_tctx_t *tctx);
void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks);
void arena_decay_tick(tsdn_t *tsdn, arena_t *arena);
void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
@@ -630,330 +386,6 @@ void arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
# ifdef JEMALLOC_ARENA_INLINE_A
JEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t *
arena_bitselm_get_mutable(arena_chunk_t *chunk, size_t pageind)
{
assert(pageind >= map_bias);
assert(pageind < chunk_npages);
return (&chunk->map_bits[pageind-map_bias]);
}
JEMALLOC_ALWAYS_INLINE const arena_chunk_map_bits_t *
arena_bitselm_get_const(const arena_chunk_t *chunk, size_t pageind)
{
return (arena_bitselm_get_mutable((arena_chunk_t *)chunk, pageind));
}
JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t *
arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind)
{
assert(pageind >= map_bias);
assert(pageind < chunk_npages);
return ((arena_chunk_map_misc_t *)((uintptr_t)chunk +
(uintptr_t)map_misc_offset) + pageind-map_bias);
}
JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t *
arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind)
{
return (arena_miscelm_get_mutable((arena_chunk_t *)chunk, pageind));
}
JEMALLOC_ALWAYS_INLINE size_t
arena_miscelm_to_pageind(const extent_t *extent,
const arena_chunk_map_misc_t *miscelm)
{
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk +
map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias;
assert(pageind >= map_bias);
assert(pageind < chunk_npages);
return (pageind);
}
JEMALLOC_ALWAYS_INLINE void *
arena_miscelm_to_rpages(const extent_t *extent,
const arena_chunk_map_misc_t *miscelm)
{
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = arena_miscelm_to_pageind(extent, miscelm);
return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE)));
}
JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t *
arena_rd_to_miscelm(const extent_t *extent, arena_runs_dirty_link_t *rd)
{
arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t
*)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd));
assert(arena_miscelm_to_pageind(extent, miscelm) >= map_bias);
assert(arena_miscelm_to_pageind(extent, miscelm) < chunk_npages);
return (miscelm);
}
JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t *
arena_run_to_miscelm(const extent_t *extent, arena_run_t *run)
{
arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t
*)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run));
assert(arena_miscelm_to_pageind(extent, miscelm) >= map_bias);
assert(arena_miscelm_to_pageind(extent, miscelm) < chunk_npages);
return (miscelm);
}
JEMALLOC_ALWAYS_INLINE size_t *
arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind)
{
return (&arena_bitselm_get_mutable(chunk, pageind)->bits);
}
JEMALLOC_ALWAYS_INLINE const size_t *
arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind)
{
return (arena_mapbitsp_get_mutable((arena_chunk_t *)chunk, pageind));
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbitsp_read(const size_t *mapbitsp)
{
return (*mapbitsp);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind)
{
return (arena_mapbitsp_read(arena_mapbitsp_get_const(chunk, pageind)));
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_size_decode(size_t mapbits)
{
size_t size;
#if CHUNK_MAP_SIZE_SHIFT > 0
size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT;
#elif CHUNK_MAP_SIZE_SHIFT == 0
size = mapbits & CHUNK_MAP_SIZE_MASK;
#else
size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT;
#endif
return (size);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
return (arena_mapbits_size_decode(mapbits));
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_large_size_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
(CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED));
return (arena_mapbits_size_decode(mapbits));
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_small_runind_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
CHUNK_MAP_ALLOCATED);
return (mapbits >> CHUNK_MAP_RUNIND_SHIFT);
}
JEMALLOC_ALWAYS_INLINE szind_t
arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
szind_t binind;
mapbits = arena_mapbits_get(chunk, pageind);
binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
assert(binind < NBINS || binind == BININD_INVALID);
return (binind);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
(CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
return (mapbits & CHUNK_MAP_DIRTY);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
(CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
return (mapbits & CHUNK_MAP_UNZEROED);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_decommitted_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
(CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
return (mapbits & CHUNK_MAP_DECOMMITTED);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
return (mapbits & CHUNK_MAP_LARGE);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
return (mapbits & CHUNK_MAP_ALLOCATED);
}
JEMALLOC_ALWAYS_INLINE void
arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits)
{
*mapbitsp = mapbits;
}
JEMALLOC_ALWAYS_INLINE size_t
arena_mapbits_size_encode(size_t size)
{
size_t mapbits;
#if CHUNK_MAP_SIZE_SHIFT > 0
mapbits = size << CHUNK_MAP_SIZE_SHIFT;
#elif CHUNK_MAP_SIZE_SHIFT == 0
mapbits = size;
#else
mapbits = size >> -CHUNK_MAP_SIZE_SHIFT;
#endif
assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0);
return (mapbits);
}
JEMALLOC_ALWAYS_INLINE void
arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
size_t flags)
{
size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
assert((size & PAGE_MASK) == 0);
assert((flags & CHUNK_MAP_FLAGS_MASK) == flags);
assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags &
(CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) |
CHUNK_MAP_BININD_INVALID | flags);
}
JEMALLOC_ALWAYS_INLINE void
arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
size_t size)
{
size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
size_t mapbits = arena_mapbitsp_read(mapbitsp);
assert((size & PAGE_MASK) == 0);
assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) |
(mapbits & ~CHUNK_MAP_SIZE_MASK));
}
JEMALLOC_ALWAYS_INLINE void
arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, size_t flags)
{
size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
assert((flags & CHUNK_MAP_UNZEROED) == flags);
arena_mapbitsp_write(mapbitsp, flags);
}
JEMALLOC_ALWAYS_INLINE void
arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
size_t flags)
{
size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
assert((size & PAGE_MASK) == 0);
assert((flags & CHUNK_MAP_FLAGS_MASK) == flags);
assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags &
(CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) |
CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE |
CHUNK_MAP_ALLOCATED);
}
JEMALLOC_ALWAYS_INLINE void
arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
szind_t binind)
{
size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
size_t mapbits = arena_mapbitsp_read(mapbitsp);
assert(binind <= BININD_INVALID);
assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS +
large_pad);
arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) |
(binind << CHUNK_MAP_BININD_SHIFT));
}
JEMALLOC_ALWAYS_INLINE void
arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
szind_t binind, size_t flags)
{
size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
assert(binind < BININD_INVALID);
assert(pageind - runind >= map_bias);
assert((flags & CHUNK_MAP_UNZEROED) == flags);
arena_mapbitsp_write(mapbitsp, (runind << CHUNK_MAP_RUNIND_SHIFT) |
(binind << CHUNK_MAP_BININD_SHIFT) | flags | CHUNK_MAP_ALLOCATED);
}
JEMALLOC_INLINE void
arena_metadata_allocated_add(arena_t *arena, size_t size)
{
@@ -1022,54 +454,6 @@ arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes)
# endif /* JEMALLOC_ARENA_INLINE_A */
# ifdef JEMALLOC_ARENA_INLINE_B
JEMALLOC_ALWAYS_INLINE szind_t
arena_ptr_small_binind_get(tsdn_t *tsdn, const void *ptr, size_t mapbits)
{
szind_t binind;
binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
if (config_debug) {
const extent_t *extent;
arena_chunk_t *chunk;
arena_t *arena;
size_t pageind;
size_t actual_mapbits;
size_t rpages_ind;
const arena_run_t *run;
arena_bin_t *bin;
szind_t run_binind, actual_binind;
const arena_bin_info_t *bin_info;
const arena_chunk_map_misc_t *miscelm;
const void *rpages;
assert(binind != BININD_INVALID);
assert(binind < NBINS);
extent = iealloc(tsdn, ptr);
chunk = (arena_chunk_t *)extent_base_get(extent);
arena = extent_arena_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
actual_mapbits = arena_mapbits_get(chunk, pageind);
assert(mapbits == actual_mapbits);
assert(arena_mapbits_large_get(chunk, pageind) == 0);
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
rpages_ind = pageind - arena_mapbits_small_runind_get(chunk,
pageind);
miscelm = arena_miscelm_get_const(chunk, rpages_ind);
run = &miscelm->run;
run_binind = run->binind;
bin = &arena->bins[run_binind];
actual_binind = (szind_t)(bin - arena->bins);
assert(run_binind == actual_binind);
bin_info = &arena_bin_info[actual_binind];
rpages = arena_miscelm_to_rpages(extent, miscelm);
assert(((uintptr_t)ptr - (uintptr_t)rpages) % bin_info->reg_size
== 0);
}
return (binind);
}
JEMALLOC_INLINE szind_t
arena_bin_index(arena_t *arena, arena_bin_t *bin)
{
@@ -1081,27 +465,13 @@ arena_bin_index(arena_t *arena, arena_bin_t *bin)
JEMALLOC_INLINE prof_tctx_t *
arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{
prof_tctx_t *ret;
cassert(config_prof);
assert(ptr != NULL);
if (likely(extent_slab_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
if (likely((mapbits & CHUNK_MAP_LARGE) == 0))
ret = (prof_tctx_t *)(uintptr_t)1U;
else {
arena_chunk_map_misc_t *elm =
arena_miscelm_get_mutable(chunk, pageind);
ret = atomic_read_p(&elm->prof_tctx_pun);
}
} else
ret = huge_prof_tctx_get(tsdn, extent);
return (ret);
if (unlikely(!extent_slab_get(extent)))
return (huge_prof_tctx_get(tsdn, extent));
return ((prof_tctx_t *)(uintptr_t)1U);
}
JEMALLOC_INLINE void
@@ -1112,61 +482,20 @@ arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
cassert(config_prof);
assert(ptr != NULL);
if (likely(extent_slab_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
if (unlikely(usize > SMALL_MAXCLASS || (uintptr_t)tctx >
(uintptr_t)1U)) {
arena_chunk_map_misc_t *elm;
assert(arena_mapbits_large_get(chunk, pageind) != 0);
elm = arena_miscelm_get_mutable(chunk, pageind);
atomic_write_p(&elm->prof_tctx_pun, tctx);
} else {
/*
* tctx must always be initialized for large runs.
* Assert that the surrounding conditional logic is
* equivalent to checking whether ptr refers to a large
* run.
*/
assert(arena_mapbits_large_get(chunk, pageind) == 0);
}
} else
if (unlikely(!extent_slab_get(extent)))
huge_prof_tctx_set(tsdn, extent, tctx);
}
JEMALLOC_INLINE void
arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, const void *old_ptr, prof_tctx_t *old_tctx)
prof_tctx_t *tctx)
{
cassert(config_prof);
assert(ptr != NULL);
assert(!extent_slab_get(extent));
if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr &&
(uintptr_t)old_tctx > (uintptr_t)1U))) {
if (likely(extent_slab_get(extent))) {
arena_chunk_t *chunk =
(arena_chunk_t *)extent_base_get(extent);
size_t pageind;
arena_chunk_map_misc_t *elm;
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) !=
0);
assert(arena_mapbits_large_get(chunk, pageind) != 0);
elm = arena_miscelm_get_mutable(chunk, pageind);
atomic_write_p(&elm->prof_tctx_pun,
(prof_tctx_t *)(uintptr_t)1U);
} else
huge_prof_tctx_reset(tsdn, extent);
}
huge_prof_tctx_reset(tsdn, extent);
}
JEMALLOC_ALWAYS_INLINE void
@@ -1231,20 +560,9 @@ arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
assert(ptr != NULL);
if (likely(extent_slab_get(extent))) {
const arena_chunk_t *chunk =
(const arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
szind_t binind;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
binind = arena_mapbits_binind_get(chunk, pageind);
/* Small allocation. */
assert(arena_mapbits_large_get(chunk, pageind) != 0 ||
arena_ptr_small_binind_get(tsdn, ptr,
arena_mapbits_get(chunk, pageind)) == binind);
ret = index2size(binind);
} else
if (likely(extent_slab_get(extent)))
ret = index2size(extent_slab_data_get_const(extent)->binind);
else
ret = huge_salloc(tsdn, extent);
return (ret);
@@ -1260,19 +578,13 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
if (likely(extent_slab_get(extent))) {
/* Small allocation. */
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind);
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
assert((mapbits & CHUNK_MAP_LARGE) == 0);
if (likely(tcache != NULL)) {
szind_t binind = arena_ptr_small_binind_get(tsdn, ptr,
mapbits);
szind_t binind = extent_slab_data_get(extent)->binind;
tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, binind,
slow_path);
} else {
arena_dalloc_small(tsdn, extent_arena_get(extent),
chunk, extent, ptr, pageind);
extent, ptr);
}
} else {
size_t usize = extent_usize_get(extent);
@@ -1282,8 +594,8 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
arena_dalloc_promoted(tsdn, extent, ptr,
tcache, slow_path);
} else {
tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, ptr,
usize, slow_path);
tcache_dalloc_huge(tsdn_tsd(tsdn), tcache,
ptr, usize, slow_path);
}
} else
huge_dalloc(tsdn, extent);
@@ -1302,15 +614,12 @@ arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
/* Small allocation. */
if (likely(tcache != NULL)) {
szind_t binind = size2index(size);
assert(binind == extent_slab_data_get(extent)->binind);
tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, binind,
slow_path);
} else {
arena_chunk_t *chunk =
(arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)ptr -
(uintptr_t)chunk) >> LG_PAGE;
arena_dalloc_small(tsdn, extent_arena_get(extent),
chunk, extent, ptr, pageind);
extent, ptr);
}
} else {
if (likely(tcache != NULL) && size <= tcache_maxclass) {

2
include/jemalloc/internal/bitmap.h
View File

@@ -2,7 +2,7 @@
#ifdef JEMALLOC_H_TYPES
/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS
#define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS
#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
typedef struct bitmap_level_s bitmap_level_t;

86
include/jemalloc/internal/extent.h
View File

@@ -27,9 +27,6 @@ struct extent_s {
/* True if extent is active (in use). */
bool e_active;
/* True if extent is dirty (touched). */
bool e_dirty;
/*
* The zeroed flag is used by chunk recycling code to track whether
* memory is zero-filled.
@@ -50,21 +47,27 @@ struct extent_s {
*/
bool e_slab;
/* Profile counters, used for huge objects. */
union {
void *e_prof_tctx_pun;
prof_tctx_t *e_prof_tctx;
/* Small region slab metadata. */
arena_slab_data_t e_slab_data;
/* Profile counters, used for huge objects. */
union {
void *e_prof_tctx_pun;
prof_tctx_t *e_prof_tctx;
};
};
/* Linkage for arena's runs_dirty and chunks_cache rings. */
arena_runs_dirty_link_t rd;
qr(extent_t) cc_link;
/*
* Linkage for arena's extents_dirty and arena_bin_t's slabs_full rings.
*/
qr(extent_t) qr_link;
union {
/* Linkage for per size class address-ordered heaps. */
phn(extent_t) ph_link;
/* Linkage for arena's achunks, huge, and node_cache lists. */
/* Linkage for arena's huge and extent_cache lists. */
ql_elm(extent_t) ql_link;
};
};
@@ -102,11 +105,12 @@ void *extent_before_get(const extent_t *extent);
void *extent_last_get(const extent_t *extent);
void *extent_past_get(const extent_t *extent);
bool extent_active_get(const extent_t *extent);
bool extent_dirty_get(const extent_t *extent);
bool extent_retained_get(const extent_t *extent);
bool extent_zeroed_get(const extent_t *extent);
bool extent_committed_get(const extent_t *extent);
bool extent_slab_get(const extent_t *extent);
arena_slab_data_t *extent_slab_data_get(extent_t *extent);
const arena_slab_data_t *extent_slab_data_get_const(const extent_t *extent);
prof_tctx_t *extent_prof_tctx_get(const extent_t *extent);
void extent_arena_set(extent_t *extent, arena_t *arena);
void extent_addr_set(extent_t *extent, void *addr);
@@ -114,17 +118,15 @@ void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
void extent_size_set(extent_t *extent, size_t size);
void extent_usize_set(extent_t *extent, size_t usize);
void extent_active_set(extent_t *extent, bool active);
void extent_dirty_set(extent_t *extent, bool dirty);
void extent_zeroed_set(extent_t *extent, bool zeroed);
void extent_committed_set(extent_t *extent, bool committed);
void extent_slab_set(extent_t *extent, bool slab);
void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx);
void extent_init(extent_t *extent, arena_t *arena, void *addr,
size_t size, size_t usize, bool active, bool dirty, bool zeroed,
bool committed, bool slab);
void extent_dirty_insert(extent_t *extent,
arena_runs_dirty_link_t *runs_dirty, extent_t *chunks_dirty);
void extent_dirty_remove(extent_t *extent);
size_t size, size_t usize, bool active, bool zeroed, bool committed,
bool slab);
void extent_ring_insert(extent_t *sentinel, extent_t *extent);
void extent_ring_remove(extent_t *extent);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
@@ -197,18 +199,11 @@ extent_active_get(const extent_t *extent)
return (extent->e_active);
}
JEMALLOC_INLINE bool
extent_dirty_get(const extent_t *extent)
{
return (extent->e_dirty);
}
JEMALLOC_INLINE bool
extent_retained_get(const extent_t *extent)
{
return (qr_next(&extent->rd, rd_link) == &extent->rd);
return (qr_next(extent, qr_link) == extent);
}
JEMALLOC_INLINE bool
@@ -232,6 +227,22 @@ extent_slab_get(const extent_t *extent)
return (extent->e_slab);
}
JEMALLOC_INLINE arena_slab_data_t *
extent_slab_data_get(extent_t *extent)
{
assert(extent->e_slab);
return (&extent->e_slab_data);
}
JEMALLOC_INLINE const arena_slab_data_t *
extent_slab_data_get_const(const extent_t *extent)
{
assert(extent->e_slab);
return (&extent->e_slab_data);
}
JEMALLOC_INLINE prof_tctx_t *
extent_prof_tctx_get(const extent_t *extent)
{
@@ -296,13 +307,6 @@ extent_active_set(extent_t *extent, bool active)
extent->e_active = active;
}
JEMALLOC_INLINE void
extent_dirty_set(extent_t *extent, bool dirty)
{
extent->e_dirty = dirty;
}
JEMALLOC_INLINE void
extent_zeroed_set(extent_t *extent, bool zeroed)
{
@@ -333,8 +337,7 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx)
JEMALLOC_INLINE void
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
size_t usize, bool active, bool dirty, bool zeroed, bool committed,
bool slab)
size_t usize, bool active, bool zeroed, bool committed, bool slab)
{
assert(addr == PAGE_ADDR2BASE(addr) || !slab);
@@ -344,31 +347,26 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
extent_size_set(extent, size);
extent_usize_set(extent, usize);
extent_active_set(extent, active);
extent_dirty_set(extent, dirty);
extent_zeroed_set(extent, zeroed);
extent_committed_set(extent, committed);
extent_slab_set(extent, slab);
if (config_prof)
extent_prof_tctx_set(extent, NULL);
qr_new(&extent->rd, rd_link);
qr_new(extent, cc_link);
qr_new(extent, qr_link);
}
JEMALLOC_INLINE void
extent_dirty_insert(extent_t *extent,
arena_runs_dirty_link_t *runs_dirty, extent_t *chunks_dirty)
extent_ring_insert(extent_t *sentinel, extent_t *extent)
{
qr_meld(runs_dirty, &extent->rd, rd_link);
qr_meld(chunks_dirty, extent, cc_link);
qr_meld(sentinel, extent, qr_link);
}
JEMALLOC_INLINE void
extent_dirty_remove(extent_t *extent)
extent_ring_remove(extent_t *extent)
{
qr_remove(&extent->rd, rd_link);
qr_remove(extent, cc_link);
qr_remove(extent, qr_link);
}
#endif

45
include/jemalloc/internal/private_symbols.txt
View File

@@ -5,8 +5,6 @@ arena_alloc_junk_small
arena_basic_stats_merge
arena_bin_index
arena_bin_info
arena_bitselm_get_const
arena_bitselm_get_mutable
arena_boot
arena_choose
arena_choose_hard
@@ -43,38 +41,11 @@ arena_lg_dirty_mult_get
arena_lg_dirty_mult_set
arena_malloc
arena_malloc_hard
arena_mapbits_allocated_get
arena_mapbits_binind_get
arena_mapbits_decommitted_get
arena_mapbits_dirty_get
arena_mapbits_get
arena_mapbits_internal_set
arena_mapbits_large_binind_set
arena_mapbits_large_get
arena_mapbits_large_set
arena_mapbits_large_size_get
arena_mapbits_size_decode
arena_mapbits_size_encode
arena_mapbits_small_runind_get
arena_mapbits_small_set
arena_mapbits_unallocated_set
arena_mapbits_unallocated_size_get
arena_mapbits_unallocated_size_set
arena_mapbits_unzeroed_get
arena_mapbitsp_get_const
arena_mapbitsp_get_mutable
arena_mapbitsp_read
arena_mapbitsp_write
arena_maxrun
arena_maybe_purge
arena_metadata_allocated_add
arena_metadata_allocated_get
arena_metadata_allocated_sub
arena_migrate
arena_miscelm_get_const
arena_miscelm_get_mutable
arena_miscelm_to_pageind
arena_miscelm_to_rpages
arena_new
arena_nthreads_dec
arena_nthreads_get
@@ -93,14 +64,11 @@ arena_prof_promote
arena_prof_tctx_get
arena_prof_tctx_reset
arena_prof_tctx_set
arena_ptr_small_binind_get
arena_purge
arena_ralloc
arena_ralloc_junk_large
arena_ralloc_no_move
arena_rd_to_miscelm
arena_reset
arena_run_to_miscelm
arena_salloc
arena_sdalloc
arena_stats_merge
@@ -213,22 +181,23 @@ extent_before_get
extent_committed_get
extent_committed_set
extent_dalloc
extent_dirty_get
extent_dirty_insert
extent_dirty_remove
extent_dirty_set
extent_init
extent_last_get
extent_past_get
extent_prof_tctx_get
extent_prof_tctx_set
extent_retained_get
extent_ring_insert
extent_ring_remove
extent_size_get
extent_size_set
extent_size_quantize_ceil
extent_size_quantize_floor
extent_slab_data_get
extent_slab_data_get_const
extent_slab_get
extent_slab_set
extent_slab_data_get
extent_usize_get
extent_zeroed_get
extent_zeroed_set
@@ -309,8 +278,6 @@ malloc_tsd_no_cleanup
malloc_vcprintf
malloc_vsnprintf
malloc_write
map_bias
map_misc_offset
mb_write
narenas_auto
narenas_tdata_cleanup
@@ -451,8 +418,6 @@ rtree_subtree_read
rtree_subtree_read_hard
rtree_subtree_tryread
rtree_write
run_quantize_ceil
run_quantize_floor
s2u
s2u_compute
s2u_lookup

44
include/jemalloc/internal/prof.h
View File

@@ -335,8 +335,8 @@ prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
void prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx);
void prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, const void *old_ptr, prof_tctx_t *tctx);
bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
prof_tctx_t *tctx);
bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
prof_tdata_t **tdata_out);
prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
bool update);
@@ -344,7 +344,8 @@ void prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx);
void prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx, bool prof_active, bool updated,
const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx);
extent_t *old_extent, const void *old_ptr, size_t old_usize,
prof_tctx_t *old_tctx);
void prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr,
size_t usize);
#endif
@@ -421,14 +422,14 @@ prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
}
JEMALLOC_ALWAYS_INLINE void
prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
const void *old_ptr, prof_tctx_t *old_tctx)
prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
prof_tctx_t *tctx)
{
cassert(config_prof);
assert(ptr != NULL);
arena_prof_tctx_reset(tsdn, extent, ptr, usize, old_ptr, old_tctx);
arena_prof_tctx_reset(tsdn, extent, ptr, tctx);
}
JEMALLOC_ALWAYS_INLINE bool
@@ -501,10 +502,10 @@ prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
JEMALLOC_ALWAYS_INLINE void
prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
size_t old_usize, prof_tctx_t *old_tctx)
prof_tctx_t *tctx, bool prof_active, bool updated, extent_t *old_extent,
const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx)
{
bool sampled, old_sampled;
bool sampled, old_sampled, moved;
cassert(config_prof);
assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
@@ -523,19 +524,30 @@ prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
}
}
/*
* The following code must differentiate among eight possible cases,
* based on three boolean conditions.
*/
sampled = ((uintptr_t)tctx > (uintptr_t)1U);
old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);
moved = (ptr != old_ptr);
/*
* The following block must only execute if this is a non-moving
* reallocation, because for moving reallocation the old allocation will
* be deallocated via a separate call.
*/
if (unlikely(old_sampled) && !moved)
prof_free_sampled_object(tsd, old_usize, old_tctx);
if (unlikely(sampled)) {
prof_malloc_sample_object(tsd_tsdn(tsd), extent, ptr, usize,
tctx);
} else {
prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, usize, old_ptr,
old_tctx);
}
if (unlikely(old_sampled))
prof_free_sampled_object(tsd, old_usize, old_tctx);
} else if (moved) {
prof_tctx_set(tsd_tsdn(tsd), extent, ptr, usize,
(prof_tctx_t *)(uintptr_t)1U);
} else if (unlikely(old_sampled))
prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, tctx);
}
JEMALLOC_ALWAYS_INLINE void

22
include/jemalloc/internal/size_classes.sh
View File

@@ -50,7 +50,7 @@ reg_size_compute() {
reg_size=$((${grp} + ${delta}*${ndelta}))
}
run_size() {
slab_size() {
lg_p=$1
lg_grp=$2
lg_delta=$3
@@ -59,22 +59,22 @@ run_size() {
pow2 ${lg_p}; p=${pow2_result}
reg_size_compute ${lg_grp} ${lg_delta} ${ndelta}
# Compute smallest run size that is an integer multiple of reg_size.
try_run_size=${p}
try_nregs=$((${try_run_size} / ${reg_size}))
# Compute smallest slab size that is an integer multiple of reg_size.
try_slab_size=${p}
try_nregs=$((${try_slab_size} / ${reg_size}))
perfect=0
while [ ${perfect} -eq 0 ] ; do
perfect_run_size=${try_run_size}
perfect_slab_size=${try_slab_size}
perfect_nregs=${try_nregs}
try_run_size=$((${try_run_size} + ${p}))
try_nregs=$((${try_run_size} / ${reg_size}))
if [ ${perfect_run_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then
try_slab_size=$((${try_slab_size} + ${p}))
try_nregs=$((${try_slab_size} / ${reg_size}))
if [ ${perfect_slab_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then
perfect=1
fi
done
run_size_pgs=$((${perfect_run_size} / ${p}))
slab_size_pgs=$((${perfect_slab_size} / ${p}))
}
size_class() {
@@ -117,7 +117,7 @@ size_class() {
if [ ${lg_size} -lt $((${lg_p} + ${lg_g})) ] ; then
bin="yes"
run_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${run_size_pgs}
slab_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${slab_size_pgs}
else
bin="no"
pgs=0
@@ -278,7 +278,7 @@ cat <<EOF
* ndelta: Delta multiplier. size == 1<<lg_grp + ndelta<<lg_delta
* psz: 'yes' if a multiple of the page size, 'no' otherwise.
* bin: 'yes' if a small bin size class, 'no' otherwise.
* pgs: Run page count if a small bin size class, 0 otherwise.
* pgs: Slab page count if a small bin size class, 0 otherwise.
* lg_delta_lookup: Same as lg_delta if a lookup table size class, 'no'
* otherwise.
* NTBINS: Number of tiny bins.

14
include/jemalloc/internal/stats.h
View File

@@ -48,17 +48,17 @@ struct malloc_bin_stats_s {
/* Number of tcache flushes to this bin. */
uint64_t nflushes;
/* Total number of runs created for this bin's size class. */
uint64_t nruns;
/* Total number of slabs created for this bin's size class. */
uint64_t nslabs;
/*
* Total number of runs reused by extracting them from the runs tree for
* this bin's size class.
* Total number of slabs reused by extracting them from the slabs heap
* for this bin's size class.
*/
uint64_t reruns;
uint64_t reslabs;
/* Current number of runs in this bin. */
size_t curruns;
/* Current number of slabs in this bin. */
size_t curslabs;
};
struct malloc_huge_stats_s {

2
include/jemalloc/internal/tcache.h
View File

@@ -24,7 +24,7 @@ typedef struct tcaches_s tcaches_t;
/*
* Absolute maximum number of cache slots for each small bin in the thread
* cache. This is an additional constraint beyond that imposed as: twice the
* number of regions per run for this size class.
* number of regions per slab for this size class.
*
* This constant must be an even number.
*/