Fix huge allocation statistics.
This commit is contained in:
parent
0cdabd2d48
commit
9b41ac909f
@ -1719,9 +1719,8 @@ malloc_conf = "xmalloc:true";]]></programlisting>
|
|||||||
</term>
|
</term>
|
||||||
<listitem><para>Pointer to a counter that contains an approximate count
|
<listitem><para>Pointer to a counter that contains an approximate count
|
||||||
of the current number of bytes in active pages. The estimate may be
|
of the current number of bytes in active pages. The estimate may be
|
||||||
high, but never low, because each arena rounds up to the nearest
|
high, but never low, because each arena rounds up when computing its
|
||||||
multiple of the chunk size when computing its contribution to the
|
contribution to the counter. Note that the <link
|
||||||
counter. Note that the <link
|
|
||||||
linkend="epoch"><mallctl>epoch</mallctl></link> mallctl has no bearing
|
linkend="epoch"><mallctl>epoch</mallctl></link> mallctl has no bearing
|
||||||
on this counter. Furthermore, counter consistency is maintained via
|
on this counter. Furthermore, counter consistency is maintained via
|
||||||
atomic operations, so it is necessary to use an atomic operation in
|
atomic operations, so it is necessary to use an atomic operation in
|
||||||
|
@ -338,9 +338,15 @@ extern size_t arena_maxclass; /* Max size class for arenas. */
|
|||||||
extern unsigned nlclasses; /* Number of large size classes. */
|
extern unsigned nlclasses; /* Number of large size classes. */
|
||||||
extern unsigned nhclasses; /* Number of huge size classes. */
|
extern unsigned nhclasses; /* Number of huge size classes. */
|
||||||
|
|
||||||
void *arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t usize,
|
void *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment,
|
||||||
size_t alignment, bool *zero);
|
bool *zero);
|
||||||
void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize);
|
void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize);
|
||||||
|
void arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk,
|
||||||
|
size_t oldsize, size_t usize);
|
||||||
|
void arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk,
|
||||||
|
size_t oldsize, size_t usize);
|
||||||
|
bool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk,
|
||||||
|
size_t oldsize, size_t usize, bool *zero);
|
||||||
void arena_purge_all(arena_t *arena);
|
void arena_purge_all(arena_t *arena);
|
||||||
void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
|
void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
|
||||||
index_t binind, uint64_t prof_accumbytes);
|
index_t binind, uint64_t prof_accumbytes);
|
||||||
|
@ -13,6 +13,9 @@ arena_choose
|
|||||||
arena_choose_hard
|
arena_choose_hard
|
||||||
arena_chunk_alloc_huge
|
arena_chunk_alloc_huge
|
||||||
arena_chunk_dalloc_huge
|
arena_chunk_dalloc_huge
|
||||||
|
arena_chunk_ralloc_huge_expand
|
||||||
|
arena_chunk_ralloc_huge_shrink
|
||||||
|
arena_chunk_ralloc_huge_similar
|
||||||
arena_cleanup
|
arena_cleanup
|
||||||
arena_dalloc
|
arena_dalloc
|
||||||
arena_dalloc_bin
|
arena_dalloc_bin
|
||||||
|
303
src/arena.c
303
src/arena.c
@ -411,52 +411,6 @@ arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment,
|
|||||||
return (chunk);
|
return (chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
void *
|
|
||||||
arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t usize,
|
|
||||||
size_t alignment, bool *zero)
|
|
||||||
{
|
|
||||||
void *ret;
|
|
||||||
chunk_alloc_t *chunk_alloc;
|
|
||||||
chunk_dalloc_t *chunk_dalloc;
|
|
||||||
|
|
||||||
malloc_mutex_lock(&arena->lock);
|
|
||||||
chunk_alloc = arena->chunk_alloc;
|
|
||||||
chunk_dalloc = arena->chunk_dalloc;
|
|
||||||
if (config_stats) {
|
|
||||||
index_t index = size2index(usize) - nlclasses - NBINS;
|
|
||||||
|
|
||||||
/* Optimistically update stats prior to unlocking. */
|
|
||||||
arena->stats.allocated_huge += usize;
|
|
||||||
arena->stats.nmalloc_huge++;
|
|
||||||
arena->stats.hstats[index].nmalloc++;
|
|
||||||
arena->stats.hstats[index].curhchunks++;
|
|
||||||
arena->stats.mapped += usize;
|
|
||||||
}
|
|
||||||
arena->nactive += (usize >> LG_PAGE);
|
|
||||||
malloc_mutex_unlock(&arena->lock);
|
|
||||||
|
|
||||||
ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind,
|
|
||||||
new_addr, usize, alignment, zero);
|
|
||||||
if (config_stats) {
|
|
||||||
if (ret != NULL)
|
|
||||||
stats_cactive_add(usize);
|
|
||||||
else {
|
|
||||||
index_t index = size2index(usize) - nlclasses - NBINS;
|
|
||||||
|
|
||||||
malloc_mutex_lock(&arena->lock);
|
|
||||||
/* Revert optimistic stats updates. */
|
|
||||||
arena->stats.allocated_huge -= usize;
|
|
||||||
arena->stats.nmalloc_huge--;
|
|
||||||
arena->stats.hstats[index].nmalloc--;
|
|
||||||
arena->stats.hstats[index].curhchunks--;
|
|
||||||
arena->stats.mapped -= usize;
|
|
||||||
malloc_mutex_unlock(&arena->lock);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return (ret);
|
|
||||||
}
|
|
||||||
|
|
||||||
static arena_chunk_t *
|
static arena_chunk_t *
|
||||||
arena_chunk_init_hard(arena_t *arena)
|
arena_chunk_init_hard(arena_t *arena)
|
||||||
{
|
{
|
||||||
@ -528,41 +482,6 @@ arena_chunk_alloc(arena_t *arena)
|
|||||||
return (chunk);
|
return (chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
|
||||||
arena_chunk_dalloc_internal(arena_t *arena, arena_chunk_t *chunk)
|
|
||||||
{
|
|
||||||
chunk_dalloc_t *chunk_dalloc;
|
|
||||||
|
|
||||||
chunk_dalloc = arena->chunk_dalloc;
|
|
||||||
malloc_mutex_unlock(&arena->lock);
|
|
||||||
chunk_dalloc((void *)chunk, chunksize, arena->ind);
|
|
||||||
malloc_mutex_lock(&arena->lock);
|
|
||||||
if (config_stats)
|
|
||||||
arena->stats.mapped -= chunksize;
|
|
||||||
}
|
|
||||||
|
|
||||||
void
|
|
||||||
arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize)
|
|
||||||
{
|
|
||||||
chunk_dalloc_t *chunk_dalloc;
|
|
||||||
|
|
||||||
malloc_mutex_lock(&arena->lock);
|
|
||||||
chunk_dalloc = arena->chunk_dalloc;
|
|
||||||
if (config_stats) {
|
|
||||||
index_t index = size2index(usize) - nlclasses - NBINS;
|
|
||||||
|
|
||||||
arena->stats.ndalloc_huge++;
|
|
||||||
arena->stats.allocated_huge -= usize;
|
|
||||||
arena->stats.hstats[index].ndalloc++;
|
|
||||||
arena->stats.hstats[index].curhchunks--;
|
|
||||||
arena->stats.mapped -= usize;
|
|
||||||
stats_cactive_sub(usize);
|
|
||||||
}
|
|
||||||
arena->nactive -= (usize >> LG_PAGE);
|
|
||||||
malloc_mutex_unlock(&arena->lock);
|
|
||||||
chunk_dalloc(chunk, usize, arena->ind);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
static void
|
||||||
arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
|
arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
|
||||||
{
|
{
|
||||||
@ -584,17 +503,237 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
|
|||||||
|
|
||||||
if (arena->spare != NULL) {
|
if (arena->spare != NULL) {
|
||||||
arena_chunk_t *spare = arena->spare;
|
arena_chunk_t *spare = arena->spare;
|
||||||
|
chunk_dalloc_t *chunk_dalloc;
|
||||||
|
|
||||||
arena->spare = chunk;
|
arena->spare = chunk;
|
||||||
if (arena_mapbits_dirty_get(spare, map_bias) != 0) {
|
if (arena_mapbits_dirty_get(spare, map_bias) != 0) {
|
||||||
arena_dirty_remove(arena, spare, map_bias,
|
arena_dirty_remove(arena, spare, map_bias,
|
||||||
chunk_npages-map_bias);
|
chunk_npages-map_bias);
|
||||||
}
|
}
|
||||||
arena_chunk_dalloc_internal(arena, spare);
|
chunk_dalloc = arena->chunk_dalloc;
|
||||||
|
malloc_mutex_unlock(&arena->lock);
|
||||||
|
chunk_dalloc((void *)spare, chunksize, arena->ind);
|
||||||
|
malloc_mutex_lock(&arena->lock);
|
||||||
|
if (config_stats)
|
||||||
|
arena->stats.mapped -= chunksize;
|
||||||
} else
|
} else
|
||||||
arena->spare = chunk;
|
arena->spare = chunk;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
arena_huge_malloc_stats_update(arena_t *arena, size_t usize)
|
||||||
|
{
|
||||||
|
index_t index = size2index(usize) - nlclasses - NBINS;
|
||||||
|
|
||||||
|
cassert(config_stats);
|
||||||
|
|
||||||
|
arena->stats.nmalloc_huge++;
|
||||||
|
arena->stats.allocated_huge += usize;
|
||||||
|
arena->stats.hstats[index].nmalloc++;
|
||||||
|
arena->stats.hstats[index].curhchunks++;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize)
|
||||||
|
{
|
||||||
|
index_t index = size2index(usize) - nlclasses - NBINS;
|
||||||
|
|
||||||
|
cassert(config_stats);
|
||||||
|
|
||||||
|
arena->stats.nmalloc_huge--;
|
||||||
|
arena->stats.allocated_huge -= usize;
|
||||||
|
arena->stats.hstats[index].nmalloc--;
|
||||||
|
arena->stats.hstats[index].curhchunks--;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
arena_huge_dalloc_stats_update(arena_t *arena, size_t usize)
|
||||||
|
{
|
||||||
|
index_t index = size2index(usize) - nlclasses - NBINS;
|
||||||
|
|
||||||
|
cassert(config_stats);
|
||||||
|
|
||||||
|
arena->stats.ndalloc_huge++;
|
||||||
|
arena->stats.allocated_huge -= usize;
|
||||||
|
arena->stats.hstats[index].ndalloc++;
|
||||||
|
arena->stats.hstats[index].curhchunks--;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize)
|
||||||
|
{
|
||||||
|
index_t index = size2index(usize) - nlclasses - NBINS;
|
||||||
|
|
||||||
|
cassert(config_stats);
|
||||||
|
|
||||||
|
arena->stats.ndalloc_huge--;
|
||||||
|
arena->stats.allocated_huge += usize;
|
||||||
|
arena->stats.hstats[index].ndalloc--;
|
||||||
|
arena->stats.hstats[index].curhchunks++;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize)
|
||||||
|
{
|
||||||
|
|
||||||
|
arena_huge_dalloc_stats_update(arena, oldsize);
|
||||||
|
arena_huge_malloc_stats_update(arena, usize);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize,
|
||||||
|
size_t usize)
|
||||||
|
{
|
||||||
|
|
||||||
|
arena_huge_dalloc_stats_update_undo(arena, oldsize);
|
||||||
|
arena_huge_malloc_stats_update_undo(arena, usize);
|
||||||
|
}
|
||||||
|
|
||||||
|
void *
|
||||||
|
arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment,
|
||||||
|
bool *zero)
|
||||||
|
{
|
||||||
|
void *ret;
|
||||||
|
chunk_alloc_t *chunk_alloc;
|
||||||
|
chunk_dalloc_t *chunk_dalloc;
|
||||||
|
size_t csize = CHUNK_CEILING(usize);
|
||||||
|
|
||||||
|
malloc_mutex_lock(&arena->lock);
|
||||||
|
chunk_alloc = arena->chunk_alloc;
|
||||||
|
chunk_dalloc = arena->chunk_dalloc;
|
||||||
|
if (config_stats) {
|
||||||
|
/* Optimistically update stats prior to unlocking. */
|
||||||
|
arena_huge_malloc_stats_update(arena, usize);
|
||||||
|
arena->stats.mapped += usize;
|
||||||
|
}
|
||||||
|
arena->nactive += (usize >> LG_PAGE);
|
||||||
|
malloc_mutex_unlock(&arena->lock);
|
||||||
|
|
||||||
|
ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, NULL,
|
||||||
|
csize, alignment, zero);
|
||||||
|
if (ret == NULL) {
|
||||||
|
/* Revert optimistic stats updates. */
|
||||||
|
malloc_mutex_lock(&arena->lock);
|
||||||
|
if (config_stats) {
|
||||||
|
arena_huge_malloc_stats_update_undo(arena, usize);
|
||||||
|
arena->stats.mapped -= usize;
|
||||||
|
}
|
||||||
|
arena->nactive -= (usize >> LG_PAGE);
|
||||||
|
malloc_mutex_unlock(&arena->lock);
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config_stats)
|
||||||
|
stats_cactive_add(usize);
|
||||||
|
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize)
|
||||||
|
{
|
||||||
|
chunk_dalloc_t *chunk_dalloc;
|
||||||
|
|
||||||
|
malloc_mutex_lock(&arena->lock);
|
||||||
|
chunk_dalloc = arena->chunk_dalloc;
|
||||||
|
if (config_stats) {
|
||||||
|
arena_huge_dalloc_stats_update(arena, usize);
|
||||||
|
arena->stats.mapped -= usize;
|
||||||
|
stats_cactive_sub(usize);
|
||||||
|
}
|
||||||
|
arena->nactive -= (usize >> LG_PAGE);
|
||||||
|
malloc_mutex_unlock(&arena->lock);
|
||||||
|
chunk_dalloc(chunk, CHUNK_CEILING(usize), arena->ind);
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, size_t oldsize,
|
||||||
|
size_t usize)
|
||||||
|
{
|
||||||
|
|
||||||
|
assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize));
|
||||||
|
assert(oldsize != usize);
|
||||||
|
|
||||||
|
malloc_mutex_lock(&arena->lock);
|
||||||
|
if (config_stats)
|
||||||
|
arena_huge_ralloc_stats_update(arena, oldsize, usize);
|
||||||
|
if (oldsize < usize) {
|
||||||
|
size_t udiff = usize - oldsize;
|
||||||
|
arena->nactive += udiff >> LG_PAGE;
|
||||||
|
if (config_stats)
|
||||||
|
stats_cactive_add(udiff);
|
||||||
|
} else {
|
||||||
|
size_t udiff = oldsize - usize;
|
||||||
|
arena->nactive -= udiff >> LG_PAGE;
|
||||||
|
if (config_stats)
|
||||||
|
stats_cactive_sub(udiff);
|
||||||
|
}
|
||||||
|
malloc_mutex_unlock(&arena->lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize,
|
||||||
|
size_t usize)
|
||||||
|
{
|
||||||
|
chunk_dalloc_t *chunk_dalloc;
|
||||||
|
size_t udiff = oldsize - usize;
|
||||||
|
size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
|
||||||
|
|
||||||
|
malloc_mutex_lock(&arena->lock);
|
||||||
|
chunk_dalloc = arena->chunk_dalloc;
|
||||||
|
if (config_stats) {
|
||||||
|
arena_huge_ralloc_stats_update(arena, oldsize, usize);
|
||||||
|
if (cdiff != 0) {
|
||||||
|
arena->stats.mapped -= cdiff;
|
||||||
|
stats_cactive_sub(udiff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
arena->nactive -= udiff >> LG_PAGE;
|
||||||
|
malloc_mutex_unlock(&arena->lock);
|
||||||
|
if (cdiff != 0)
|
||||||
|
chunk_dalloc(chunk + CHUNK_CEILING(usize), cdiff, arena->ind);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool
|
||||||
|
arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize,
|
||||||
|
size_t usize, bool *zero)
|
||||||
|
{
|
||||||
|
chunk_alloc_t *chunk_alloc;
|
||||||
|
chunk_dalloc_t *chunk_dalloc;
|
||||||
|
size_t udiff = usize - oldsize;
|
||||||
|
size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
|
||||||
|
|
||||||
|
malloc_mutex_lock(&arena->lock);
|
||||||
|
chunk_alloc = arena->chunk_alloc;
|
||||||
|
chunk_dalloc = arena->chunk_dalloc;
|
||||||
|
if (config_stats) {
|
||||||
|
/* Optimistically update stats prior to unlocking. */
|
||||||
|
arena_huge_ralloc_stats_update(arena, oldsize, usize);
|
||||||
|
arena->stats.mapped += cdiff;
|
||||||
|
}
|
||||||
|
arena->nactive += (udiff >> LG_PAGE);
|
||||||
|
malloc_mutex_unlock(&arena->lock);
|
||||||
|
|
||||||
|
if (chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, chunk +
|
||||||
|
CHUNK_CEILING(oldsize), cdiff, chunksize, zero) == NULL) {
|
||||||
|
/* Revert optimistic stats updates. */
|
||||||
|
malloc_mutex_lock(&arena->lock);
|
||||||
|
if (config_stats) {
|
||||||
|
arena_huge_ralloc_stats_update_undo(arena,
|
||||||
|
oldsize, usize);
|
||||||
|
arena->stats.mapped -= cdiff;
|
||||||
|
}
|
||||||
|
arena->nactive -= (udiff >> LG_PAGE);
|
||||||
|
malloc_mutex_unlock(&arena->lock);
|
||||||
|
return (true);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config_stats)
|
||||||
|
stats_cactive_add(udiff);
|
||||||
|
|
||||||
|
return (false);
|
||||||
|
}
|
||||||
|
|
||||||
static arena_run_t *
|
static arena_run_t *
|
||||||
arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
|
arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
|
||||||
{
|
{
|
||||||
|
95
src/huge.c
95
src/huge.c
@ -31,15 +31,11 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
|
|||||||
bool zero, bool try_tcache)
|
bool zero, bool try_tcache)
|
||||||
{
|
{
|
||||||
void *ret;
|
void *ret;
|
||||||
size_t csize;
|
|
||||||
extent_node_t *node;
|
extent_node_t *node;
|
||||||
bool is_zeroed;
|
bool is_zeroed;
|
||||||
|
|
||||||
/* Allocate one or more contiguous chunks for this request. */
|
/* Allocate one or more contiguous chunks for this request. */
|
||||||
|
|
||||||
csize = CHUNK_CEILING(usize);
|
|
||||||
assert(csize >= usize);
|
|
||||||
|
|
||||||
/* Allocate an extent node with which to track the chunk. */
|
/* Allocate an extent node with which to track the chunk. */
|
||||||
node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
|
node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
|
||||||
CACHELINE, false, try_tcache, NULL);
|
CACHELINE, false, try_tcache, NULL);
|
||||||
@ -56,7 +52,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
|
|||||||
base_node_dalloc(node);
|
base_node_dalloc(node);
|
||||||
return (NULL);
|
return (NULL);
|
||||||
}
|
}
|
||||||
ret = arena_chunk_alloc_huge(arena, NULL, csize, alignment, &is_zeroed);
|
ret = arena_chunk_alloc_huge(arena, usize, alignment, &is_zeroed);
|
||||||
if (ret == NULL) {
|
if (ret == NULL) {
|
||||||
idalloct(tsd, node, try_tcache);
|
idalloct(tsd, node, try_tcache);
|
||||||
return (NULL);
|
return (NULL);
|
||||||
@ -104,25 +100,6 @@ huge_dalloc_junk(void *ptr, size_t usize)
|
|||||||
huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
|
huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
static void
|
|
||||||
huge_ralloc_no_move_stats_update(arena_t *arena, size_t oldsize, size_t usize)
|
|
||||||
{
|
|
||||||
index_t oldindex = size2index(oldsize) - nlclasses - NBINS;
|
|
||||||
index_t index = size2index(usize) - nlclasses - NBINS;
|
|
||||||
|
|
||||||
cassert(config_stats);
|
|
||||||
|
|
||||||
arena->stats.ndalloc_huge++;
|
|
||||||
arena->stats.allocated_huge -= oldsize;
|
|
||||||
arena->stats.hstats[oldindex].ndalloc++;
|
|
||||||
arena->stats.hstats[oldindex].curhchunks--;
|
|
||||||
|
|
||||||
arena->stats.nmalloc_huge++;
|
|
||||||
arena->stats.allocated_huge += usize;
|
|
||||||
arena->stats.hstats[index].nmalloc++;
|
|
||||||
arena->stats.hstats[index].curhchunks++;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
static void
|
||||||
huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize,
|
huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize,
|
||||||
size_t size, size_t extra, bool zero)
|
size_t size, size_t extra, bool zero)
|
||||||
@ -135,34 +112,33 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize,
|
|||||||
while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < oldsize)
|
while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < oldsize)
|
||||||
usize = usize_next;
|
usize = usize_next;
|
||||||
|
|
||||||
malloc_mutex_lock(&huge_mtx);
|
if (oldsize == usize)
|
||||||
|
return;
|
||||||
|
|
||||||
|
malloc_mutex_lock(&huge_mtx);
|
||||||
key.addr = ptr;
|
key.addr = ptr;
|
||||||
node = extent_tree_ad_search(&huge, &key);
|
node = extent_tree_ad_search(&huge, &key);
|
||||||
assert(node != NULL);
|
assert(node != NULL);
|
||||||
assert(node->addr == ptr);
|
assert(node->addr == ptr);
|
||||||
|
|
||||||
arena = node->arena;
|
arena = node->arena;
|
||||||
|
/* Update the size of the huge allocation. */
|
||||||
/* Update the size of the huge allocation if it changed. */
|
assert(node->size != usize);
|
||||||
if (oldsize != usize) {
|
node->size = usize;
|
||||||
assert(node->size != usize);
|
|
||||||
node->size = usize;
|
|
||||||
}
|
|
||||||
|
|
||||||
malloc_mutex_unlock(&huge_mtx);
|
malloc_mutex_unlock(&huge_mtx);
|
||||||
|
|
||||||
/* Fill if necessary. */
|
/* Fill if necessary (shrinking). */
|
||||||
|
if (config_fill && unlikely(opt_junk) && oldsize > usize)
|
||||||
|
memset(ptr + usize, 0x5a, oldsize - usize);
|
||||||
|
|
||||||
|
arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize);
|
||||||
|
|
||||||
|
/* Fill if necessary (growing). */
|
||||||
if (oldsize < usize) {
|
if (oldsize < usize) {
|
||||||
if (zero || (config_fill && unlikely(opt_zero)))
|
if (zero || (config_fill && unlikely(opt_zero)))
|
||||||
memset(ptr + oldsize, 0, usize - oldsize);
|
memset(ptr + oldsize, 0, usize - oldsize);
|
||||||
else if (config_fill && unlikely(opt_junk))
|
else if (config_fill && unlikely(opt_junk))
|
||||||
memset(ptr + oldsize, 0xa5, usize - oldsize);
|
memset(ptr + oldsize, 0xa5, usize - oldsize);
|
||||||
} else if (config_fill && unlikely(opt_junk) && oldsize > usize)
|
}
|
||||||
memset(ptr + usize, 0x5a, oldsize - usize);
|
|
||||||
|
|
||||||
if (config_stats)
|
|
||||||
huge_ralloc_no_move_stats_update(arena, oldsize, usize);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
@ -170,44 +146,28 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
|
|||||||
{
|
{
|
||||||
extent_node_t *node, key;
|
extent_node_t *node, key;
|
||||||
arena_t *arena;
|
arena_t *arena;
|
||||||
void *excess_addr;
|
|
||||||
size_t excess_size;
|
|
||||||
|
|
||||||
malloc_mutex_lock(&huge_mtx);
|
malloc_mutex_lock(&huge_mtx);
|
||||||
|
|
||||||
key.addr = ptr;
|
key.addr = ptr;
|
||||||
node = extent_tree_ad_search(&huge, &key);
|
node = extent_tree_ad_search(&huge, &key);
|
||||||
assert(node != NULL);
|
assert(node != NULL);
|
||||||
assert(node->addr == ptr);
|
assert(node->addr == ptr);
|
||||||
|
|
||||||
arena = node->arena;
|
arena = node->arena;
|
||||||
|
|
||||||
/* Update the size of the huge allocation. */
|
/* Update the size of the huge allocation. */
|
||||||
node->size = usize;
|
node->size = usize;
|
||||||
|
|
||||||
malloc_mutex_unlock(&huge_mtx);
|
malloc_mutex_unlock(&huge_mtx);
|
||||||
|
|
||||||
excess_addr = node->addr + CHUNK_CEILING(usize);
|
|
||||||
excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
|
|
||||||
|
|
||||||
/* Zap the excess chunks. */
|
/* Zap the excess chunks. */
|
||||||
huge_dalloc_junk(ptr + usize, oldsize - usize);
|
huge_dalloc_junk(ptr + usize, oldsize - usize);
|
||||||
if (excess_size > 0)
|
arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize);
|
||||||
arena_chunk_dalloc_huge(arena, excess_addr, excess_size);
|
|
||||||
|
|
||||||
if (config_stats)
|
|
||||||
huge_ralloc_no_move_stats_update(arena, oldsize, usize);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool
|
static bool
|
||||||
huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
|
huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
|
||||||
size_t usize;
|
size_t usize;
|
||||||
void *expand_addr;
|
|
||||||
size_t expand_size;
|
|
||||||
extent_node_t *node, key;
|
extent_node_t *node, key;
|
||||||
arena_t *arena;
|
arena_t *arena;
|
||||||
bool is_zeroed;
|
bool is_zeroed;
|
||||||
void *ret;
|
|
||||||
|
|
||||||
usize = s2u(size);
|
usize = s2u(size);
|
||||||
if (usize == 0) {
|
if (usize == 0) {
|
||||||
@ -215,19 +175,12 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
|
|||||||
return (true);
|
return (true);
|
||||||
}
|
}
|
||||||
|
|
||||||
expand_addr = ptr + CHUNK_CEILING(oldsize);
|
|
||||||
expand_size = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
|
|
||||||
assert(expand_size > 0);
|
|
||||||
|
|
||||||
malloc_mutex_lock(&huge_mtx);
|
malloc_mutex_lock(&huge_mtx);
|
||||||
|
|
||||||
key.addr = ptr;
|
key.addr = ptr;
|
||||||
node = extent_tree_ad_search(&huge, &key);
|
node = extent_tree_ad_search(&huge, &key);
|
||||||
assert(node != NULL);
|
assert(node != NULL);
|
||||||
assert(node->addr == ptr);
|
assert(node->addr == ptr);
|
||||||
|
|
||||||
arena = node->arena;
|
arena = node->arena;
|
||||||
|
|
||||||
malloc_mutex_unlock(&huge_mtx);
|
malloc_mutex_unlock(&huge_mtx);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -235,12 +188,10 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
|
|||||||
* it is possible to make correct junk/zero fill decisions below.
|
* it is possible to make correct junk/zero fill decisions below.
|
||||||
*/
|
*/
|
||||||
is_zeroed = zero;
|
is_zeroed = zero;
|
||||||
ret = arena_chunk_alloc_huge(arena, expand_addr, expand_size, chunksize,
|
|
||||||
&is_zeroed);
|
|
||||||
if (ret == NULL)
|
|
||||||
return (true);
|
|
||||||
|
|
||||||
assert(ret == expand_addr);
|
if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize,
|
||||||
|
&is_zeroed))
|
||||||
|
return (true);
|
||||||
|
|
||||||
malloc_mutex_lock(&huge_mtx);
|
malloc_mutex_lock(&huge_mtx);
|
||||||
/* Update the size of the huge allocation. */
|
/* Update the size of the huge allocation. */
|
||||||
@ -254,9 +205,6 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
|
|||||||
memset(ptr + oldsize, 0, usize - oldsize);
|
memset(ptr + oldsize, 0, usize - oldsize);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config_stats)
|
|
||||||
huge_ralloc_no_move_stats_update(arena, oldsize, usize);
|
|
||||||
|
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -363,19 +311,16 @@ huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache)
|
|||||||
extent_node_t *node, key;
|
extent_node_t *node, key;
|
||||||
|
|
||||||
malloc_mutex_lock(&huge_mtx);
|
malloc_mutex_lock(&huge_mtx);
|
||||||
|
|
||||||
/* Extract from tree of huge allocations. */
|
/* Extract from tree of huge allocations. */
|
||||||
key.addr = ptr;
|
key.addr = ptr;
|
||||||
node = extent_tree_ad_search(&huge, &key);
|
node = extent_tree_ad_search(&huge, &key);
|
||||||
assert(node != NULL);
|
assert(node != NULL);
|
||||||
assert(node->addr == ptr);
|
assert(node->addr == ptr);
|
||||||
extent_tree_ad_remove(&huge, node);
|
extent_tree_ad_remove(&huge, node);
|
||||||
|
|
||||||
malloc_mutex_unlock(&huge_mtx);
|
malloc_mutex_unlock(&huge_mtx);
|
||||||
|
|
||||||
huge_dalloc_junk(node->addr, node->size);
|
huge_dalloc_junk(node->addr, node->size);
|
||||||
arena_chunk_dalloc_huge(node->arena, node->addr,
|
arena_chunk_dalloc_huge(node->arena, node->addr, node->size);
|
||||||
CHUNK_CEILING(node->size));
|
|
||||||
idalloct(tsd, node, try_tcache);
|
idalloct(tsd, node, try_tcache);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user