project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement per arena base allocators.

Browse Source 
Add/rename related mallctls:
- Add stats.arenas.<i>.base .
- Rename stats.arenas.<i>.metadata to stats.arenas.<i>.internal .
- Add stats.arenas.<i>.resident .

Modify the arenas.extend mallctl to take an optional (extent_hooks_t *)
argument so that it is possible for all base allocations to be serviced
by the specified extent hooks.

This resolves #463.
This commit is contained in:
Jason Evans
2016-12-22 16:39:10 -06:00
parent a6e86810d8
commit a0dd3a4483
18 changed files with 957 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/arena.c
View File

@@ -1550,6 +1550,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_stats_t *astats, malloc_bin_stats_t *bstats,
malloc_large_stats_t *lstats)
{
size_t base_allocated, base_resident, base_mapped;
unsigned i;
cassert(config_stats);
@@ -1558,12 +1559,18 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_basic_stats_merge_locked(arena, nthreads, dss, decay_time,
nactive, ndirty);
astats->mapped += arena->stats.mapped;
base_stats_get(tsdn, arena->base, &base_allocated, &base_resident,
&base_mapped);
astats->mapped += base_mapped + arena->stats.mapped;
astats->retained += arena->stats.retained;
astats->npurge += arena->stats.npurge;
astats->nmadvise += arena->stats.nmadvise;
astats->purged += arena->stats.purged;
astats->metadata += arena_metadata_get(arena);
astats->base += base_allocated;
astats->internal += arena_internal_get(arena);
astats->resident += base_resident + (((arena->nactive + arena->ndirty)
<< LG_PAGE));
astats->allocated_large += arena->stats.allocated_large;
astats->nmalloc_large += arena->stats.nmalloc_large;
astats->ndalloc_large += arena->stats.ndalloc_large;
@@ -1625,19 +1632,27 @@ arena_extent_sn_next(arena_t *arena)
}
arena_t *
arena_new(tsdn_t *tsdn, unsigned ind)
arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks)
{
arena_t *arena;
base_t *base;
unsigned i;
arena = (arena_t *)base_alloc(tsdn, sizeof(arena_t));
if (arena == NULL)
return (NULL);
if (ind == 0)
base = b0get();
else {
base = base_new(tsdn, ind, extent_hooks);
if (base == NULL)
return (NULL);
}
arena = (arena_t *)base_alloc(tsdn, base, sizeof(arena_t), CACHELINE);
if (arena == NULL)
goto label_error;
arena->ind = ind;
arena->nthreads[0] = arena->nthreads[1] = 0;
if (malloc_mutex_init(&arena->lock, "arena", WITNESS_RANK_ARENA))
return (NULL);
goto label_error;
if (config_stats && config_tcache)
ql_new(&arena->tcache_ql);
@@ -1670,7 +1685,7 @@ arena_new(tsdn_t *tsdn, unsigned ind)
ql_new(&arena->large);
if (malloc_mutex_init(&arena->large_mtx, "arena_large",
WITNESS_RANK_ARENA_LARGE))
return (NULL);
goto label_error;
for (i = 0; i < NPSIZES+1; i++) {
extent_heap_new(&arena->extents_cached[i]);
@@ -1682,9 +1697,7 @@ arena_new(tsdn_t *tsdn, unsigned ind)
if (malloc_mutex_init(&arena->extents_mtx, "arena_extents",
WITNESS_RANK_ARENA_EXTENTS))
return (NULL);
arena->extent_hooks = (extent_hooks_t *)&extent_hooks_default;
goto label_error;
if (!config_munmap)
arena->extent_grow_next = psz2ind(HUGEPAGE);
@@ -1692,14 +1705,14 @@ arena_new(tsdn_t *tsdn, unsigned ind)
ql_new(&arena->extent_cache);
if (malloc_mutex_init(&arena->extent_cache_mtx, "arena_extent_cache",
WITNESS_RANK_ARENA_EXTENT_CACHE))
return (NULL);
goto label_error;
/* Initialize bins. */
for (i = 0; i < NBINS; i++) {
arena_bin_t *bin = &arena->bins[i];
if (malloc_mutex_init(&bin->lock, "arena_bin",
WITNESS_RANK_ARENA_BIN))
return (NULL);
goto label_error;
bin->slabcur = NULL;
extent_heap_new(&bin->slabs_nonfull);
extent_init(&bin->slabs_full, arena, NULL, 0, 0, 0, false,
@@ -1708,7 +1721,13 @@ arena_new(tsdn_t *tsdn, unsigned ind)
memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
}
arena->base = base;
return (arena);
label_error:
if (ind != 0)
base_delete(base);
return (NULL);
}
void
@@ -1744,6 +1763,7 @@ arena_prefork3(tsdn_t *tsdn, arena_t *arena)
{
unsigned i;
base_prefork(tsdn, arena->base);
for (i = 0; i < NBINS; i++)
malloc_mutex_prefork(tsdn, &arena->bins[i].lock);
malloc_mutex_prefork(tsdn, &arena->large_mtx);
@@ -1757,6 +1777,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena)
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
for (i = 0; i < NBINS; i++)
malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock);
base_postfork_parent(tsdn, arena->base);
malloc_mutex_postfork_parent(tsdn, &arena->extent_cache_mtx);
malloc_mutex_postfork_parent(tsdn, &arena->extents_mtx);
malloc_mutex_postfork_parent(tsdn, &arena->lock);
@@ -1770,6 +1791,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena)
malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
for (i = 0; i < NBINS; i++)
malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock);
base_postfork_child(tsdn, arena->base);
malloc_mutex_postfork_child(tsdn, &arena->extent_cache_mtx);
malloc_mutex_postfork_child(tsdn, &arena->extents_mtx);
malloc_mutex_postfork_child(tsdn, &arena->lock);

439
src/base.c
View File

@@ -4,112 +4,308 @@
/******************************************************************************/
/* Data. */
static malloc_mutex_t base_mtx;
static size_t base_extent_sn_next;
static extent_heap_t base_avail[NSIZES];
static extent_t *base_extents;
static size_t base_allocated;
static size_t base_resident;
static size_t base_mapped;
static base_t *b0;
/******************************************************************************/
static extent_t *
base_extent_try_alloc(tsdn_t *tsdn)
static void *
base_map(extent_hooks_t *extent_hooks, unsigned ind, size_t size)
{
extent_t *extent;
void *addr;
bool zero = true;
bool commit = true;
malloc_mutex_assert_owner(tsdn, &base_mtx);
assert(size == HUGEPAGE_CEILING(size));
if (base_extents == NULL)
return (NULL);
extent = base_extents;
base_extents = *(extent_t **)extent;
return (extent);
if (extent_hooks == &extent_hooks_default)
addr = extent_alloc_mmap(NULL, size, PAGE, &zero, &commit);
else {
addr = extent_hooks->alloc(extent_hooks, NULL, size, PAGE,
&zero, &commit, ind);
}
return (addr);
}
static void
base_extent_dalloc(tsdn_t *tsdn, extent_t *extent)
base_unmap(extent_hooks_t *extent_hooks, unsigned ind, void *addr, size_t size)
{
malloc_mutex_assert_owner(tsdn, &base_mtx);
*(extent_t **)extent = base_extents;
base_extents = extent;
/*
* Cascade through dalloc, decommit, purge_lazy, and purge_forced,
* stopping at first success. This cascade is performed for consistency
* with the cascade in extent_dalloc_wrapper() because an application's
* custom hooks may not support e.g. dalloc. This function is only ever
* called as a side effect of arena destruction, so although it might
* seem pointless to do anything besides dalloc here, the application
* may in fact want the end state of all associated virtual memory to in
* some consistent-but-allocated state.
*/
if (extent_hooks == &extent_hooks_default) {
if (!extent_dalloc_mmap(addr, size))
return;
if (!pages_decommit(addr, size))
return;
if (!pages_purge_lazy(addr, size))
return;
if (!pages_purge_forced(addr, size))
return;
/* Nothing worked. This should never happen. */
not_reached();
} else {
if (extent_hooks->dalloc != NULL &&
!extent_hooks->dalloc(extent_hooks, addr, size, true, ind))
return;
if (extent_hooks->decommit != NULL &&
!extent_hooks->decommit(extent_hooks, addr, size, 0, size,
ind))
return;
if (extent_hooks->purge_lazy != NULL &&
!extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size,
ind))
return;
if (extent_hooks->purge_forced != NULL &&
!extent_hooks->purge_forced(extent_hooks, addr, size, 0,
size, ind))
return;
/* Nothing worked. That's the application's problem. */
}
}
static void
base_extent_init(extent_t *extent, void *addr, size_t size)
base_extent_init(size_t *extent_sn_next, extent_t *extent, void *addr,
size_t size)
{
size_t sn = atomic_add_zu(&base_extent_sn_next, 1) - 1;
size_t sn;
sn = *extent_sn_next;
(*extent_sn_next)++;
extent_init(extent, NULL, addr, size, 0, sn, true, true, true, false);
}
static extent_t *
base_extent_alloc(tsdn_t *tsdn, size_t minsize)
static void *
base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size,
size_t alignment)
{
extent_t *extent;
size_t esize, nsize;
void *addr;
void *ret;
malloc_mutex_assert_owner(tsdn, &base_mtx);
assert(minsize != 0);
extent = base_extent_try_alloc(tsdn);
/* Allocate enough space to also carve an extent out if necessary. */
nsize = (extent == NULL) ? CACHELINE_CEILING(sizeof(extent_t)) : 0;
esize = PAGE_CEILING(minsize + nsize);
/*
* Directly call extent_alloc_mmap() because it's critical to allocate
* untouched demand-zeroed virtual memory.
*/
{
bool zero = true;
bool commit = true;
addr = extent_alloc_mmap(NULL, esize, PAGE, &zero, &commit);
assert(alignment == ALIGNMENT_CEILING(alignment, QUANTUM));
assert(size == ALIGNMENT_CEILING(size, alignment));
*gap_size = ALIGNMENT_CEILING((uintptr_t)extent_addr_get(extent),
alignment) - (uintptr_t)extent_addr_get(extent);
ret = (void *)((uintptr_t)extent_addr_get(extent) + *gap_size);
assert(extent_size_get(extent) >= *gap_size + size);
extent_init(extent, NULL, (void *)((uintptr_t)extent_addr_get(extent) +
*gap_size + size), extent_size_get(extent) - *gap_size - size, 0,
extent_sn_get(extent), true, true, true, false);
return (ret);
}
static void
base_extent_bump_alloc_post(tsdn_t *tsdn, base_t *base, extent_t *extent,
size_t gap_size, void *addr, size_t size)
{
if (extent_size_get(extent) > 0) {
/*
* Compute the index for the largest size class that does not
* exceed extent's size.
*/
szind_t index_floor = size2index(extent_size_get(extent) + 1) -
1;
extent_heap_insert(&base->avail[index_floor], extent);
}
if (addr == NULL) {
if (extent != NULL)
base_extent_dalloc(tsdn, extent);
return (NULL);
if (config_stats) {
base->allocated += size;
/*
* Add one PAGE to base_resident for every page boundary that is
* crossed by the new allocation.
*/
base->resident += PAGE_CEILING((uintptr_t)addr + size) -
PAGE_CEILING((uintptr_t)addr - gap_size);
assert(base->allocated <= base->resident);
assert(base->resident <= base->mapped);
}
base_mapped += esize;
if (extent == NULL) {
extent = (extent_t *)addr;
addr = (void *)((uintptr_t)addr + nsize);
esize -= nsize;
if (config_stats) {
base_allocated += nsize;
base_resident += PAGE_CEILING(nsize);
}
}
base_extent_init(extent, addr, esize);
return (extent);
}
static void *
base_extent_bump_alloc(tsdn_t *tsdn, base_t *base, extent_t *extent,
size_t size, size_t alignment)
{
void *ret;
size_t gap_size;
ret = base_extent_bump_alloc_helper(extent, &gap_size, size, alignment);
base_extent_bump_alloc_post(tsdn, base, extent, gap_size, ret, size);
return (ret);
}
/*
* base_alloc() guarantees demand-zeroed memory, in order to make multi-page
* sparse data structures such as radix tree nodes efficient with respect to
* physical memory usage.
* Allocate a block of virtual memory that is large enough to start with a
* base_block_t header, followed by an object of specified size and alignment.
* On success a pointer to the initialized base_block_t header is returned.
*/
static base_block_t *
base_block_alloc(extent_hooks_t *extent_hooks, unsigned ind,
size_t *extent_sn_next, size_t size, size_t alignment)
{
base_block_t *block;
size_t usize, header_size, gap_size, block_size;
alignment = ALIGNMENT_CEILING(alignment, QUANTUM);
usize = ALIGNMENT_CEILING(size, alignment);
header_size = sizeof(base_block_t);
gap_size = ALIGNMENT_CEILING(header_size, alignment) - header_size;
block_size = HUGEPAGE_CEILING(header_size + gap_size + usize);
block = (base_block_t *)base_map(extent_hooks, ind, block_size);
if (block == NULL)
return (NULL);
block->size = block_size;
block->next = NULL;
assert(block_size >= header_size);
base_extent_init(extent_sn_next, &block->extent,
(void *)((uintptr_t)block + header_size), block_size - header_size);
return (block);
}
/*
* Allocate an extent that is at least as large as specified size, with
* specified alignment.
*/
static extent_t *
base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment)
{
extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
base_block_t *block;
malloc_mutex_assert_owner(tsdn, &base->mtx);
block = base_block_alloc(extent_hooks, base_ind_get(base),
&base->extent_sn_next, size, alignment);
if (block == NULL)
return (NULL);
block->next = base->blocks;
base->blocks = block;
if (config_stats) {
base->allocated += sizeof(base_block_t);
base->resident += PAGE_CEILING(sizeof(base_block_t));
base->mapped += block->size;
assert(base->allocated <= base->resident);
assert(base->resident <= base->mapped);
}
return (&block->extent);
}
base_t *
b0get(void)
{
return (b0);
}
base_t *
base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks)
{
base_t *base;
size_t extent_sn_next, base_alignment, base_size, gap_size;
base_block_t *block;
szind_t i;
extent_sn_next = 0;
block = base_block_alloc(extent_hooks, ind, &extent_sn_next,
sizeof(base_t), QUANTUM);
if (block == NULL)
return (NULL);
base_alignment = CACHELINE;
base_size = ALIGNMENT_CEILING(sizeof(base_t), base_alignment);
base = (base_t *)base_extent_bump_alloc_helper(&block->extent,
&gap_size, base_size, base_alignment);
base->ind = ind;
base->extent_hooks = extent_hooks;
if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE)) {
base_unmap(extent_hooks, ind, block, block->size);
return (NULL);
}
base->extent_sn_next = extent_sn_next;
base->blocks = block;
for (i = 0; i < NSIZES; i++)
extent_heap_new(&base->avail[i]);
if (config_stats) {
base->allocated = sizeof(base_block_t);
base->resident = PAGE_CEILING(sizeof(base_block_t));
base->mapped = block->size;
assert(base->allocated <= base->resident);
assert(base->resident <= base->mapped);
}
base_extent_bump_alloc_post(tsdn, base, &block->extent, gap_size, base,
base_size);
return (base);
}
void
base_delete(base_t *base)
{
extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
base_block_t *next = base->blocks;
do {
base_block_t *block = next;
next = block->next;
base_unmap(extent_hooks, base_ind_get(base), block,
block->size);
} while (next != NULL);
}
extent_hooks_t *
base_extent_hooks_get(base_t *base)
{
return ((extent_hooks_t *)atomic_read_p(&base->extent_hooks_pun));
}
extent_hooks_t *
base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks)
{
extent_hooks_t *old_extent_hooks = base_extent_hooks_get(base);
union {
extent_hooks_t **h;
void **v;
} u;
u.h = &base->extent_hooks;
atomic_write_p(u.v, extent_hooks);
return (old_extent_hooks);
}
/*
* base_alloc() returns zeroed memory, which is always demand-zeroed for the
* auto arenas, in order to make multi-page sparse data structures such as radix
* tree nodes efficient with respect to physical memory usage. Upon success a
* pointer to at least size bytes with specified alignment is returned. Note
* that size is rounded up to the nearest multiple of alignment to avoid false
* sharing.
*/
void *
base_alloc(tsdn_t *tsdn, size_t size)
base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment)
{
void *ret;
size_t csize;
size_t usize, asize;
szind_t i;
extent_t *extent;
/*
* Round size up to nearest multiple of the cacheline size, so that
* there is no chance of false cache line sharing.
*/
csize = CACHELINE_CEILING(size);
alignment = QUANTUM_CEILING(alignment);
usize = ALIGNMENT_CEILING(size, alignment);
asize = usize + alignment - QUANTUM;
extent = NULL;
malloc_mutex_lock(tsdn, &base_mtx);
for (i = size2index(csize); i < NSIZES; i++) {
extent = extent_heap_remove_first(&base_avail[i]);
malloc_mutex_lock(tsdn, &base->mtx);
for (i = size2index(asize); i < NSIZES; i++) {
extent = extent_heap_remove_first(&base->avail[i]);
if (extent != NULL) {
/* Use existing space. */
break;
@@ -117,87 +313,60 @@ base_alloc(tsdn_t *tsdn, size_t size)
}
if (extent == NULL) {
/* Try to allocate more space. */
extent = base_extent_alloc(tsdn, csize);
extent = base_extent_alloc(tsdn, base, usize, alignment);
}
if (extent == NULL) {
ret = NULL;
goto label_return;
}
ret = extent_addr_get(extent);
if (extent_size_get(extent) > csize) {
szind_t index_floor;
extent_addr_set(extent, (void *)((uintptr_t)ret + csize));
extent_size_set(extent, extent_size_get(extent) - csize);
/*
* Compute the index for the largest size class that does not
* exceed extent's size.
*/
index_floor = size2index(extent_size_get(extent) + 1) - 1;
extent_heap_insert(&base_avail[index_floor], extent);
} else
base_extent_dalloc(tsdn, extent);
if (config_stats) {
base_allocated += csize;
/*
* Add one PAGE to base_resident for every page boundary that is
* crossed by the new allocation.
*/
base_resident += PAGE_CEILING((uintptr_t)ret + csize) -
PAGE_CEILING((uintptr_t)ret);
}
ret = base_extent_bump_alloc(tsdn, base, extent, usize, alignment);
label_return:
malloc_mutex_unlock(tsdn, &base_mtx);
malloc_mutex_unlock(tsdn, &base->mtx);
return (ret);
}
void
base_stats_get(tsdn_t *tsdn, size_t *allocated, size_t *resident,
base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident,
size_t *mapped)
{
malloc_mutex_lock(tsdn, &base_mtx);
assert(base_allocated <= base_resident);
assert(base_resident <= base_mapped);
*allocated = base_allocated;
*resident = base_resident;
*mapped = base_mapped;
malloc_mutex_unlock(tsdn, &base_mtx);
cassert(config_stats);
malloc_mutex_lock(tsdn, &base->mtx);
assert(base->allocated <= base->resident);
assert(base->resident <= base->mapped);
*allocated = base->allocated;
*resident = base->resident;
*mapped = base->mapped;
malloc_mutex_unlock(tsdn, &base->mtx);
}
void
base_prefork(tsdn_t *tsdn, base_t *base)
{
malloc_mutex_prefork(tsdn, &base->mtx);
}
void
base_postfork_parent(tsdn_t *tsdn, base_t *base)
{
malloc_mutex_postfork_parent(tsdn, &base->mtx);
}
void
base_postfork_child(tsdn_t *tsdn, base_t *base)
{
malloc_mutex_postfork_child(tsdn, &base->mtx);
}
bool
base_boot(void)
{
szind_t i;
if (malloc_mutex_init(&base_mtx, "base", WITNESS_RANK_BASE))
return (true);
base_extent_sn_next = 0;
for (i = 0; i < NSIZES; i++)
extent_heap_new(&base_avail[i]);
base_extents = NULL;
return (false);
}
void
base_prefork(tsdn_t *tsdn)
base_boot(tsdn_t *tsdn)
{
malloc_mutex_prefork(tsdn, &base_mtx);
}
void
base_postfork_parent(tsdn_t *tsdn)
{
malloc_mutex_postfork_parent(tsdn, &base_mtx);
}
void
base_postfork_child(tsdn_t *tsdn)
{
malloc_mutex_postfork_child(tsdn, &base_mtx);
b0 = base_new(tsdn, 0, (extent_hooks_t *)&extent_hooks_default);
return (b0 == NULL);
}

50
src/ctl.c
View File

@@ -55,7 +55,7 @@ static void ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_stats_t *cstats,
static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,
ctl_arena_stats_t *astats);
static void ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i);
static bool ctl_grow(tsdn_t *tsdn);
static bool ctl_grow(tsdn_t *tsdn, extent_hooks_t *extent_hooks);
static void ctl_refresh(tsdn_t *tsdn);
static bool ctl_init(tsdn_t *tsdn);
static int ctl_lookup(tsdn_t *tsdn, const char *name,
@@ -174,7 +174,9 @@ CTL_PROTO(stats_arenas_i_retained)
CTL_PROTO(stats_arenas_i_npurge)
CTL_PROTO(stats_arenas_i_nmadvise)
CTL_PROTO(stats_arenas_i_purged)
CTL_PROTO(stats_arenas_i_metadata)
CTL_PROTO(stats_arenas_i_base)
CTL_PROTO(stats_arenas_i_internal)
CTL_PROTO(stats_arenas_i_resident)
INDEX_PROTO(stats_arenas_i)
CTL_PROTO(stats_allocated)
CTL_PROTO(stats_active)
@@ -392,7 +394,9 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
{NAME("npurge"), CTL(stats_arenas_i_npurge)},
{NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)},
{NAME("purged"), CTL(stats_arenas_i_purged)},
{NAME("metadata"), CTL(stats_arenas_i_metadata)},
{NAME("base"), CTL(stats_arenas_i_base)},
{NAME("internal"), CTL(stats_arenas_i_internal)},
{NAME("resident"), CTL(stats_arenas_i_resident)},
{NAME("small"), CHILD(named, stats_arenas_i_small)},
{NAME("large"), CHILD(named, stats_arenas_i_large)},
{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
@@ -500,7 +504,9 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
sstats->astats.nmadvise += astats->astats.nmadvise;
sstats->astats.purged += astats->astats.purged;
sstats->astats.metadata += astats->astats.metadata;
sstats->astats.base += astats->astats.base;
sstats->astats.internal += astats->astats.internal;
sstats->astats.resident += astats->astats.resident;
sstats->allocated_small += astats->allocated_small;
sstats->nmalloc_small += astats->nmalloc_small;
@@ -556,12 +562,12 @@ ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i)
}
static bool
ctl_grow(tsdn_t *tsdn)
ctl_grow(tsdn_t *tsdn, extent_hooks_t *extent_hooks)
{
ctl_arena_stats_t *astats;
/* Initialize new arena. */
if (arena_init(tsdn, ctl_stats.narenas) == NULL)
if (arena_init(tsdn, ctl_stats.narenas, extent_hooks) == NULL)
return (true);
/* Allocate extended arena stats. */
@@ -615,20 +621,17 @@ ctl_refresh(tsdn_t *tsdn)
}
if (config_stats) {
size_t base_allocated, base_resident, base_mapped;
base_stats_get(tsdn, &base_allocated, &base_resident,
&base_mapped);
ctl_stats.allocated =
ctl_stats.arenas[ctl_stats.narenas].allocated_small +
ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large;
ctl_stats.active =
(ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE);
ctl_stats.metadata = base_allocated +
ctl_stats.arenas[ctl_stats.narenas].astats.metadata;
ctl_stats.resident = base_resident +
((ctl_stats.arenas[ctl_stats.narenas].pactive +
ctl_stats.arenas[ctl_stats.narenas].pdirty) << LG_PAGE);
ctl_stats.mapped = base_mapped +
ctl_stats.metadata =
ctl_stats.arenas[ctl_stats.narenas].astats.base +
ctl_stats.arenas[ctl_stats.narenas].astats.internal;
ctl_stats.resident =
ctl_stats.arenas[ctl_stats.narenas].astats.resident;
ctl_stats.mapped =
ctl_stats.arenas[ctl_stats.narenas].astats.mapped;
ctl_stats.retained =
ctl_stats.arenas[ctl_stats.narenas].astats.retained;
@@ -1167,7 +1170,7 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
if (oldarena == NULL)
return (EAGAIN);
newind = oldind = oldarena->ind;
newind = oldind = arena_ind_get(oldarena);
WRITE(newind, unsigned);
READ(oldind, unsigned);
if (newind != oldind) {
@@ -1738,11 +1741,14 @@ arenas_extend_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
extent_hooks_t *extent_hooks;
unsigned narenas;
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
READONLY();
if (ctl_grow(tsd_tsdn(tsd))) {
extent_hooks = (extent_hooks_t *)&extent_hooks_default;
WRITE(extent_hooks, extent_hooks_t *);
if (ctl_grow(tsd_tsdn(tsd), extent_hooks)) {
ret = EAGAIN;
goto label_return;
}
@@ -1906,8 +1912,12 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise,
ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_purged,
ctl_stats.arenas[mib[2]].astats.purged, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_metadata,
ctl_stats.arenas[mib[2]].astats.metadata, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_base,
ctl_stats.arenas[mib[2]].astats.base, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_internal,
ctl_stats.arenas[mib[2]].astats.internal, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_resident,
ctl_stats.arenas[mib[2]].astats.resident, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
ctl_stats.arenas[mib[2]].allocated_small, size_t)

36
src/extent.c
View File

@@ -83,7 +83,8 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena)
extent = ql_last(&arena->extent_cache, ql_link);
if (extent == NULL) {
malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
return (base_alloc(tsdn, sizeof(extent_t)));
return (base_alloc(tsdn, arena->base, sizeof(extent_t),
QUANTUM));
}
ql_tail_remove(&arena->extent_cache, extent_t, ql_link);
malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
@@ -104,22 +105,14 @@ extent_hooks_t *
extent_hooks_get(arena_t *arena)
{
return ((extent_hooks_t *)atomic_read_p(&arena->extent_hooks_pun));
return (base_extent_hooks_get(arena->base));
}
extent_hooks_t *
extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks)
{
extent_hooks_t *old_extent_hooks = extent_hooks_get(arena);
union {
extent_hooks_t **h;
void **v;
} u;
u.h = &arena->extent_hooks;
atomic_write_p(u.v, extent_hooks);
return (old_extent_hooks);
return (base_extent_hooks_set(arena->base, extent_hooks));
}
static void
@@ -873,7 +866,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
alignment, zero, commit);
} else {
addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, size,
alignment, zero, commit, arena->ind);
alignment, zero, commit, arena_ind_get(arena));
}
if (addr == NULL) {
extent_dalloc(tsdn, arena, extent);
@@ -1071,7 +1064,7 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
err = ((*r_extent_hooks)->dalloc == NULL ||
(*r_extent_hooks)->dalloc(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent),
extent_committed_get(extent), arena->ind));
extent_committed_get(extent), arena_ind_get(arena)));
}
if (!err) {
@@ -1088,12 +1081,12 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
else if ((*r_extent_hooks)->purge_lazy != NULL &&
!(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena->ind))
extent_size_get(extent), arena_ind_get(arena)))
zeroed = false;
else if ((*r_extent_hooks)->purge_forced != NULL &&
!(*r_extent_hooks)->purge_forced(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena->ind))
extent_size_get(extent), arena_ind_get(arena)))
zeroed = true;
else
zeroed = false;
@@ -1129,7 +1122,7 @@ extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_assure_initialized(arena, r_extent_hooks);
err = ((*r_extent_hooks)->commit == NULL ||
(*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), offset, length, arena->ind));
extent_size_get(extent), offset, length, arena_ind_get(arena)));
extent_committed_set(extent, extent_committed_get(extent) || !err);
return (err);
}
@@ -1157,7 +1150,7 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
err = ((*r_extent_hooks)->decommit == NULL ||
(*r_extent_hooks)->decommit(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
arena_ind_get(arena)));
extent_committed_set(extent, extent_committed_get(extent) && err);
return (err);
}
@@ -1189,7 +1182,7 @@ extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
return ((*r_extent_hooks)->purge_lazy == NULL ||
(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
arena_ind_get(arena)));
}
#ifdef PAGES_CAN_PURGE_FORCED
@@ -1219,7 +1212,7 @@ extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
return ((*r_extent_hooks)->purge_forced == NULL ||
(*r_extent_hooks)->purge_forced(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
arena_ind_get(arena)));
}
#ifdef JEMALLOC_MAPS_COALESCE
@@ -1280,7 +1273,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
if ((*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent),
size_a + size_b, size_a, size_b, extent_committed_get(extent),
arena->ind))
arena_ind_get(arena)))
goto label_error_d;
extent_size_set(extent, size_a);
@@ -1348,7 +1341,8 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
} else {
err = (*r_extent_hooks)->merge(*r_extent_hooks,
extent_base_get(a), extent_size_get(a), extent_base_get(b),
extent_size_get(b), extent_committed_get(a), arena->ind);
extent_size_get(b), extent_committed_get(a),
arena_ind_get(arena));
}
if (err)

35
src/jemalloc.c
View File

@@ -304,21 +304,21 @@ malloc_init(void)
*/
static void *
a0ialloc(size_t size, bool zero, bool is_metadata)
a0ialloc(size_t size, bool zero, bool is_internal)
{
if (unlikely(malloc_init_a0()))
return (NULL);
return (iallocztm(TSDN_NULL, size, size2index(size), zero, NULL,
is_metadata, arena_get(TSDN_NULL, 0, true), true));
is_internal, arena_get(TSDN_NULL, 0, true), true));
}
static void
a0idalloc(extent_t *extent, void *ptr, bool is_metadata)
a0idalloc(extent_t *extent, void *ptr, bool is_internal)
{
idalloctm(TSDN_NULL, extent, ptr, false, is_metadata, true);
idalloctm(TSDN_NULL, extent, ptr, false, is_internal, true);
}
void *
@@ -405,7 +405,7 @@ narenas_total_get(void)
/* Create a new arena and insert it into the arenas array at index ind. */
static arena_t *
arena_init_locked(tsdn_t *tsdn, unsigned ind)
arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks)
{
arena_t *arena;
@@ -426,18 +426,18 @@ arena_init_locked(tsdn_t *tsdn, unsigned ind)
}
/* Actually initialize the arena. */
arena = arena_new(tsdn, ind);
arena = arena_new(tsdn, ind, extent_hooks);
arena_set(ind, arena);
return (arena);
}
arena_t *
arena_init(tsdn_t *tsdn, unsigned ind)
arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks)
{
arena_t *arena;
malloc_mutex_lock(tsdn, &arenas_lock);
arena = arena_init_locked(tsdn, ind);
arena = arena_init_locked(tsdn, ind, extent_hooks);
malloc_mutex_unlock(tsdn, &arenas_lock);
return (arena);
}
@@ -629,7 +629,8 @@ arena_choose_hard(tsd_t *tsd, bool internal)
/* Initialize a new arena. */
choose[j] = first_null;
arena = arena_init_locked(tsd_tsdn(tsd),
choose[j]);
choose[j],
(extent_hooks_t *)&extent_hooks_default);
if (arena == NULL) {
malloc_mutex_unlock(tsd_tsdn(tsd),
&arenas_lock);
@@ -657,7 +658,7 @@ iarena_cleanup(tsd_t *tsd)
iarena = tsd_iarena_get(tsd);
if (iarena != NULL)
arena_unbind(tsd, iarena->ind, true);
arena_unbind(tsd, arena_ind_get(iarena), true);
}
void
@@ -667,7 +668,7 @@ arena_cleanup(tsd_t *tsd)
arena = tsd_arena_get(tsd);
if (arena != NULL)
arena_unbind(tsd, arena->ind, false);
arena_unbind(tsd, arena_ind_get(arena), false);
}
void
@@ -1211,7 +1212,7 @@ malloc_init_hard_a0_locked()
}
}
pages_boot();
if (base_boot())
if (base_boot(TSDN_NULL))
return (true);
if (extent_boot())
return (true);
@@ -1236,7 +1237,8 @@ malloc_init_hard_a0_locked()
* Initialize one arena here. The rest are lazily created in
* arena_choose_hard().
*/
if (arena_init(TSDN_NULL, 0) == NULL)
if (arena_init(TSDN_NULL, 0, (extent_hooks_t *)&extent_hooks_default) ==
NULL)
return (true);
malloc_init_state = malloc_init_a0_initialized;
@@ -1309,8 +1311,8 @@ malloc_init_hard_finish(tsdn_t *tsdn)
narenas_total_set(narenas_auto);
/* Allocate and initialize arenas. */
arenas = (arena_t **)base_alloc(tsdn, sizeof(arena_t *) *
(MALLOCX_ARENA_MAX+1));
arenas = (arena_t **)base_alloc(tsdn, a0->base, sizeof(arena_t *) *
(MALLOCX_ARENA_MAX+1), CACHELINE);
if (arenas == NULL)
return (true);
/* Copy the pointer to the one arena that was already initialized. */
@@ -2690,7 +2692,6 @@ _malloc_prefork(void)
}
}
}
base_prefork(tsd_tsdn(tsd));
for (i = 0; i < narenas; i++) {
if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
arena_prefork3(tsd_tsdn(tsd), arena);
@@ -2719,7 +2720,6 @@ _malloc_postfork(void)
witness_postfork_parent(tsd);
/* Release all mutexes, now that fork() has completed. */
base_postfork_parent(tsd_tsdn(tsd));
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;
@@ -2743,7 +2743,6 @@ jemalloc_postfork_child(void)
witness_postfork_child(tsd);
/* Release all mutexes, now that fork() has completed. */
base_postfork_child(tsd_tsdn(tsd));
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;

6
src/prof.c
View File

@@ -2254,7 +2254,8 @@ prof_boot2(tsd_t *tsd)
}
gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
PROF_NCTX_LOCKS * sizeof(malloc_mutex_t));
b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t),
CACHELINE);
if (gctx_locks == NULL)
return (true);
for (i = 0; i < PROF_NCTX_LOCKS; i++) {
@@ -2264,7 +2265,8 @@ prof_boot2(tsd_t *tsd)
}
tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd),
PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t));
b0get(), PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t),
CACHELINE);
if (tdata_locks == NULL)
return (true);
for (i = 0; i < PROF_NTDATA_LOCKS; i++) {

3
src/rtree.c
View File

@@ -72,7 +72,8 @@ static rtree_elm_t *
rtree_node_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms)
{
return ((rtree_elm_t *)base_alloc(tsdn, nelms * sizeof(rtree_elm_t)));
return ((rtree_elm_t *)base_alloc(tsdn, b0get(), nelms *
sizeof(rtree_elm_t), CACHELINE));
}
#ifdef JEMALLOC_JET
#undef rtree_node_alloc

27
src/stats.c
View File

@@ -254,7 +254,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
unsigned nthreads;
const char *dss;
ssize_t decay_time;
size_t page, pactive, pdirty, mapped, retained, metadata;
size_t page, pactive, pdirty, mapped, retained;
size_t base, internal, resident;
uint64_t npurge, nmadvise, purged;
size_t small_allocated;
uint64_t small_nmalloc, small_ndalloc, small_nrequests;
@@ -404,14 +405,32 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
"retained: %12zu\n", retained);
}
CTL_M2_GET("stats.arenas.0.metadata", i, &metadata, size_t);
CTL_M2_GET("stats.arenas.0.base", i, &base, size_t);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"metadata\": %zu%s\n", metadata, (bins || large) ?
"\t\t\t\t\"base\": %zu,\n", base);
} else {
malloc_cprintf(write_cb, cbopaque,
"base: %12zu\n", base);
}
CTL_M2_GET("stats.arenas.0.internal", i, &internal, size_t);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"internal\": %zu,\n", internal);
} else {
malloc_cprintf(write_cb, cbopaque,
"internal: %12zu\n", internal);
}
CTL_M2_GET("stats.arenas.0.resident", i, &resident, size_t);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"resident\": %zu%s\n", resident, (bins || large) ?
"," : "");
} else {
malloc_cprintf(write_cb, cbopaque,
"metadata: %12zu\n", metadata);
"resident: %12zu\n", resident);
}
if (bins)

8
src/tcache.c
View File

@@ -440,8 +440,8 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
tcaches_t *elm;
if (tcaches == NULL) {
tcaches = base_alloc(tsd_tsdn(tsd), sizeof(tcache_t *) *
(MALLOCX_TCACHE_MAX+1));
tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *)
* (MALLOCX_TCACHE_MAX+1), CACHELINE);
if (tcaches == NULL)
return (true);
}
@@ -510,8 +510,8 @@ tcache_boot(tsdn_t *tsdn)
nhbins = size2index(tcache_maxclass) + 1;
/* Initialize tcache_bin_info. */
tcache_bin_info = (tcache_bin_info_t *)base_alloc(tsdn, nhbins *
sizeof(tcache_bin_info_t));
tcache_bin_info = (tcache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins
* sizeof(tcache_bin_info_t), CACHELINE);
if (tcache_bin_info == NULL)
return (true);
stack_nelms = 0;