project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update brace style.

Browse Source 
Add braces around single-line blocks, and remove line breaks before
function-opening braces.

This resolves #537.
This commit is contained in:
Jason Evans
2017-01-15 16:56:30 -08:00
parent 5154ff32ee
commit c4c2592c83
119 changed files with 2971 additions and 3572 deletions
Download Patch File Download Diff File Expand all files Collapse all files

516
src/arena.c
View File

File diff suppressed because it is too large Load Diff

99
src/base.c
View File

@@ -9,17 +9,16 @@ static base_t *b0;
/******************************************************************************/
static void *
base_map(extent_hooks_t *extent_hooks, unsigned ind, size_t size)
{
base_map(extent_hooks_t *extent_hooks, unsigned ind, size_t size) {
void *addr;
bool zero = true;
bool commit = true;
assert(size == HUGEPAGE_CEILING(size));
if (extent_hooks == &extent_hooks_default)
if (extent_hooks == &extent_hooks_default) {
addr = extent_alloc_mmap(NULL, size, PAGE, &zero, &commit);
else {
} else {
addr = extent_hooks->alloc(extent_hooks, NULL, size, PAGE,
&zero, &commit, ind);
}
@@ -28,8 +27,8 @@ base_map(extent_hooks_t *extent_hooks, unsigned ind, size_t size)
}
static void
base_unmap(extent_hooks_t *extent_hooks, unsigned ind, void *addr, size_t size)
{
base_unmap(extent_hooks_t *extent_hooks, unsigned ind, void *addr,
size_t size) {
/*
* Cascade through dalloc, decommit, purge_lazy, and purge_forced,
* stopping at first success. This cascade is performed for consistency
@@ -41,40 +40,48 @@ base_unmap(extent_hooks_t *extent_hooks, unsigned ind, void *addr, size_t size)
* some consistent-but-allocated state.
*/
if (extent_hooks == &extent_hooks_default) {
if (!extent_dalloc_mmap(addr, size))
if (!extent_dalloc_mmap(addr, size)) {
return;
if (!pages_decommit(addr, size))
}
if (!pages_decommit(addr, size)) {
return;
if (!pages_purge_lazy(addr, size))
}
if (!pages_purge_lazy(addr, size)) {
return;
if (!pages_purge_forced(addr, size))
}
if (!pages_purge_forced(addr, size)) {
return;
}
/* Nothing worked. This should never happen. */
not_reached();
} else {
if (extent_hooks->dalloc != NULL &&
!extent_hooks->dalloc(extent_hooks, addr, size, true, ind))
!extent_hooks->dalloc(extent_hooks, addr, size, true,
ind)) {
return;
}
if (extent_hooks->decommit != NULL &&
!extent_hooks->decommit(extent_hooks, addr, size, 0, size,
ind))
ind)) {
return;
}
if (extent_hooks->purge_lazy != NULL &&
!extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size,
ind))
ind)) {
return;
}
if (extent_hooks->purge_forced != NULL &&
!extent_hooks->purge_forced(extent_hooks, addr, size, 0,
size, ind))
size, ind)) {
return;
}
/* Nothing worked. That's the application's problem. */
}
}
static void
base_extent_init(size_t *extent_sn_next, extent_t *extent, void *addr,
size_t size)
{
size_t size) {
size_t sn;
sn = *extent_sn_next;
@@ -85,8 +92,7 @@ base_extent_init(size_t *extent_sn_next, extent_t *extent, void *addr,
static void *
base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size,
size_t alignment)
{
size_t alignment) {
void *ret;
assert(alignment == ALIGNMENT_CEILING(alignment, QUANTUM));
@@ -104,8 +110,7 @@ base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size,
static void
base_extent_bump_alloc_post(tsdn_t *tsdn, base_t *base, extent_t *extent,
size_t gap_size, void *addr, size_t size)
{
size_t gap_size, void *addr, size_t size) {
if (extent_size_get(extent) > 0) {
/*
* Compute the index for the largest size class that does not
@@ -131,8 +136,7 @@ base_extent_bump_alloc_post(tsdn_t *tsdn, base_t *base, extent_t *extent,
static void *
base_extent_bump_alloc(tsdn_t *tsdn, base_t *base, extent_t *extent,
size_t size, size_t alignment)
{
size_t size, size_t alignment) {
void *ret;
size_t gap_size;
@@ -148,8 +152,7 @@ base_extent_bump_alloc(tsdn_t *tsdn, base_t *base, extent_t *extent,
*/
static base_block_t *
base_block_alloc(extent_hooks_t *extent_hooks, unsigned ind,
size_t *extent_sn_next, size_t size, size_t alignment)
{
size_t *extent_sn_next, size_t size, size_t alignment) {
base_block_t *block;
size_t usize, header_size, gap_size, block_size;
@@ -159,8 +162,9 @@ base_block_alloc(extent_hooks_t *extent_hooks, unsigned ind,
gap_size = ALIGNMENT_CEILING(header_size, alignment) - header_size;
block_size = HUGEPAGE_CEILING(header_size + gap_size + usize);
block = (base_block_t *)base_map(extent_hooks, ind, block_size);
if (block == NULL)
if (block == NULL) {
return (NULL);
}
block->size = block_size;
block->next = NULL;
assert(block_size >= header_size);
@@ -174,8 +178,7 @@ base_block_alloc(extent_hooks_t *extent_hooks, unsigned ind,
* specified alignment.
*/
static extent_t *
base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment)
{
base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
base_block_t *block;
@@ -183,8 +186,9 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment)
block = base_block_alloc(extent_hooks, base_ind_get(base),
&base->extent_sn_next, size, alignment);
if (block == NULL)
if (block == NULL) {
return (NULL);
}
block->next = base->blocks;
base->blocks = block;
if (config_stats) {
@@ -198,14 +202,12 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment)
}
base_t *
b0get(void)
{
b0get(void) {
return (b0);
}
base_t *
base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks)
{
base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
base_t *base;
size_t extent_sn_next, base_alignment, base_size, gap_size;
base_block_t *block;
@@ -214,8 +216,9 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks)
extent_sn_next = 0;
block = base_block_alloc(extent_hooks, ind, &extent_sn_next,
sizeof(base_t), QUANTUM);
if (block == NULL)
if (block == NULL) {
return (NULL);
}
base_alignment = CACHELINE;
base_size = ALIGNMENT_CEILING(sizeof(base_t), base_alignment);
@@ -229,8 +232,9 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks)
}
base->extent_sn_next = extent_sn_next;
base->blocks = block;
for (i = 0; i < NSIZES; i++)
for (i = 0; i < NSIZES; i++) {
extent_heap_new(&base->avail[i]);
}
if (config_stats) {
base->allocated = sizeof(base_block_t);
base->resident = PAGE_CEILING(sizeof(base_block_t));
@@ -245,8 +249,7 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks)
}
void
base_delete(base_t *base)
{
base_delete(base_t *base) {
extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
base_block_t *next = base->blocks;
do {
@@ -258,14 +261,12 @@ base_delete(base_t *base)
}
extent_hooks_t *
base_extent_hooks_get(base_t *base)
{
base_extent_hooks_get(base_t *base) {
return ((extent_hooks_t *)atomic_read_p(&base->extent_hooks_pun));
}
extent_hooks_t *
base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks)
{
base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks) {
extent_hooks_t *old_extent_hooks = base_extent_hooks_get(base);
union {
extent_hooks_t **h;
@@ -287,8 +288,7 @@ base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks)
* sharing.
*/
void *
base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment)
{
base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
void *ret;
size_t usize, asize;
szind_t i;
@@ -324,8 +324,7 @@ label_return:
void
base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident,
size_t *mapped)
{
size_t *mapped) {
cassert(config_stats);
malloc_mutex_lock(tsdn, &base->mtx);
@@ -338,26 +337,22 @@ base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident,
}
void
base_prefork(tsdn_t *tsdn, base_t *base)
{
base_prefork(tsdn_t *tsdn, base_t *base) {
malloc_mutex_prefork(tsdn, &base->mtx);
}
void
base_postfork_parent(tsdn_t *tsdn, base_t *base)
{
base_postfork_parent(tsdn_t *tsdn, base_t *base) {
malloc_mutex_postfork_parent(tsdn, &base->mtx);
}
void
base_postfork_child(tsdn_t *tsdn, base_t *base)
{
base_postfork_child(tsdn_t *tsdn, base_t *base) {
malloc_mutex_postfork_child(tsdn, &base->mtx);
}
bool
base_boot(tsdn_t *tsdn)
{
base_boot(tsdn_t *tsdn) {
b0 = base_new(tsdn, 0, (extent_hooks_t *)&extent_hooks_default);
return (b0 == NULL);
}

30
src/bitmap.c
View File

@@ -6,8 +6,7 @@
#ifdef BITMAP_USE_TREE
void
bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
{
bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
unsigned i;
size_t group_count;
@@ -35,14 +34,12 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
}
static size_t
bitmap_info_ngroups(const bitmap_info_t *binfo)
{
bitmap_info_ngroups(const bitmap_info_t *binfo) {
return (binfo->levels[binfo->nlevels].group_offset);
}
void
bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo)
{
bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo) {
size_t extra;
unsigned i;
@@ -56,23 +53,24 @@ bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo)
memset(bitmap, 0xffU, bitmap_size(binfo));
extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK))
& BITMAP_GROUP_NBITS_MASK;
if (extra != 0)
if (extra != 0) {
bitmap[binfo->levels[1].group_offset - 1] >>= extra;
}
for (i = 1; i < binfo->nlevels; i++) {
size_t group_count = binfo->levels[i].group_offset -
binfo->levels[i-1].group_offset;
extra = (BITMAP_GROUP_NBITS - (group_count &
BITMAP_GROUP_NBITS_MASK)) & BITMAP_GROUP_NBITS_MASK;
if (extra != 0)
if (extra != 0) {
bitmap[binfo->levels[i+1].group_offset - 1] >>= extra;
}
}
}
#else /* BITMAP_USE_TREE */
void
bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
{
bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
assert(nbits > 0);
assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS));
@@ -81,27 +79,25 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits)
}
static size_t
bitmap_info_ngroups(const bitmap_info_t *binfo)
{
bitmap_info_ngroups(const bitmap_info_t *binfo) {
return (binfo->ngroups);
}
void
bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo)
{
bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo) {
size_t extra;
memset(bitmap, 0xffU, bitmap_size(binfo));
extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK))
& BITMAP_GROUP_NBITS_MASK;
if (extra != 0)
if (extra != 0) {
bitmap[binfo->ngroups - 1] >>= extra;
}
}
#endif /* BITMAP_USE_TREE */
size_t
bitmap_size(const bitmap_info_t *binfo)
{
bitmap_size(const bitmap_info_t *binfo) {
return (bitmap_info_ngroups(binfo) << LG_SIZEOF_BITMAP);
}

101
src/ckh.c
View File

@@ -50,15 +50,15 @@ static void ckh_shrink(tsd_t *tsd, ckh_t *ckh);
* otherwise.
*/
JEMALLOC_INLINE_C size_t
ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key)
{
ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) {
ckhc_t *cell;
unsigned i;
for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) {
cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i];
if (cell->key != NULL && ckh->keycomp(key, cell->key))
if (cell->key != NULL && ckh->keycomp(key, cell->key)) {
return ((bucket << LG_CKH_BUCKET_CELLS) + i);
}
}
return (SIZE_T_MAX);
@@ -68,8 +68,7 @@ ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key)
* Search table for key and return cell number if found; SIZE_T_MAX otherwise.
*/
JEMALLOC_INLINE_C size_t
ckh_isearch(ckh_t *ckh, const void *key)
{
ckh_isearch(ckh_t *ckh, const void *key) {
size_t hashes[2], bucket, cell;
assert(ckh != NULL);
@@ -79,8 +78,9 @@ ckh_isearch(ckh_t *ckh, const void *key)
/* Search primary bucket. */
bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
cell = ckh_bucket_search(ckh, bucket, key);
if (cell != SIZE_T_MAX)
if (cell != SIZE_T_MAX) {
return (cell);
}
/* Search secondary bucket. */
bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
@@ -90,8 +90,7 @@ ckh_isearch(ckh_t *ckh, const void *key)
JEMALLOC_INLINE_C bool
ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
const void *data)
{
const void *data) {
ckhc_t *cell;
unsigned offset, i;
@@ -123,8 +122,7 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
*/
JEMALLOC_INLINE_C bool
ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
void const **argdata)
{
void const **argdata) {
const void *key, *data, *tkey, *tdata;
ckhc_t *cell;
size_t hashes[2], bucket, tbucket;
@@ -187,14 +185,14 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
}
bucket = tbucket;
if (!ckh_try_bucket_insert(ckh, bucket, key, data))
if (!ckh_try_bucket_insert(ckh, bucket, key, data)) {
return (false);
}
}
}
JEMALLOC_INLINE_C bool
ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
{
ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) {
size_t hashes[2], bucket;
const void *key = *argkey;
const void *data = *argdata;
@@ -203,13 +201,15 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
/* Try to insert in primary bucket. */
bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
if (!ckh_try_bucket_insert(ckh, bucket, key, data))
if (!ckh_try_bucket_insert(ckh, bucket, key, data)) {
return (false);
}
/* Try to insert in secondary bucket. */
bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
if (!ckh_try_bucket_insert(ckh, bucket, key, data))
if (!ckh_try_bucket_insert(ckh, bucket, key, data)) {
return (false);
}
/*
* Try to find a place for this item via iterative eviction/relocation.
@@ -222,8 +222,7 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
* old table into the new.
*/
JEMALLOC_INLINE_C bool
ckh_rebuild(ckh_t *ckh, ckhc_t *aTab)
{
ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) {
size_t count, i, nins;
const void *key, *data;
@@ -245,8 +244,7 @@ ckh_rebuild(ckh_t *ckh, ckhc_t *aTab)
}
static bool
ckh_grow(tsd_t *tsd, ckh_t *ckh)
{
ckh_grow(tsd_t *tsd, ckh_t *ckh) {
bool ret;
ckhc_t *tab, *ttab;
unsigned lg_prevbuckets, lg_curcells;
@@ -302,8 +300,7 @@ label_return:
}
static void
ckh_shrink(tsd_t *tsd, ckh_t *ckh)
{
ckh_shrink(tsd_t *tsd, ckh_t *ckh) {
ckhc_t *tab, *ttab;
size_t usize;
unsigned lg_prevbuckets, lg_curcells;
@@ -315,8 +312,9 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh)
lg_prevbuckets = ckh->lg_curbuckets;
lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1;
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
return;
}
tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL,
true, arena_ichoose(tsd, NULL));
if (tab == NULL) {
@@ -353,8 +351,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh)
bool
ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh_keycomp_t *keycomp)
{
ckh_keycomp_t *keycomp) {
bool ret;
size_t mincells, usize;
unsigned lg_mincells;
@@ -384,8 +381,9 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
mincells = ((minitems + (3 - (minitems % 3))) / 3) << 2;
for (lg_mincells = LG_CKH_BUCKET_CELLS;
(ZU(1) << lg_mincells) < mincells;
lg_mincells++)
; /* Do nothing. */
lg_mincells++) {
/* Do nothing. */
}
ckh->lg_minbuckets = lg_mincells - LG_CKH_BUCKET_CELLS;
ckh->lg_curbuckets = lg_mincells - LG_CKH_BUCKET_CELLS;
ckh->hash = hash;
@@ -409,8 +407,7 @@ label_return:
}
void
ckh_delete(tsd_t *tsd, ckh_t *ckh)
{
ckh_delete(tsd_t *tsd, ckh_t *ckh) {
assert(ckh != NULL);
#ifdef CKH_VERBOSE
@@ -427,30 +424,31 @@ ckh_delete(tsd_t *tsd, ckh_t *ckh)
idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), ckh->tab), ckh->tab,
NULL, true, true);
if (config_debug)
if (config_debug) {
memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t));
}
}
size_t
ckh_count(ckh_t *ckh)
{
ckh_count(ckh_t *ckh) {
assert(ckh != NULL);
return (ckh->count);
}
bool
ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data)
{
ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data) {
size_t i, ncells;
for (i = *tabind, ncells = (ZU(1) << (ckh->lg_curbuckets +
LG_CKH_BUCKET_CELLS)); i < ncells; i++) {
if (ckh->tab[i].key != NULL) {
if (key != NULL)
if (key != NULL) {
*key = (void *)ckh->tab[i].key;
if (data != NULL)
}
if (data != NULL) {
*data = (void *)ckh->tab[i].data;
}
*tabind = i + 1;
return (false);
}
@@ -460,8 +458,7 @@ ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data)
}
bool
ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data)
{
ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data) {
bool ret;
assert(ckh != NULL);
@@ -485,18 +482,19 @@ label_return:
bool
ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
void **data)
{
void **data) {
size_t cell;
assert(ckh != NULL);
cell = ckh_isearch(ckh, searchkey);
if (cell != SIZE_T_MAX) {
if (key != NULL)
if (key != NULL) {
*key = (void *)ckh->tab[cell].key;
if (data != NULL)
}
if (data != NULL) {
*data = (void *)ckh->tab[cell].data;
}
ckh->tab[cell].key = NULL;
ckh->tab[cell].data = NULL; /* Not necessary. */
@@ -516,18 +514,19 @@ ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
}
bool
ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data)
{
ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) {
size_t cell;
assert(ckh != NULL);
cell = ckh_isearch(ckh, searchkey);
if (cell != SIZE_T_MAX) {
if (key != NULL)
if (key != NULL) {
*key = (void *)ckh->tab[cell].key;
if (data != NULL)
}
if (data != NULL) {
*data = (void *)ckh->tab[cell].data;
}
return (false);
}
@@ -535,14 +534,12 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data)
}
void
ckh_string_hash(const void *key, size_t r_hash[2])
{
ckh_string_hash(const void *key, size_t r_hash[2]) {
hash(key, strlen((const char *)key), 0x94122f33U, r_hash);
}
bool
ckh_string_keycomp(const void *k1, const void *k2)
{
ckh_string_keycomp(const void *k1, const void *k2) {
assert(k1 != NULL);
assert(k2 != NULL);
@@ -550,8 +547,7 @@ ckh_string_keycomp(const void *k1, const void *k2)
}
void
ckh_pointer_hash(const void *key, size_t r_hash[2])
{
ckh_pointer_hash(const void *key, size_t r_hash[2]) {
union {
const void *v;
size_t i;
@@ -563,7 +559,6 @@ ckh_pointer_hash(const void *key, size_t r_hash[2])
}
bool
ckh_pointer_keycomp(const void *k1, const void *k2)
{
ckh_pointer_keycomp(const void *k1, const void *k2) {
return ((k1 == k2) ? true : false);
}

344
src/ctl.c
View File

@@ -17,22 +17,19 @@ static ctl_arenas_t *ctl_arenas;
/* Helpers for named and indexed nodes. */
JEMALLOC_INLINE_C const ctl_named_node_t *
ctl_named_node(const ctl_node_t *node)
{
ctl_named_node(const ctl_node_t *node) {
return ((node->named) ? (const ctl_named_node_t *)node : NULL);
}
JEMALLOC_INLINE_C const ctl_named_node_t *
ctl_named_children(const ctl_named_node_t *node, size_t index)
{
ctl_named_children(const ctl_named_node_t *node, size_t index) {
const ctl_named_node_t *children = ctl_named_node(node->children);
return (children ? &children[index] : NULL);
}
JEMALLOC_INLINE_C const ctl_indexed_node_t *
ctl_indexed_node(const ctl_node_t *node)
{
ctl_indexed_node(const ctl_node_t *node) {
return (!node->named ? (const ctl_indexed_node_t *)node : NULL);
}
@@ -433,8 +430,7 @@ static const ctl_named_node_t super_root_node[] = {
/******************************************************************************/
static unsigned
arenas_i2a_impl(size_t i, bool compat, bool validate)
{
arenas_i2a_impl(size_t i, bool compat, bool validate) {
unsigned a;
switch (i) {
@@ -453,9 +449,9 @@ arenas_i2a_impl(size_t i, bool compat, bool validate)
* removal in 6.0.0.
*/
a = 0;
} else if (validate && i >= ctl_arenas->narenas)
} else if (validate && i >= ctl_arenas->narenas) {
a = UINT_MAX;
else {
} else {
/*
* This function should never be called for an index
* more than one past the range of indices that have
@@ -472,14 +468,12 @@ arenas_i2a_impl(size_t i, bool compat, bool validate)
}
static unsigned
arenas_i2a(size_t i)
{
arenas_i2a(size_t i) {
return (arenas_i2a_impl(i, true, false));
}
static ctl_arena_t *
arenas_i_impl(tsdn_t *tsdn, size_t i, bool compat, bool init)
{
arenas_i_impl(tsdn_t *tsdn, size_t i, bool compat, bool init) {
ctl_arena_t *ret;
assert(!compat || !init);
@@ -515,16 +509,14 @@ arenas_i_impl(tsdn_t *tsdn, size_t i, bool compat, bool init)
}
static ctl_arena_t *
arenas_i(size_t i)
{
arenas_i(size_t i) {
ctl_arena_t *ret = arenas_i_impl(TSDN_NULL, i, true, false);
assert(ret != NULL);
return (ret);
}
static void
ctl_arena_clear(ctl_arena_t *ctl_arena)
{
ctl_arena_clear(ctl_arena_t *ctl_arena) {
ctl_arena->nthreads = 0;
ctl_arena->dss = dss_prec_names[dss_prec_limit];
ctl_arena->decay_time = -1;
@@ -544,8 +536,7 @@ ctl_arena_clear(ctl_arena_t *ctl_arena)
}
static void
ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena)
{
ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
unsigned i;
if (config_stats) {
@@ -575,8 +566,7 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena)
static void
ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
bool destroyed)
{
bool destroyed) {
unsigned i;
if (!destroyed) {
@@ -605,13 +595,15 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
sdstats->astats.base += astats->astats.base;
sdstats->astats.internal += astats->astats.internal;
sdstats->astats.resident += astats->astats.resident;
} else
} else {
assert(astats->astats.internal == 0);
}
if (!destroyed)
if (!destroyed) {
sdstats->allocated_small += astats->allocated_small;
else
} else {
assert(astats->allocated_small == 0);
}
sdstats->nmalloc_small += astats->nmalloc_small;
sdstats->ndalloc_small += astats->ndalloc_small;
sdstats->nrequests_small += astats->nrequests_small;
@@ -619,8 +611,9 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
if (!destroyed) {
sdstats->astats.allocated_large +=
astats->astats.allocated_large;
} else
} else {
assert(astats->astats.allocated_large == 0);
}
sdstats->astats.nmalloc_large += astats->astats.nmalloc_large;
sdstats->astats.ndalloc_large += astats->astats.ndalloc_large;
sdstats->astats.nrequests_large +=
@@ -639,8 +632,9 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
if (!destroyed) {
sdstats->bstats[i].curregs +=
astats->bstats[i].curregs;
} else
} else {
assert(astats->bstats[i].curregs == 0);
}
if (config_tcache) {
sdstats->bstats[i].nfills +=
astats->bstats[i].nfills;
@@ -652,8 +646,9 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
if (!destroyed) {
sdstats->bstats[i].curslabs +=
astats->bstats[i].curslabs;
} else
} else {
assert(astats->bstats[i].curslabs == 0);
}
}
for (i = 0; i < NSIZES - NBINS; i++) {
@@ -664,16 +659,16 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
if (!destroyed) {
sdstats->lstats[i].curlextents +=
astats->lstats[i].curlextents;
} else
} else {
assert(astats->lstats[i].curlextents == 0);
}
}
}
}
static void
ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena,
unsigned i, bool destroyed)
{
unsigned i, bool destroyed) {
ctl_arena_t *ctl_arena = arenas_i(i);
ctl_arena_clear(ctl_arena);
@@ -683,8 +678,7 @@ ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena,
}
static unsigned
ctl_arena_init(tsdn_t *tsdn, extent_hooks_t *extent_hooks)
{
ctl_arena_init(tsdn_t *tsdn, extent_hooks_t *extent_hooks) {
unsigned arena_ind;
ctl_arena_t *ctl_arena;
@@ -692,26 +686,29 @@ ctl_arena_init(tsdn_t *tsdn, extent_hooks_t *extent_hooks)
NULL) {
ql_remove(&ctl_arenas->destroyed, ctl_arena, destroyed_link);
arena_ind = ctl_arena->arena_ind;
} else
} else {
arena_ind = ctl_arenas->narenas;
}
/* Trigger stats allocation. */
if (arenas_i_impl(tsdn, arena_ind, false, true) == NULL)
if (arenas_i_impl(tsdn, arena_ind, false, true) == NULL) {
return (UINT_MAX);
}
/* Initialize new arena. */
if (arena_init(tsdn, arena_ind, extent_hooks) == NULL)
if (arena_init(tsdn, arena_ind, extent_hooks) == NULL) {
return (UINT_MAX);
}
if (arena_ind == ctl_arenas->narenas)
if (arena_ind == ctl_arenas->narenas) {
ctl_arenas->narenas++;
}
return (arena_ind);
}
static void
ctl_refresh(tsdn_t *tsdn)
{
ctl_refresh(tsdn_t *tsdn) {
unsigned i;
ctl_arena_t *ctl_sarena = arenas_i(MALLCTL_ARENAS_ALL);
VARIABLE_ARRAY(arena_t *, tarenas, ctl_arenas->narenas);
@@ -751,8 +748,7 @@ ctl_refresh(tsdn_t *tsdn)
}
static bool
ctl_init(tsdn_t *tsdn)
{
ctl_init(tsdn_t *tsdn) {
bool ret;
malloc_mutex_lock(tsdn, &ctl_mtx);
@@ -828,8 +824,7 @@ label_return:
static int
ctl_lookup(tsdn_t *tsdn, const char *name, ctl_node_t const **nodesp,
size_t *mibp, size_t *depthp)
{
size_t *mibp, size_t *depthp) {
int ret;
const char *elm, *tdot, *dot;
size_t elen, i, j;
@@ -857,9 +852,10 @@ ctl_lookup(tsdn_t *tsdn, const char *name, ctl_node_t const **nodesp,
if (strlen(child->name) == elen &&
strncmp(elm, child->name, elen) == 0) {
node = child;
if (nodesp != NULL)
if (nodesp != NULL) {
nodesp[i] =
(const ctl_node_t *)node;
}
mibp[i] = j;
break;
}
@@ -886,8 +882,9 @@ ctl_lookup(tsdn_t *tsdn, const char *name, ctl_node_t const **nodesp,
goto label_return;
}
if (nodesp != NULL)
if (nodesp != NULL) {
nodesp[i] = (const ctl_node_t *)node;
}
mibp[i] = (size_t)index;
}
@@ -925,8 +922,7 @@ label_return:
int
ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
void *newp, size_t newlen)
{
void *newp, size_t newlen) {
int ret;
size_t depth;
ctl_node_t const *nodes[CTL_MAX_DEPTH];
@@ -940,12 +936,14 @@ ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
depth = CTL_MAX_DEPTH;
ret = ctl_lookup(tsd_tsdn(tsd), name, nodes, mib, &depth);
if (ret != 0)
if (ret != 0) {
goto label_return;
}
node = ctl_named_node(nodes[depth-1]);
if (node != NULL && node->ctl)
if (node != NULL && node->ctl) {
ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen);
}
else {
/* The name refers to a partial path through the ctl tree. */
ret = ENOENT;
@@ -956,8 +954,7 @@ label_return:
}
int
ctl_nametomib(tsdn_t *tsdn, const char *name, size_t *mibp, size_t *miblenp)
{
ctl_nametomib(tsdn_t *tsdn, const char *name, size_t *mibp, size_t *miblenp) {
int ret;
if (!ctl_initialized && ctl_init(tsdn)) {
@@ -972,8 +969,7 @@ label_return:
int
ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
const ctl_named_node_t *node;
size_t i;
@@ -1009,9 +1005,9 @@ ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
}
/* Call the ctl function. */
if (node && node->ctl)
if (node && node->ctl) {
ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
else {
} else {
/* Partial MIB. */
ret = ENOENT;
}
@@ -1021,10 +1017,10 @@ label_return:
}
bool
ctl_boot(void)
{
if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL))
ctl_boot(void) {
if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL)) {
return (true);
}
ctl_initialized = false;
@@ -1032,20 +1028,17 @@ ctl_boot(void)
}
void
ctl_prefork(tsdn_t *tsdn)
{
ctl_prefork(tsdn_t *tsdn) {
malloc_mutex_prefork(tsdn, &ctl_mtx);
}
void
ctl_postfork_parent(tsdn_t *tsdn)
{
ctl_postfork_parent(tsdn_t *tsdn) {
malloc_mutex_postfork_parent(tsdn, &ctl_mtx);
}
void
ctl_postfork_child(tsdn_t *tsdn)
{
ctl_postfork_child(tsdn_t *tsdn) {
malloc_mutex_postfork_child(tsdn, &ctl_mtx);
}
@@ -1112,36 +1105,38 @@ ctl_postfork_child(tsdn_t *tsdn)
#define CTL_RO_CLGEN(c, l, n, v, t) \
static int \
n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen) \
{ \
size_t *oldlenp, void *newp, size_t newlen) { \
int ret; \
t oldval; \
\
if (!(c)) \
if (!(c)) { \
return (ENOENT); \
if (l) \
} \
if (l) { \
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
} \
READONLY(); \
oldval = (v); \
READ(oldval, t); \
\
ret = 0; \
label_return: \
if (l) \
if (l) { \
malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); \
} \
return (ret); \
}
#define CTL_RO_CGEN(c, n, v, t) \
static int \
n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen) \
{ \
size_t *oldlenp, void *newp, size_t newlen) { \
int ret; \
t oldval; \
\
if (!(c)) \
if (!(c)) { \
return (ENOENT); \
} \
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); \
READONLY(); \
oldval = (v); \
@@ -1156,8 +1151,7 @@ label_return: \
#define CTL_RO_GEN(n, v, t) \
static int \
n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen) \
{ \
size_t *oldlenp, void *newp, size_t newlen) { \
int ret; \
t oldval; \
\
@@ -1179,13 +1173,13 @@ label_return: \
#define CTL_RO_NL_CGEN(c, n, v, t) \
static int \
n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen) \
{ \
size_t *oldlenp, void *newp, size_t newlen) { \
int ret; \
t oldval; \
\
if (!(c)) \
if (!(c)) { \
return (ENOENT); \
} \
READONLY(); \
oldval = (v); \
READ(oldval, t); \
@@ -1198,8 +1192,7 @@ label_return: \
#define CTL_RO_NL_GEN(n, v, t) \
static int \
n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen) \
{ \
size_t *oldlenp, void *newp, size_t newlen) { \
int ret; \
t oldval; \
\
@@ -1215,13 +1208,13 @@ label_return: \
#define CTL_TSD_RO_NL_CGEN(c, n, m, t) \
static int \
n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen) \
{ \
size_t *oldlenp, void *newp, size_t newlen) { \
int ret; \
t oldval; \
\
if (!(c)) \
if (!(c)) { \
return (ENOENT); \
} \
READONLY(); \
oldval = (m(tsd)); \
READ(oldval, t); \
@@ -1234,8 +1227,7 @@ label_return: \
#define CTL_RO_CONFIG_GEN(n, t) \
static int \
n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen) \
{ \
size_t *oldlenp, void *newp, size_t newlen) { \
int ret; \
t oldval; \
\
@@ -1254,15 +1246,15 @@ CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *)
static int
epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
UNUSED uint64_t newval;
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
WRITE(newval, uint64_t);
if (newp != NULL)
if (newp != NULL) {
ctl_refresh(tsd_tsdn(tsd));
}
READ(ctl_arenas->epoch, uint64_t);
ret = 0;
@@ -1317,15 +1309,15 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool)
static int
thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
arena_t *oldarena;
unsigned newind, oldind;
oldarena = arena_choose(tsd, NULL);
if (oldarena == NULL)
if (oldarena == NULL) {
return (EAGAIN);
}
newind = oldind = arena_ind_get(oldarena);
WRITE(newind, unsigned);
@@ -1372,13 +1364,13 @@ CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp,
static int
thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
bool oldval;
if (!config_tcache)
if (!config_tcache) {
return (ENOENT);
}
oldval = tcache_enabled_get();
if (newp != NULL) {
@@ -1397,12 +1389,12 @@ label_return:
static int
thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
if (!config_tcache)
if (!config_tcache) {
return (ENOENT);
}
READONLY();
WRITEONLY();
@@ -1416,12 +1408,12 @@ label_return:
static int
thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
if (!config_prof)
if (!config_prof) {
return (ENOENT);
}
READ_XOR_WRITE();
@@ -1432,8 +1424,9 @@ thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
}
if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) !=
0)
0) {
goto label_return;
}
} else {
const char *oldname = prof_thread_name_get(tsd);
READ(oldname, const char *);
@@ -1446,13 +1439,13 @@ label_return:
static int
thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
bool oldval;
if (!config_prof)
if (!config_prof) {
return (ENOENT);
}
oldval = prof_thread_active_get(tsd);
if (newp != NULL) {
@@ -1476,13 +1469,13 @@ label_return:
static int
tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned tcache_ind;
if (!config_tcache)
if (!config_tcache) {
return (ENOENT);
}
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
READONLY();
@@ -1500,13 +1493,13 @@ label_return:
static int
tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned tcache_ind;
if (!config_tcache)
if (!config_tcache) {
return (ENOENT);
}
WRITEONLY();
tcache_ind = UINT_MAX;
@@ -1524,13 +1517,13 @@ label_return:
static int
tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned tcache_ind;
if (!config_tcache)
if (!config_tcache) {
return (ENOENT);
}
WRITEONLY();
tcache_ind = UINT_MAX;
@@ -1550,8 +1543,7 @@ label_return:
static int
arena_i_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
tsdn_t *tsdn = tsd_tsdn(tsd);
unsigned arena_ind;
@@ -1572,8 +1564,7 @@ label_return:
}
static void
arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all)
{
arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all) {
malloc_mutex_lock(tsdn, &ctl_mtx);
{
unsigned narenas = ctl_arenas->narenas;
@@ -1586,8 +1577,9 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all)
unsigned i;
VARIABLE_ARRAY(arena_t *, tarenas, narenas);
for (i = 0; i < narenas; i++)
for (i = 0; i < narenas; i++) {
tarenas[i] = arena_get(tsdn, i, false);
}
/*
* No further need to hold ctl_mtx, since narenas and
@@ -1596,8 +1588,9 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all)
malloc_mutex_unlock(tsdn, &ctl_mtx);
for (i = 0; i < narenas; i++) {
if (tarenas[i] != NULL)
if (tarenas[i] != NULL) {
arena_purge(tsdn, tarenas[i], all);
}
}
} else {
arena_t *tarena;
@@ -1609,16 +1602,16 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all)
/* No further need to hold ctl_mtx. */
malloc_mutex_unlock(tsdn, &ctl_mtx);
if (tarena != NULL)
if (tarena != NULL) {
arena_purge(tsdn, tarena, all);
}
}
}
}
static int
arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
@@ -1634,8 +1627,7 @@ label_return:
static int
arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
@@ -1652,8 +1644,7 @@ label_return:
static int
arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen, unsigned *arena_ind,
arena_t **arena)
{
arena_t **arena) {
int ret;
READONLY();
@@ -1678,16 +1669,16 @@ label_return:
static int
arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
arena_t *arena;
ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
newp, newlen, &arena_ind, &arena);
if (ret != 0)
if (ret != 0) {
return (ret);
}
arena_reset(tsd, arena);
@@ -1696,8 +1687,7 @@ arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
static int
arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
arena_t *arena;
@@ -1705,8 +1695,9 @@ arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = arena_i_reset_destroy_helper(tsd, mib, miblen, oldp, oldlenp,
newp, newlen, &arena_ind, &arena);
if (ret != 0)
if (ret != 0) {
goto label_return;
}
if (arena_nthreads_get(arena, false) != 0 || arena_nthreads_get(arena,
true) != 0) {
@@ -1735,8 +1726,7 @@ label_return:
static int
arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
const char *dss = NULL;
unsigned arena_ind;
@@ -1797,8 +1787,7 @@ label_return:
static int
arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
arena_t *arena;
@@ -1833,8 +1822,7 @@ label_return:
static int
arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
arena_t *arena;
@@ -1867,8 +1855,7 @@ label_return:
}
static const ctl_named_node_t *
arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
{
arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
const ctl_named_node_t *ret;
malloc_mutex_lock(tsdn, &ctl_mtx);
@@ -1894,8 +1881,7 @@ label_return:
static int
arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned narenas;
@@ -1916,8 +1902,7 @@ label_return:
static int
arenas_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
if (oldp != NULL && oldlenp != NULL) {
@@ -1949,27 +1934,27 @@ CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t)
CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t)
CTL_RO_NL_GEN(arenas_bin_i_slab_size, arena_bin_info[mib[2]].slab_size, size_t)
static const ctl_named_node_t *
arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
{
if (i > NBINS)
arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
if (i > NBINS) {
return (NULL);
}
return (super_arenas_bin_i_node);
}
CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned)
CTL_RO_NL_GEN(arenas_lextent_i_size, index2size(NBINS+(szind_t)mib[2]), size_t)
static const ctl_named_node_t *
arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
{
if (i > NSIZES - NBINS)
arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
size_t i) {
if (i > NSIZES - NBINS) {
return (NULL);
}
return (super_arenas_lextent_i_node);
}
static int
arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
extent_hooks_t *extent_hooks;
unsigned arena_ind;
@@ -1995,13 +1980,13 @@ label_return:
static int
prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
bool oldval;
if (!config_prof)
if (!config_prof) {
return (ENOENT);
}
if (newp != NULL) {
if (newlen != sizeof(bool)) {
@@ -2010,8 +1995,9 @@ prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
}
oldval = prof_thread_active_init_set(tsd_tsdn(tsd),
*(bool *)newp);
} else
} else {
oldval = prof_thread_active_init_get(tsd_tsdn(tsd));
}
READ(oldval, bool);
ret = 0;
@@ -2021,13 +2007,13 @@ label_return:
static int
prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
bool oldval;
if (!config_prof)
if (!config_prof) {
return (ENOENT);
}
if (newp != NULL) {
if (newlen != sizeof(bool)) {
@@ -2035,8 +2021,9 @@ prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
goto label_return;
}
oldval = prof_active_set(tsd_tsdn(tsd), *(bool *)newp);
} else
} else {
oldval = prof_active_get(tsd_tsdn(tsd));
}
READ(oldval, bool);
ret = 0;
@@ -2046,13 +2033,13 @@ label_return:
static int
prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
const char *filename = NULL;
if (!config_prof)
if (!config_prof) {
return (ENOENT);
}
WRITEONLY();
WRITE(filename, const char *);
@@ -2069,13 +2056,13 @@ label_return:
static int
prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
bool oldval;
if (!config_prof)
if (!config_prof) {
return (ENOENT);
}
if (newp != NULL) {
if (newlen != sizeof(bool)) {
@@ -2083,8 +2070,9 @@ prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
goto label_return;
}
oldval = prof_gdump_set(tsd_tsdn(tsd), *(bool *)newp);
} else
} else {
oldval = prof_gdump_get(tsd_tsdn(tsd));
}
READ(oldval, bool);
ret = 0;
@@ -2094,18 +2082,19 @@ label_return:
static int
prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
size_t lg_sample = lg_prof_sample;
if (!config_prof)
if (!config_prof) {
return (ENOENT);
}
WRITEONLY();
WRITE(lg_sample, size_t);
if (lg_sample >= (sizeof(uint64_t) << 3))
if (lg_sample >= (sizeof(uint64_t) << 3)) {
lg_sample = (sizeof(uint64_t) << 3) - 1;
}
prof_reset(tsd, lg_sample);
@@ -2189,10 +2178,10 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs,
static const ctl_named_node_t *
stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
size_t j)
{
if (j > NBINS)
size_t j) {
if (j > NBINS) {
return (NULL);
}
return (super_stats_arenas_i_bins_j_node);
}
@@ -2207,16 +2196,15 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents,
static const ctl_named_node_t *
stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
size_t j)
{
if (j > NSIZES - NBINS)
size_t j) {
if (j > NSIZES - NBINS) {
return (NULL);
}
return (super_stats_arenas_i_lextents_j_node);
}
static const ctl_named_node_t *
stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
{
stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
const ctl_named_node_t *ret;
size_t a;

375
src/extent.c
View File

@@ -75,8 +75,7 @@ static void extent_record(tsdn_t *tsdn, arena_t *arena,
/******************************************************************************/
extent_t *
extent_alloc(tsdn_t *tsdn, arena_t *arena)
{
extent_alloc(tsdn_t *tsdn, arena_t *arena) {
extent_t *extent;
malloc_mutex_lock(tsdn, &arena->extent_cache_mtx);
@@ -92,8 +91,7 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena)
}
void
extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
{
extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
malloc_mutex_lock(tsdn, &arena->extent_cache_mtx);
ql_elm_new(extent, ql_link);
ql_tail_insert(&arena->extent_cache, extent, ql_link);
@@ -101,22 +99,21 @@ extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
}
extent_hooks_t *
extent_hooks_get(arena_t *arena)
{
extent_hooks_get(arena_t *arena) {
return (base_extent_hooks_get(arena->base));
}
extent_hooks_t *
extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks)
{
extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks) {
return (base_extent_hooks_set(arena->base, extent_hooks));
}
static void
extent_hooks_assure_initialized(arena_t *arena, extent_hooks_t **r_extent_hooks)
{
if (*r_extent_hooks == EXTENT_HOOKS_INITIALIZER)
extent_hooks_assure_initialized(arena_t *arena,
extent_hooks_t **r_extent_hooks) {
if (*r_extent_hooks == EXTENT_HOOKS_INITIALIZER) {
*r_extent_hooks = extent_hooks_get(arena);
}
}
#ifdef JEMALLOC_JET
@@ -124,8 +121,7 @@ extent_hooks_assure_initialized(arena_t *arena, extent_hooks_t **r_extent_hooks)
#define extent_size_quantize_floor JEMALLOC_N(n_extent_size_quantize_floor)
#endif
size_t
extent_size_quantize_floor(size_t size)
{
extent_size_quantize_floor(size_t size) {
size_t ret;
pszind_t pind;
@@ -161,8 +157,7 @@ extent_size_quantize_t *extent_size_quantize_floor =
#define extent_size_quantize_ceil JEMALLOC_N(n_extent_size_quantize_ceil)
#endif
size_t
extent_size_quantize_ceil(size_t size)
{
extent_size_quantize_ceil(size_t size) {
size_t ret;
assert(size > 0);
@@ -195,8 +190,7 @@ ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp)
static void
extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1],
extent_t *extent)
{
extent_t *extent) {
size_t psz = extent_size_quantize_floor(extent_size_get(extent));
pszind_t pind = psz2ind(psz);
@@ -207,8 +201,7 @@ extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1],
static void
extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1],
extent_t *extent)
{
extent_t *extent) {
size_t psz = extent_size_quantize_floor(extent_size_get(extent));
pszind_t pind = psz2ind(psz);
@@ -220,12 +213,12 @@ extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1],
static bool
extent_rtree_acquire(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
const extent_t *extent, bool dependent, bool init_missing,
rtree_elm_t **r_elm_a, rtree_elm_t **r_elm_b)
{
rtree_elm_t **r_elm_a, rtree_elm_t **r_elm_b) {
*r_elm_a = rtree_elm_acquire(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)extent_base_get(extent), dependent, init_missing);
if (!dependent && *r_elm_a == NULL)
if (!dependent && *r_elm_a == NULL) {
return (true);
}
assert(*r_elm_a != NULL);
if (extent_size_get(extent) > PAGE) {
@@ -237,33 +230,33 @@ extent_rtree_acquire(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
return (true);
}
assert(*r_elm_b != NULL);
} else
} else {
*r_elm_b = NULL;
}
return (false);
}
static void
extent_rtree_write_acquired(tsdn_t *tsdn, rtree_elm_t *elm_a,
rtree_elm_t *elm_b, const extent_t *extent)
{
rtree_elm_t *elm_b, const extent_t *extent) {
rtree_elm_write_acquired(tsdn, &extents_rtree, elm_a, extent);
if (elm_b != NULL)
if (elm_b != NULL) {
rtree_elm_write_acquired(tsdn, &extents_rtree, elm_b, extent);
}
}
static void
extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b)
{
extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b) {
rtree_elm_release(tsdn, &extents_rtree, elm_a);
if (elm_b != NULL)
if (elm_b != NULL) {
rtree_elm_release(tsdn, &extents_rtree, elm_b);
}
}
static void
extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
const extent_t *extent)
{
const extent_t *extent) {
size_t i;
assert(extent_slab_get(extent));
@@ -276,8 +269,7 @@ extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
}
static void
extent_gprof_add(tsdn_t *tsdn, const extent_t *extent)
{
extent_gprof_add(tsdn_t *tsdn, const extent_t *extent) {
cassert(config_prof);
if (opt_prof && extent_active_get(extent)) {
@@ -291,14 +283,14 @@ extent_gprof_add(tsdn_t *tsdn, const extent_t *extent)
*/
high = atomic_read_zu(&highpages);
}
if (cur > high && prof_gdump_get_unlocked())
if (cur > high && prof_gdump_get_unlocked()) {
prof_gdump(tsdn);
}
}
}
static void
extent_gprof_sub(tsdn_t *tsdn, const extent_t *extent)
{
extent_gprof_sub(tsdn_t *tsdn, const extent_t *extent) {
cassert(config_prof);
if (opt_prof && extent_active_get(extent)) {
@@ -309,37 +301,37 @@ extent_gprof_sub(tsdn_t *tsdn, const extent_t *extent)
}
static bool
extent_register(tsdn_t *tsdn, const extent_t *extent)
{
extent_register(tsdn_t *tsdn, const extent_t *extent) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_elm_t *elm_a, *elm_b;
if (extent_rtree_acquire(tsdn, rtree_ctx, extent, false, true, &elm_a,
&elm_b))
&elm_b)) {
return (true);
}
extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent);
if (extent_slab_get(extent))
if (extent_slab_get(extent)) {
extent_interior_register(tsdn, rtree_ctx, extent);
}
extent_rtree_release(tsdn, elm_a, elm_b);
if (config_prof)
if (config_prof) {
extent_gprof_add(tsdn, extent);
}
return (false);
}
static void
extent_reregister(tsdn_t *tsdn, const extent_t *extent)
{
extent_reregister(tsdn_t *tsdn, const extent_t *extent) {
bool err = extent_register(tsdn, extent);
assert(!err);
}
static void
extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
const extent_t *extent)
{
const extent_t *extent) {
size_t i;
assert(extent_slab_get(extent));
@@ -352,8 +344,7 @@ extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
}
static void
extent_deregister(tsdn_t *tsdn, extent_t *extent)
{
extent_deregister(tsdn_t *tsdn, extent_t *extent) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_elm_t *elm_a, *elm_b;
@@ -367,8 +358,9 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent)
}
extent_rtree_release(tsdn, elm_a, elm_b);
if (config_prof)
if (config_prof) {
extent_gprof_sub(tsdn, extent);
}
}
/*
@@ -377,8 +369,7 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent)
*/
static extent_t *
extent_first_best_fit(tsdn_t *tsdn, arena_t *arena,
extent_heap_t extent_heaps[NPSIZES+1], size_t size)
{
extent_heap_t extent_heaps[NPSIZES+1], size_t size) {
pszind_t pind, i;
malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
@@ -386,8 +377,9 @@ extent_first_best_fit(tsdn_t *tsdn, arena_t *arena,
pind = psz2ind(extent_size_quantize_ceil(size));
for (i = pind; i < NPSIZES+1; i++) {
extent_t *extent = extent_heap_first(&extent_heaps[i]);
if (extent != NULL)
if (extent != NULL) {
return (extent);
}
}
return (NULL);
@@ -395,8 +387,7 @@ extent_first_best_fit(tsdn_t *tsdn, arena_t *arena,
static void
extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
bool cache, extent_t *extent)
{
bool cache, extent_t *extent) {
/*
* Leak extent after making sure its pages have already been purged, so
* that this is only a virtual memory leak.
@@ -415,15 +406,15 @@ static extent_t *
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_heap_t extent_heaps[NPSIZES+1], bool locked, bool cache,
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
bool *commit, bool slab)
{
bool *commit, bool slab) {
extent_t *extent;
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
size_t size, alloc_size, leadsize, trailsize;
if (locked)
if (locked) {
malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
}
assert(new_addr == NULL || !slab);
assert(pad == 0 || !slab);
assert(alignment > 0);
@@ -452,10 +443,12 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
size = usize + pad;
alloc_size = size + PAGE_CEILING(alignment) - PAGE;
/* Beware size_t wrap-around. */
if (alloc_size < usize)
if (alloc_size < usize) {
return (NULL);
if (!locked)
}
if (!locked) {
malloc_mutex_lock(tsdn, &arena->extents_mtx);
}
extent_hooks_assure_initialized(arena, r_extent_hooks);
if (new_addr != NULL) {
rtree_elm_t *elm;
@@ -470,19 +463,22 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
if (extent_arena_get(extent) != arena ||
extent_size_get(extent) < size ||
extent_active_get(extent) ||
extent_retained_get(extent) == cache)
extent_retained_get(extent) == cache) {
extent = NULL;
}
}
rtree_elm_release(tsdn, &extents_rtree, elm);
} else
} else {
extent = NULL;
}
} else {
extent = extent_first_best_fit(tsdn, arena, extent_heaps,
alloc_size);
}
if (extent == NULL) {
if (!locked)
if (!locked) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
}
return (NULL);
}
extent_heaps_remove(tsdn, extent_heaps, extent);
@@ -493,10 +489,12 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
assert(new_addr == NULL || leadsize == 0);
assert(extent_size_get(extent) >= leadsize + size);
trailsize = extent_size_get(extent) - leadsize - size;
if (extent_zeroed_get(extent))
if (extent_zeroed_get(extent)) {
*zero = true;
if (extent_committed_get(extent))
}
if (extent_committed_get(extent)) {
*commit = true;
}
/* Split the lead. */
if (leadsize != 0) {
@@ -507,8 +505,9 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
if (extent == NULL) {
extent_deregister(tsdn, lead);
extent_leak(tsdn, arena, r_extent_hooks, cache, lead);
if (!locked)
if (!locked) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
}
return (NULL);
}
extent_heaps_insert(tsdn, extent_heaps, lead);
@@ -523,8 +522,9 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_deregister(tsdn, extent);
extent_leak(tsdn, arena, r_extent_hooks, cache,
extent);
if (!locked)
if (!locked) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
}
return (NULL);
}
extent_heaps_insert(tsdn, extent_heaps, trail);
@@ -540,8 +540,9 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
if (*commit && !extent_committed_get(extent)) {
if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent,
0, extent_size_get(extent))) {
if (!locked)
if (!locked) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
}
extent_record(tsdn, arena, r_extent_hooks, extent_heaps,
cache, extent);
return (NULL);
@@ -549,16 +550,18 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_zeroed_set(extent, true);
}
if (pad != 0)
if (pad != 0) {
extent_addr_randomize(tsdn, extent, alignment);
}
extent_active_set(extent, true);
if (slab) {
extent_slab_set(extent, slab);
extent_interior_register(tsdn, rtree_ctx, extent);
}
if (!locked)
if (!locked) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
}
if (*zero) {
if (!extent_zeroed_get(extent)) {
@@ -569,8 +572,9 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
size_t *p = (size_t *)(uintptr_t)
extent_addr_get(extent);
for (i = 0; i < usize / sizeof(size_t); i++)
for (i = 0; i < usize / sizeof(size_t); i++) {
assert(p[i] == 0);
}
}
}
return (extent);
@@ -584,8 +588,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
*/
static void *
extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec)
{
size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec) {
void *ret;
assert(size != 0);
@@ -594,17 +597,20 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
/* "primary" dss. */
if (have_dss && dss_prec == dss_prec_primary && (ret =
extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
commit)) != NULL)
commit)) != NULL) {
return (ret);
}
/* mmap. */
if ((ret = extent_alloc_mmap(new_addr, size, alignment, zero, commit))
!= NULL)
!= NULL) {
return (ret);
}
/* "secondary" dss. */
if (have_dss && dss_prec == dss_prec_secondary && (ret =
extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
commit)) != NULL)
commit)) != NULL) {
return (ret);
}
/* All strategies for allocation failed. */
return (NULL);
@@ -613,8 +619,7 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
static extent_t *
extent_alloc_cache_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, bool locked, void *new_addr, size_t usize,
size_t pad, size_t alignment, bool *zero, bool *commit, bool slab)
{
size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) {
extent_t *extent;
assert(usize + pad != 0);
@@ -629,8 +634,7 @@ extent_alloc_cache_impl(tsdn_t *tsdn, arena_t *arena,
extent_t *
extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
size_t alignment, bool *zero, bool *commit, bool slab) {
malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
return (extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, true,
@@ -640,16 +644,14 @@ extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena,
extent_t *
extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
size_t alignment, bool *zero, bool *commit, bool slab) {
return (extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, false,
new_addr, usize, pad, alignment, zero, commit, slab));
}
static void *
extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit)
{
size_t size, size_t alignment, bool *zero, bool *commit) {
void *ret;
ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
@@ -659,8 +661,7 @@ extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,
static void *
extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind)
{
size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
tsdn_t *tsdn;
arena_t *arena;
@@ -680,10 +681,10 @@ extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
static void
extent_retain(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_t *extent)
{
if (config_stats)
extent_t *extent) {
if (config_stats) {
arena->stats.retained += extent_size_get(extent);
}
extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained,
false, extent);
}
@@ -696,8 +697,7 @@ extent_retain(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
static extent_t *
extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
size_t alignment, bool *zero, bool *commit, bool slab) {
extent_t *extent;
void *ptr;
size_t size, alloc_size, alloc_size_min, leadsize, trailsize;
@@ -713,13 +713,16 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
alloc_size = pind2sz(arena->extent_grow_next);
alloc_size_min = size + PAGE_CEILING(alignment) - PAGE;
/* Beware size_t wrap-around. */
if (alloc_size_min < usize)
if (alloc_size_min < usize) {
return (NULL);
if (alloc_size < alloc_size_min)
}
if (alloc_size < alloc_size_min) {
return (NULL);
}
extent = extent_alloc(tsdn, arena);
if (extent == NULL)
if (extent == NULL) {
return (NULL);
}
zeroed = false;
committed = false;
ptr = extent_alloc_core(tsdn, arena, new_addr, alloc_size, PAGE,
@@ -741,10 +744,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
assert(new_addr == NULL || leadsize == 0);
assert(alloc_size >= leadsize + size);
trailsize = alloc_size - leadsize - size;
if (extent_zeroed_get(extent))
if (extent_zeroed_get(extent)) {
*zero = true;
if (extent_committed_get(extent))
}
if (extent_committed_get(extent)) {
*commit = true;
}
/* Split the lead. */
if (leadsize != 0) {
@@ -790,8 +795,9 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
/* Adjust gprof stats now that extent is final size. */
extent_gprof_add(tsdn, extent);
}
if (pad != 0)
if (pad != 0) {
extent_addr_randomize(tsdn, extent, alignment);
}
if (slab) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
@@ -800,18 +806,19 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
extent_slab_set(extent, true);
extent_interior_register(tsdn, rtree_ctx, extent);
}
if (*zero && !extent_zeroed_get(extent))
if (*zero && !extent_zeroed_get(extent)) {
memset(extent_addr_get(extent), 0, extent_usize_get(extent));
if (arena->extent_grow_next + 1 < NPSIZES)
}
if (arena->extent_grow_next + 1 < NPSIZES) {
arena->extent_grow_next++;
}
return (extent);
}
static extent_t *
extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
size_t alignment, bool *zero, bool *commit, bool slab) {
extent_t *extent;
assert(usize != 0);
@@ -825,8 +832,9 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
size_t size = usize + pad;
arena->stats.retained -= size;
}
if (config_prof)
if (config_prof) {
extent_gprof_add(tsdn, extent);
}
}
if (!config_munmap && extent == NULL) {
extent = extent_grow_retained(tsdn, arena, r_extent_hooks,
@@ -839,16 +847,16 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
static extent_t *
extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
size_t alignment, bool *zero, bool *commit, bool slab) {
extent_t *extent;
size_t size;
void *addr;
size = usize + pad;
extent = extent_alloc(tsdn, arena);
if (extent == NULL)
if (extent == NULL) {
return (NULL);
}
if (*r_extent_hooks == &extent_hooks_default) {
/* Call directly to propagate tsdn. */
addr = extent_alloc_default_impl(tsdn, arena, new_addr, size,
@@ -863,8 +871,9 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
}
extent_init(extent, arena, addr, size, usize,
arena_extent_sn_next(arena), true, zero, commit, slab);
if (pad != 0)
if (pad != 0) {
extent_addr_randomize(tsdn, extent, alignment);
}
if (extent_register(tsdn, extent)) {
extent_leak(tsdn, arena, r_extent_hooks, false, extent);
return (NULL);
@@ -876,8 +885,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
extent_t *
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
size_t alignment, bool *zero, bool *commit, bool slab) {
extent_t *extent;
extent_hooks_assure_initialized(arena, r_extent_hooks);
@@ -893,16 +901,19 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
}
static bool
extent_can_coalesce(const extent_t *a, const extent_t *b)
{
if (extent_arena_get(a) != extent_arena_get(b))
extent_can_coalesce(const extent_t *a, const extent_t *b) {
if (extent_arena_get(a) != extent_arena_get(b)) {
return (false);
if (extent_active_get(a) != extent_active_get(b))
}
if (extent_active_get(a) != extent_active_get(b)) {
return (false);
if (extent_committed_get(a) != extent_committed_get(b))
}
if (extent_committed_get(a) != extent_committed_get(b)) {
return (false);
if (extent_retained_get(a) != extent_retained_get(b))
}
if (extent_retained_get(a) != extent_retained_get(b)) {
return (false);
}
return (true);
}
@@ -910,10 +921,10 @@ extent_can_coalesce(const extent_t *a, const extent_t *b)
static void
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b,
extent_heap_t extent_heaps[NPSIZES+1], bool cache)
{
if (!extent_can_coalesce(a, b))
extent_heap_t extent_heaps[NPSIZES+1], bool cache) {
if (!extent_can_coalesce(a, b)) {
return;
}
extent_heaps_remove(tsdn, extent_heaps, a);
extent_heaps_remove(tsdn, extent_heaps, b);
@@ -937,8 +948,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
static void
extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_heap_t extent_heaps[NPSIZES+1], bool cache, extent_t *extent)
{
extent_heap_t extent_heaps[NPSIZES+1], bool cache, extent_t *extent) {
extent_t *prev, *next;
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
@@ -980,8 +990,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
}
void
extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
{
extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
if (extent_register(tsdn, extent)) {
@@ -993,8 +1002,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
void
extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent)
{
extent_hooks_t **r_extent_hooks, extent_t *extent) {
assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0);
@@ -1006,17 +1014,16 @@ extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
}
static bool
extent_dalloc_default_impl(void *addr, size_t size)
{
if (!have_dss || !extent_in_dss(addr))
extent_dalloc_default_impl(void *addr, size_t size) {
if (!have_dss || !extent_in_dss(addr)) {
return (extent_dalloc_mmap(addr, size));
}
return (true);
}
static bool
extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind)
{
bool committed, unsigned arena_ind) {
assert(extent_hooks == &extent_hooks_default);
return (extent_dalloc_default_impl(addr, size));
@@ -1024,8 +1031,7 @@ extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool
extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent)
{
extent_hooks_t **r_extent_hooks, extent_t *extent) {
bool err;
assert(extent_base_get(extent) != NULL);
@@ -1050,46 +1056,50 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
extent_committed_get(extent), arena_ind_get(arena)));
}
if (!err)
if (!err) {
extent_dalloc(tsdn, arena, extent);
}
return (err);
}
void
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent)
{
extent_hooks_t **r_extent_hooks, extent_t *extent) {
bool zeroed;
if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent))
if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) {
return;
}
extent_reregister(tsdn, extent);
/* Try to decommit; purge if that fails. */
if (!extent_committed_get(extent))
if (!extent_committed_get(extent)) {
zeroed = true;
else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
0, extent_size_get(extent)))
} else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
0, extent_size_get(extent))) {
zeroed = true;
else if ((*r_extent_hooks)->purge_lazy != NULL &&
} else if ((*r_extent_hooks)->purge_lazy != NULL &&
!(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena_ind_get(arena)))
extent_size_get(extent), arena_ind_get(arena))) {
zeroed = false;
else if ((*r_extent_hooks)->purge_forced != NULL &&
} else if ((*r_extent_hooks)->purge_forced != NULL &&
!(*r_extent_hooks)->purge_forced(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena_ind_get(arena)))
extent_size_get(extent), arena_ind_get(arena))) {
zeroed = true;
else
} else {
zeroed = false;
}
extent_zeroed_set(extent, zeroed);
if (config_stats)
if (config_stats) {
arena->stats.retained += extent_size_get(extent);
if (config_prof)
}
if (config_prof) {
extent_gprof_sub(tsdn, extent);
}
extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained,
false, extent);
@@ -1097,8 +1107,7 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
static bool
extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
size_t offset, size_t length, unsigned arena_ind) {
assert(extent_hooks == &extent_hooks_default);
return (pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset),
@@ -1108,8 +1117,7 @@ extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool
extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length)
{
size_t length) {
bool err;
extent_hooks_assure_initialized(arena, r_extent_hooks);
@@ -1122,8 +1130,7 @@ extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
static bool
extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
size_t offset, size_t length, unsigned arena_ind) {
assert(extent_hooks == &extent_hooks_default);
return (pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset),
@@ -1133,8 +1140,7 @@ extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool
extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length)
{
size_t length) {
bool err;
extent_hooks_assure_initialized(arena, r_extent_hooks);
@@ -1150,8 +1156,7 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
#ifdef PAGES_CAN_PURGE_LAZY
static bool
extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
size_t offset, size_t length, unsigned arena_ind) {
assert(extent_hooks == &extent_hooks_default);
assert(addr != NULL);
assert((offset & PAGE_MASK) == 0);
@@ -1166,8 +1171,7 @@ extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool
extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length)
{
size_t length) {
extent_hooks_assure_initialized(arena, r_extent_hooks);
return ((*r_extent_hooks)->purge_lazy == NULL ||
(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
@@ -1178,8 +1182,7 @@ extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
#ifdef PAGES_CAN_PURGE_FORCED
static bool
extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind)
{
size_t size, size_t offset, size_t length, unsigned arena_ind) {
assert(extent_hooks == &extent_hooks_default);
assert(addr != NULL);
assert((offset & PAGE_MASK) == 0);
@@ -1194,8 +1197,7 @@ extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
bool
extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length)
{
size_t length) {
extent_hooks_assure_initialized(arena, r_extent_hooks);
return ((*r_extent_hooks)->purge_forced == NULL ||
(*r_extent_hooks)->purge_forced(*r_extent_hooks,
@@ -1206,12 +1208,12 @@ extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
#ifdef JEMALLOC_MAPS_COALESCE
static bool
extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t size_a, size_t size_b, bool committed, unsigned arena_ind)
{
size_t size_a, size_t size_b, bool committed, unsigned arena_ind) {
assert(extent_hooks == &extent_hooks_default);
if (!maps_coalesce)
if (!maps_coalesce) {
return (true);
}
return (false);
}
#endif
@@ -1219,8 +1221,7 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
extent_t *
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
size_t usize_a, size_t size_b, size_t usize_b)
{
size_t usize_a, size_t size_b, size_t usize_b) {
extent_t *trail;
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
@@ -1230,12 +1231,14 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_assure_initialized(arena, r_extent_hooks);
if ((*r_extent_hooks)->split == NULL)
if ((*r_extent_hooks)->split == NULL) {
return (NULL);
}
trail = extent_alloc(tsdn, arena);
if (trail == NULL)
if (trail == NULL) {
goto label_error_a;
}
{
extent_t lead;
@@ -1246,8 +1249,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_slab_get(extent));
if (extent_rtree_acquire(tsdn, rtree_ctx, &lead, false, true,
&lead_elm_a, &lead_elm_b))
&lead_elm_a, &lead_elm_b)) {
goto label_error_b;
}
}
extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) +
@@ -1255,13 +1259,15 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_active_get(extent), extent_zeroed_get(extent),
extent_committed_get(extent), extent_slab_get(extent));
if (extent_rtree_acquire(tsdn, rtree_ctx, trail, false, true,
&trail_elm_a, &trail_elm_b))
&trail_elm_a, &trail_elm_b)) {
goto label_error_c;
}
if ((*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent),
size_a + size_b, size_a, size_b, extent_committed_get(extent),
arena_ind_get(arena)))
arena_ind_get(arena))) {
goto label_error_d;
}
extent_size_set(extent, size_a);
extent_usize_set(extent, usize_a);
@@ -1284,12 +1290,13 @@ label_error_a:
}
static bool
extent_merge_default_impl(void *addr_a, void *addr_b)
{
if (!maps_coalesce)
extent_merge_default_impl(void *addr_a, void *addr_b) {
if (!maps_coalesce) {
return (true);
if (have_dss && !extent_dss_mergeable(addr_a, addr_b))
}
if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) {
return (true);
}
return (false);
}
@@ -1297,8 +1304,7 @@ extent_merge_default_impl(void *addr_a, void *addr_b)
#ifdef JEMALLOC_MAPS_COALESCE
static bool
extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
void *addr_b, size_t size_b, bool committed, unsigned arena_ind)
{
void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
assert(extent_hooks == &extent_hooks_default);
return (extent_merge_default_impl(addr_a, addr_b));
@@ -1307,8 +1313,7 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
bool
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b)
{
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) {
bool err;
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
@@ -1316,8 +1321,9 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_assure_initialized(arena, r_extent_hooks);
if ((*r_extent_hooks)->merge == NULL)
if ((*r_extent_hooks)->merge == NULL) {
return (true);
}
if (*r_extent_hooks == &extent_hooks_default) {
/* Call directly to propagate tsdn. */
@@ -1330,8 +1336,9 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
arena_ind_get(arena));
}
if (err)
if (err) {
return (true);
}
/*
* The rtree writes must happen while all the relevant elements are
@@ -1350,8 +1357,9 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
if (b_elm_b != NULL) {
rtree_elm_write_acquired(tsdn, &extents_rtree, b_elm_a, NULL);
rtree_elm_release(tsdn, &extents_rtree, b_elm_a);
} else
} else {
b_elm_b = b_elm_a;
}
extent_size_set(a, extent_size_get(a) + extent_size_get(b));
extent_usize_set(a, extent_usize_get(a) + extent_usize_get(b));
@@ -1368,14 +1376,15 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
}
bool
extent_boot(void)
{
extent_boot(void) {
if (rtree_new(&extents_rtree, (unsigned)((ZU(1) << (LG_SIZEOF_PTR+3)) -
LG_PAGE)))
LG_PAGE))) {
return (true);
}
if (have_dss)
if (have_dss) {
extent_dss_boot();
}
return (false);
}

68
src/extent_dss.c
View File

@@ -30,8 +30,7 @@ static void *dss_max;
/******************************************************************************/
static void *
extent_dss_sbrk(intptr_t increment)
{
extent_dss_sbrk(intptr_t increment) {
#ifdef JEMALLOC_DSS
return (sbrk(increment));
#else
@@ -41,28 +40,27 @@ extent_dss_sbrk(intptr_t increment)
}
dss_prec_t
extent_dss_prec_get(void)
{
extent_dss_prec_get(void) {
dss_prec_t ret;
if (!have_dss)
if (!have_dss) {
return (dss_prec_disabled);
}
ret = (dss_prec_t)atomic_read_u(&dss_prec_default);
return (ret);
}
bool
extent_dss_prec_set(dss_prec_t dss_prec)
{
if (!have_dss)
extent_dss_prec_set(dss_prec_t dss_prec) {
if (!have_dss) {
return (dss_prec != dss_prec_disabled);
}
atomic_write_u(&dss_prec_default, (unsigned)dss_prec);
return (false);
}
static void *
extent_dss_max_update(void *new_addr)
{
extent_dss_max_update(void *new_addr) {
void *max_cur;
spin_t spinner;
@@ -83,20 +81,21 @@ extent_dss_max_update(void *new_addr)
spin_adaptive(&spinner);
continue;
}
if (!atomic_cas_p(&dss_max, max_prev, max_cur))
if (!atomic_cas_p(&dss_max, max_prev, max_cur)) {
break;
}
}
/* Fixed new_addr can only be supported if it is at the edge of DSS. */
if (new_addr != NULL && max_cur != new_addr)
if (new_addr != NULL && max_cur != new_addr) {
return (NULL);
}
return (max_cur);
}
void *
extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit)
{
size_t alignment, bool *zero, bool *commit) {
extent_t *gap;
cassert(have_dss);
@@ -107,12 +106,14 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
* sbrk() uses a signed increment argument, so take care not to
* interpret a large allocation request as a negative increment.
*/
if ((intptr_t)size < 0)
if ((intptr_t)size < 0) {
return (NULL);
}
gap = extent_alloc(tsdn, arena);
if (gap == NULL)
if (gap == NULL) {
return (NULL);
}
if (!atomic_read_u(&dss_exhausted)) {
/*
@@ -126,8 +127,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
intptr_t incr;
max_cur = extent_dss_max_update(new_addr);
if (max_cur == NULL)
if (max_cur == NULL) {
goto label_oom;
}
/*
* Compute how much gap space (if any) is necessary to
@@ -145,8 +147,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
}
dss_next = (void *)((uintptr_t)ret + size);
if ((uintptr_t)ret < (uintptr_t)max_cur ||
(uintptr_t)dss_next < (uintptr_t)max_cur)
(uintptr_t)dss_next < (uintptr_t)max_cur) {
goto label_oom; /* Wrap-around. */
}
incr = gap_size + size;
/*
@@ -155,19 +158,22 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
* DSS while dss_max is greater than the current DSS
* max reported by sbrk(0).
*/
if (atomic_cas_p(&dss_max, max_cur, dss_next))
if (atomic_cas_p(&dss_max, max_cur, dss_next)) {
continue;
}
/* Try to allocate. */
dss_prev = extent_dss_sbrk(incr);
if (dss_prev == max_cur) {
/* Success. */
if (gap_size != 0)
if (gap_size != 0) {
extent_dalloc_gap(tsdn, arena, gap);
else
} else {
extent_dalloc(tsdn, arena, gap);
if (!*commit)
}
if (!*commit) {
*commit = pages_decommit(ret, size);
}
if (*zero && *commit) {
extent_hooks_t *extent_hooks =
EXTENT_HOOKS_INITIALIZER;
@@ -177,8 +183,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
size, 0, true, false, true, false);
if (extent_purge_forced_wrapper(tsdn,
arena, &extent_hooks, &extent, 0,
size))
size)) {
memset(ret, 0, size);
}
}
return (ret);
}
@@ -204,30 +211,28 @@ label_oom:
}
static bool
extent_in_dss_helper(void *addr, void *max)
{
extent_in_dss_helper(void *addr, void *max) {
return ((uintptr_t)addr >= (uintptr_t)dss_base && (uintptr_t)addr <
(uintptr_t)max);
}
bool
extent_in_dss(void *addr)
{
extent_in_dss(void *addr) {
cassert(have_dss);
return (extent_in_dss_helper(addr, atomic_read_p(&dss_max)));
}
bool
extent_dss_mergeable(void *addr_a, void *addr_b)
{
extent_dss_mergeable(void *addr_a, void *addr_b) {
void *max;
cassert(have_dss);
if ((uintptr_t)addr_a < (uintptr_t)dss_base && (uintptr_t)addr_b <
(uintptr_t)dss_base)
(uintptr_t)dss_base) {
return (true);
}
max = atomic_read_p(&dss_max);
return (extent_in_dss_helper(addr_a, max) ==
@@ -235,8 +240,7 @@ extent_dss_mergeable(void *addr_a, void *addr_b)
}
void
extent_dss_boot(void)
{
extent_dss_boot(void) {
cassert(have_dss);
dss_base = extent_dss_sbrk(0);

22
src/extent_mmap.c
View File

@@ -4,21 +4,23 @@
/******************************************************************************/
static void *
extent_alloc_mmap_slow(size_t size, size_t alignment, bool *zero, bool *commit)
{
extent_alloc_mmap_slow(size_t size, size_t alignment, bool *zero,
bool *commit) {
void *ret;
size_t alloc_size;
alloc_size = size + alignment - PAGE;
/* Beware size_t wrap-around. */
if (alloc_size < size)
if (alloc_size < size) {
return (NULL);
}
do {
void *pages;
size_t leadsize;
pages = pages_map(NULL, alloc_size, commit);
if (pages == NULL)
if (pages == NULL) {
return (NULL);
}
leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
(uintptr_t)pages;
ret = pages_trim(pages, alloc_size, leadsize, size, commit);
@@ -31,8 +33,7 @@ extent_alloc_mmap_slow(size_t size, size_t alignment, bool *zero, bool *commit)
void *
extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
bool *commit)
{
bool *commit) {
void *ret;
size_t offset;
@@ -52,8 +53,9 @@ extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
assert(alignment != 0);
ret = pages_map(new_addr, size, commit);
if (ret == NULL || ret == new_addr)
if (ret == NULL || ret == new_addr) {
return (ret);
}
assert(new_addr == NULL);
offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
if (offset != 0) {
@@ -67,9 +69,9 @@ extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
}
bool
extent_dalloc_mmap(void *addr, size_t size)
{
if (config_munmap)
extent_dalloc_mmap(void *addr, size_t size) {
if (config_munmap) {
pages_unmap(addr, size);
}
return (!config_munmap);
}

497
src/jemalloc.c
View File

File diff suppressed because it is too large Load Diff

33
src/jemalloc_cpp.cpp
View File

@@ -33,8 +33,7 @@ void operator delete[](void *ptr, std::size_t size) noexcept;
template <bool IsNoExcept>
JEMALLOC_INLINE
void *
newImpl(std::size_t size) noexcept(IsNoExcept)
{
newImpl(std::size_t size) noexcept(IsNoExcept) {
void *ptr = je_malloc(size);
if (likely(ptr != nullptr))
return (ptr);
@@ -67,65 +66,55 @@ newImpl(std::size_t size) noexcept(IsNoExcept)
}
void *
operator new(std::size_t size)
{
operator new(std::size_t size) {
return (newImpl<false>(size));
}
void *
operator new[](std::size_t size)
{
operator new[](std::size_t size) {
return (newImpl<false>(size));
}
void *
operator new(std::size_t size, const std::nothrow_t &) noexcept
{
operator new(std::size_t size, const std::nothrow_t &) noexcept {
return (newImpl<true>(size));
}
void *
operator new[](std::size_t size, const std::nothrow_t &) noexcept
{
operator new[](std::size_t size, const std::nothrow_t &) noexcept {
return (newImpl<true>(size));
}
void
operator delete(void *ptr) noexcept
{
operator delete(void *ptr) noexcept {
je_free(ptr);
}
void
operator delete[](void *ptr) noexcept
{
operator delete[](void *ptr) noexcept {
je_free(ptr);
}
void
operator delete(void *ptr, const std::nothrow_t &) noexcept
{
operator delete(void *ptr, const std::nothrow_t &) noexcept {
je_free(ptr);
}
void operator delete[](void *ptr, const std::nothrow_t &) noexcept
{
void operator delete[](void *ptr, const std::nothrow_t &) noexcept {
je_free(ptr);
}
#if __cpp_sized_deallocation >= 201309
void
operator delete(void *ptr, std::size_t size) noexcept
{
operator delete(void *ptr, std::size_t size) noexcept {
if (unlikely(ptr == nullptr)) {
return;
}
je_sdallocx(ptr, size, /*flags=*/0);
}
void operator delete[](void *ptr, std::size_t size) noexcept
{
void operator delete[](void *ptr, std::size_t size) noexcept {
if (unlikely(ptr == nullptr)) {
return;
}

87
src/large.c
View File

@@ -4,8 +4,7 @@
/******************************************************************************/
void *
large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
{
large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) {
assert(usize == s2u(usize));
return (large_palloc(tsdn, arena, usize, CACHELINE, zero));
@@ -13,8 +12,7 @@ large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
void *
large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero)
{
bool zero) {
size_t ausize;
extent_t *extent;
bool is_zeroed;
@@ -23,27 +21,31 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
assert(!tsdn_null(tsdn) || arena != NULL);
ausize = sa2u(usize, alignment);
if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS))
if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS)) {
return (NULL);
}
/*
* Copy zero into is_zeroed and pass the copy to extent_alloc(), so that
* it is possible to make correct junk/zero fill decisions below.
*/
is_zeroed = zero;
if (likely(!tsdn_null(tsdn)))
if (likely(!tsdn_null(tsdn))) {
arena = arena_choose(tsdn_tsd(tsdn), arena);
}
if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL)
arena, usize, alignment, &is_zeroed)) == NULL) {
return (NULL);
}
/* Insert extent into large. */
malloc_mutex_lock(tsdn, &arena->large_mtx);
ql_elm_new(extent, ql_link);
ql_tail_insert(&arena->large, extent, ql_link);
malloc_mutex_unlock(tsdn, &arena->large_mtx);
if (config_prof && arena_prof_accum(tsdn, arena, usize))
if (config_prof && arena_prof_accum(tsdn, arena, usize)) {
prof_idump(tsdn);
}
if (zero || (config_fill && unlikely(opt_zero))) {
if (!is_zeroed) {
@@ -64,8 +66,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
#define large_dalloc_junk JEMALLOC_N(n_large_dalloc_junk)
#endif
void
large_dalloc_junk(void *ptr, size_t usize)
{
large_dalloc_junk(void *ptr, size_t usize) {
memset(ptr, JEMALLOC_FREE_JUNK, usize);
}
#ifdef JEMALLOC_JET
@@ -79,15 +80,15 @@ large_dalloc_junk_t *large_dalloc_junk = JEMALLOC_N(n_large_dalloc_junk);
#define large_dalloc_maybe_junk JEMALLOC_N(n_large_dalloc_maybe_junk)
#endif
void
large_dalloc_maybe_junk(void *ptr, size_t usize)
{
large_dalloc_maybe_junk(void *ptr, size_t usize) {
if (config_fill && have_dss && unlikely(opt_junk_free)) {
/*
* Only bother junk filling if the extent isn't about to be
* unmapped.
*/
if (!config_munmap || (have_dss && extent_in_dss(ptr)))
if (!config_munmap || (have_dss && extent_in_dss(ptr))) {
large_dalloc_junk(ptr, usize);
}
}
}
#ifdef JEMALLOC_JET
@@ -98,8 +99,7 @@ large_dalloc_maybe_junk_t *large_dalloc_maybe_junk =
#endif
static bool
large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
{
large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent);
extent_hooks_t *extent_hooks = extent_hooks_get(arena);
@@ -107,16 +107,18 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
assert(oldusize > usize);
if (extent_hooks->split == NULL)
if (extent_hooks->split == NULL) {
return (true);
}
/* Split excess pages. */
if (diff != 0) {
extent_t *trail = extent_split_wrapper(tsdn, arena,
&extent_hooks, extent, usize + large_pad, usize, diff,
diff);
if (trail == NULL)
if (trail == NULL) {
return (true);
}
if (config_fill && unlikely(opt_junk_free)) {
large_dalloc_maybe_junk(extent_addr_get(trail),
@@ -133,8 +135,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
static bool
large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool zero)
{
bool zero) {
arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent);
bool is_zeroed_trail = false;
@@ -142,8 +143,9 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
size_t trailsize = usize - extent_usize_get(extent);
extent_t *trail;
if (extent_hooks->merge == NULL)
if (extent_hooks->merge == NULL) {
return (true);
}
if ((trail = arena_extent_cache_alloc(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail)) ==
@@ -151,8 +153,9 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool commit = true;
if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, 0, CACHELINE,
&is_zeroed_trail, &commit, false)) == NULL)
&is_zeroed_trail, &commit, false)) == NULL) {
return (true);
}
}
if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {
@@ -193,8 +196,7 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool
large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero)
{
size_t usize_max, bool zero) {
assert(s2u(extent_usize_get(extent)) == extent_usize_get(extent));
/* The following should have been caught by callers. */
assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS);
@@ -241,17 +243,16 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
static void *
large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero)
{
if (alignment <= CACHELINE)
size_t alignment, bool zero) {
if (alignment <= CACHELINE) {
return (large_malloc(tsdn, arena, usize, zero));
}
return (large_palloc(tsdn, arena, usize, alignment, zero));
}
void *
large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
size_t alignment, bool zero, tcache_t *tcache)
{
size_t alignment, bool zero, tcache_t *tcache) {
void *ret;
size_t copysize;
@@ -262,8 +263,9 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
LARGE_MINCLASS);
/* Try to avoid moving the allocation. */
if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero))
if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) {
return (extent_addr_get(extent));
}
/*
* usize and old size are different enough that we need to use a
@@ -271,8 +273,9 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
* space and copying.
*/
ret = large_ralloc_move_helper(tsdn, arena, usize, alignment, zero);
if (ret == NULL)
if (ret == NULL) {
return (NULL);
}
copysize = (usize < extent_usize_get(extent)) ? usize :
extent_usize_get(extent);
@@ -288,8 +291,7 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
* independent of these considerations.
*/
static void
large_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked)
{
large_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked) {
arena_t *arena;
arena = extent_arena_get(extent);
@@ -302,42 +304,37 @@ large_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked)
}
arena_extent_dalloc_large(tsdn, arena, extent, junked_locked);
if (!junked_locked)
if (!junked_locked) {
arena_decay_tick(tsdn, arena);
}
}
void
large_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent)
{
large_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent) {
large_dalloc_impl(tsdn, extent, true);
}
void
large_dalloc(tsdn_t *tsdn, extent_t *extent)
{
large_dalloc(tsdn_t *tsdn, extent_t *extent) {
large_dalloc_impl(tsdn, extent, false);
}
size_t
large_salloc(tsdn_t *tsdn, const extent_t *extent)
{
large_salloc(tsdn_t *tsdn, const extent_t *extent) {
return (extent_usize_get(extent));
}
prof_tctx_t *
large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent)
{
large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent) {
return (extent_prof_tctx_get(extent));
}
void
large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx)
{
large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx) {
extent_prof_tctx_set(extent, tctx);
}
void
large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent)
{
large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) {
large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
}

40
src/mutex.c
View File

@@ -35,8 +35,7 @@ static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
void *(*)(void *), void *__restrict);
static void
pthread_create_once(void)
{
pthread_create_once(void) {
pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
if (pthread_create_fptr == NULL) {
malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
@@ -50,8 +49,7 @@ pthread_create_once(void)
JEMALLOC_EXPORT int
pthread_create(pthread_t *__restrict thread,
const pthread_attr_t *__restrict attr, void *(*start_routine)(void *),
void *__restrict arg)
{
void *__restrict arg) {
static pthread_once_t once_control = PTHREAD_ONCE_INIT;
pthread_once(&once_control, pthread_create_once);
@@ -68,15 +66,16 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
#endif
bool
malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank)
{
malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
witness_rank_t rank) {
#ifdef _WIN32
# if _WIN32_WINNT >= 0x0600
InitializeSRWLock(&mutex->lock);
# else
if (!InitializeCriticalSectionAndSpinCount(&mutex->lock,
_CRT_SPINCOUNT))
_CRT_SPINCOUNT)) {
return (true);
}
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
mutex->lock = OS_UNFAIR_LOCK_INIT;
@@ -88,14 +87,16 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank)
postponed_mutexes = mutex;
} else {
if (_pthread_mutex_init_calloc_cb(&mutex->lock,
bootstrap_calloc) != 0)
bootstrap_calloc) != 0) {
return (true);
}
}
#else
pthread_mutexattr_t attr;
if (pthread_mutexattr_init(&attr) != 0)
if (pthread_mutexattr_init(&attr) != 0) {
return (true);
}
pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE);
if (pthread_mutex_init(&mutex->lock, &attr) != 0) {
pthread_mutexattr_destroy(&attr);
@@ -103,26 +104,24 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank)
}
pthread_mutexattr_destroy(&attr);
#endif
if (config_debug)
if (config_debug) {
witness_init(&mutex->witness, name, rank, NULL, NULL);
}
return (false);
}
void
malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex)
{
malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex) {
malloc_mutex_lock(tsdn, mutex);
}
void
malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex)
{
malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex) {
malloc_mutex_unlock(tsdn, mutex);
}
void
malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex)
{
malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex) {
#ifdef JEMALLOC_MUTEX_INIT_CB
malloc_mutex_unlock(tsdn, mutex);
#else
@@ -130,21 +129,22 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex)
mutex->witness.rank)) {
malloc_printf("<jemalloc>: Error re-initializing mutex in "
"child\n");
if (opt_abort)
if (opt_abort) {
abort();
}
}
#endif
}
bool
malloc_mutex_boot(void)
{
malloc_mutex_boot(void) {
#ifdef JEMALLOC_MUTEX_INIT_CB
postpone_init = false;
while (postponed_mutexes != NULL) {
if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock,
bootstrap_calloc) != 0)
bootstrap_calloc) != 0) {
return (true);
}
postponed_mutexes = postponed_mutexes->postponed_next;
}
#endif

57
src/nstime.c
View File

@@ -3,66 +3,56 @@
#define BILLION UINT64_C(1000000000)
void
nstime_init(nstime_t *time, uint64_t ns)
{
nstime_init(nstime_t *time, uint64_t ns) {
time->ns = ns;
}
void
nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec)
{
nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec) {
time->ns = sec * BILLION + nsec;
}
uint64_t
nstime_ns(const nstime_t *time)
{
nstime_ns(const nstime_t *time) {
return (time->ns);
}
uint64_t
nstime_sec(const nstime_t *time)
{
nstime_sec(const nstime_t *time) {
return (time->ns / BILLION);
}
uint64_t
nstime_nsec(const nstime_t *time)
{
nstime_nsec(const nstime_t *time) {
return (time->ns % BILLION);
}
void
nstime_copy(nstime_t *time, const nstime_t *source)
{
nstime_copy(nstime_t *time, const nstime_t *source) {
*time = *source;
}
int
nstime_compare(const nstime_t *a, const nstime_t *b)
{
nstime_compare(const nstime_t *a, const nstime_t *b) {
return ((a->ns > b->ns) - (a->ns < b->ns));
}
void
nstime_add(nstime_t *time, const nstime_t *addend)
{
nstime_add(nstime_t *time, const nstime_t *addend) {
assert(UINT64_MAX - time->ns >= addend->ns);
time->ns += addend->ns;
}
void
nstime_subtract(nstime_t *time, const nstime_t *subtrahend)
{
nstime_subtract(nstime_t *time, const nstime_t *subtrahend) {
assert(nstime_compare(time, subtrahend) >= 0);
time->ns -= subtrahend->ns;
}
void
nstime_imultiply(nstime_t *time, uint64_t multiplier)
{
nstime_imultiply(nstime_t *time, uint64_t multiplier) {
assert((((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) <<
2))) == 0) || ((time->ns * multiplier) / multiplier == time->ns));
@@ -70,16 +60,14 @@ nstime_imultiply(nstime_t *time, uint64_t multiplier)
}
void
nstime_idivide(nstime_t *time, uint64_t divisor)
{
nstime_idivide(nstime_t *time, uint64_t divisor) {
assert(divisor != 0);
time->ns /= divisor;
}
uint64_t
nstime_divide(const nstime_t *time, const nstime_t *divisor)
{
nstime_divide(const nstime_t *time, const nstime_t *divisor) {
assert(divisor->ns != 0);
return (time->ns / divisor->ns);
@@ -88,8 +76,7 @@ nstime_divide(const nstime_t *time, const nstime_t *divisor)
#ifdef _WIN32
# define NSTIME_MONOTONIC true
static void
nstime_get(nstime_t *time)
{
nstime_get(nstime_t *time) {
FILETIME ft;
uint64_t ticks_100ns;
@@ -101,8 +88,7 @@ nstime_get(nstime_t *time)
#elif JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
# define NSTIME_MONOTONIC true
static void
nstime_get(nstime_t *time)
{
nstime_get(nstime_t *time) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);
@@ -111,8 +97,7 @@ nstime_get(nstime_t *time)
#elif JEMALLOC_HAVE_CLOCK_MONOTONIC
# define NSTIME_MONOTONIC true
static void
nstime_get(nstime_t *time)
{
nstime_get(nstime_t *time) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -121,15 +106,13 @@ nstime_get(nstime_t *time)
#elif JEMALLOC_HAVE_MACH_ABSOLUTE_TIME
# define NSTIME_MONOTONIC true
static void
nstime_get(nstime_t *time)
{
nstime_get(nstime_t *time) {
nstime_init(time, mach_absolute_time());
}
#else
# define NSTIME_MONOTONIC false
static void
nstime_get(nstime_t *time)
{
nstime_get(nstime_t *time) {
struct timeval tv;
gettimeofday(&tv, NULL);
@@ -142,8 +125,7 @@ nstime_get(nstime_t *time)
#define nstime_monotonic JEMALLOC_N(n_nstime_monotonic)
#endif
bool
nstime_monotonic(void)
{
nstime_monotonic(void) {
return (NSTIME_MONOTONIC);
#undef NSTIME_MONOTONIC
}
@@ -158,8 +140,7 @@ nstime_monotonic_t *nstime_monotonic = JEMALLOC_N(n_nstime_monotonic);
#define nstime_update JEMALLOC_N(n_nstime_update)
#endif
bool
nstime_update(nstime_t *time)
{
nstime_update(nstime_t *time) {
nstime_t old_time;
nstime_copy(&old_time, time);

85
src/pages.c
View File

@@ -18,14 +18,14 @@ static bool os_overcommits;
/******************************************************************************/
void *
pages_map(void *addr, size_t size, bool *commit)
{
pages_map(void *addr, size_t size, bool *commit) {
void *ret;
assert(size != 0);
if (os_overcommits)
if (os_overcommits) {
*commit = true;
}
#ifdef _WIN32
/*
@@ -46,9 +46,9 @@ pages_map(void *addr, size_t size, bool *commit)
}
assert(ret != NULL);
if (ret == MAP_FAILED)
if (ret == MAP_FAILED) {
ret = NULL;
else if (addr != NULL && ret != addr) {
} else if (addr != NULL && ret != addr) {
/*
* We succeeded in mapping memory, but not in the right place.
*/
@@ -62,8 +62,7 @@ pages_map(void *addr, size_t size, bool *commit)
}
void
pages_unmap(void *addr, size_t size)
{
pages_unmap(void *addr, size_t size) {
#ifdef _WIN32
if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
#else
@@ -80,15 +79,15 @@ pages_unmap(void *addr, size_t size)
"munmap"
#endif
"(): %s\n", buf);
if (opt_abort)
if (opt_abort) {
abort();
}
}
}
void *
pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
bool *commit)
{
bool *commit) {
void *ret = (void *)((uintptr_t)addr + leadsize);
assert(alloc_size >= leadsize + size);
@@ -98,30 +97,34 @@ pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
pages_unmap(addr, alloc_size);
new_addr = pages_map(ret, size, commit);
if (new_addr == ret)
if (new_addr == ret) {
return (ret);
if (new_addr)
}
if (new_addr) {
pages_unmap(new_addr, size);
}
return (NULL);
}
#else
{
size_t trailsize = alloc_size - leadsize - size;
if (leadsize != 0)
if (leadsize != 0) {
pages_unmap(addr, leadsize);
if (trailsize != 0)
}
if (trailsize != 0) {
pages_unmap((void *)((uintptr_t)ret + size), trailsize);
}
return (ret);
}
#endif
}
static bool
pages_commit_impl(void *addr, size_t size, bool commit)
{
if (os_overcommits)
pages_commit_impl(void *addr, size_t size, bool commit) {
if (os_overcommits) {
return (true);
}
#ifdef _WIN32
return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
@@ -131,8 +134,9 @@ pages_commit_impl(void *addr, size_t size, bool commit)
int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,
-1, 0);
if (result == MAP_FAILED)
if (result == MAP_FAILED) {
return (true);
}
if (result != addr) {
/*
* We succeeded in mapping memory, but not in the right
@@ -147,22 +151,20 @@ pages_commit_impl(void *addr, size_t size, bool commit)
}
bool
pages_commit(void *addr, size_t size)
{
pages_commit(void *addr, size_t size) {
return (pages_commit_impl(addr, size, true));
}
bool
pages_decommit(void *addr, size_t size)
{
pages_decommit(void *addr, size_t size) {
return (pages_commit_impl(addr, size, false));
}
bool
pages_purge_lazy(void *addr, size_t size)
{
if (!pages_can_purge_lazy)
pages_purge_lazy(void *addr, size_t size) {
if (!pages_can_purge_lazy) {
return (true);
}
#ifdef _WIN32
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
@@ -175,10 +177,10 @@ pages_purge_lazy(void *addr, size_t size)
}
bool
pages_purge_forced(void *addr, size_t size)
{
if (!pages_can_purge_forced)
pages_purge_forced(void *addr, size_t size) {
if (!pages_can_purge_forced) {
return (true);
}
#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
return (madvise(addr, size, MADV_DONTNEED) != 0);
@@ -188,8 +190,7 @@ pages_purge_forced(void *addr, size_t size)
}
bool
pages_huge(void *addr, size_t size)
{
pages_huge(void *addr, size_t size) {
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
@@ -201,8 +202,7 @@ pages_huge(void *addr, size_t size)
}
bool
pages_nohuge(void *addr, size_t size)
{
pages_nohuge(void *addr, size_t size) {
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
@@ -215,14 +215,14 @@ pages_nohuge(void *addr, size_t size)
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
static bool
os_overcommits_sysctl(void)
{
os_overcommits_sysctl(void) {
int vm_overcommit;
size_t sz;
sz = sizeof(vm_overcommit);
if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0)
if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) {
return (false); /* Error. */
}
return ((vm_overcommit & 0x3) == 0);
}
@@ -235,8 +235,7 @@ os_overcommits_sysctl(void)
* wrappers.
*/
static bool
os_overcommits_proc(void)
{
os_overcommits_proc(void) {
int fd;
char buf[1];
ssize_t nread;
@@ -246,8 +245,9 @@ os_overcommits_proc(void)
#else
fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
#endif
if (fd == -1)
if (fd == -1) {
return (false); /* Error. */
}
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
@@ -261,8 +261,9 @@ os_overcommits_proc(void)
close(fd);
#endif
if (nread < 1)
if (nread < 1) {
return (false); /* Error. */
}
/*
* /proc/sys/vm/overcommit_memory meanings:
* 0: Heuristic overcommit.
@@ -274,8 +275,7 @@ os_overcommits_proc(void)
#endif
void
pages_boot(void)
{
pages_boot(void) {
#ifndef _WIN32
mmap_flags = MAP_PRIVATE | MAP_ANON;
#endif
@@ -285,8 +285,9 @@ pages_boot(void)
#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY)
os_overcommits = os_overcommits_proc();
# ifdef MAP_NORESERVE
if (os_overcommits)
if (os_overcommits) {
mmap_flags |= MAP_NORESERVE;
}
# endif
#else
os_overcommits = false;

568
src/prof.c
View File

File diff suppressed because it is too large Load Diff

70
src/rtree.c
View File

@@ -2,8 +2,7 @@
#include "jemalloc/internal/jemalloc_internal.h"
static unsigned
hmin(unsigned ha, unsigned hb)
{
hmin(unsigned ha, unsigned hb) {
return (ha < hb ? ha : hb);
}
@@ -12,8 +11,7 @@ hmin(unsigned ha, unsigned hb)
* used.
*/
bool
rtree_new(rtree_t *rtree, unsigned bits)
{
rtree_new(rtree_t *rtree, unsigned bits) {
unsigned bits_in_leaf, height, i;
assert(RTREE_HEIGHT_MAX == ((ZU(1) << (LG_SIZEOF_PTR+3)) /
@@ -24,10 +22,12 @@ rtree_new(rtree_t *rtree, unsigned bits)
: (bits % RTREE_BITS_PER_LEVEL);
if (bits > bits_in_leaf) {
height = 1 + (bits - bits_in_leaf) / RTREE_BITS_PER_LEVEL;
if ((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf != bits)
if ((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf != bits) {
height++;
} else
}
} else {
height = 1;
}
assert((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf == bits);
rtree->height = height;
@@ -68,8 +68,7 @@ rtree_new(rtree_t *rtree, unsigned bits)
#define rtree_node_alloc JEMALLOC_N(rtree_node_alloc_impl)
#endif
static rtree_elm_t *
rtree_node_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms)
{
rtree_node_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
return ((rtree_elm_t *)base_alloc(tsdn, b0get(), nelms *
sizeof(rtree_elm_t), CACHELINE));
}
@@ -84,8 +83,7 @@ rtree_node_alloc_t *rtree_node_alloc = JEMALLOC_N(rtree_node_alloc_impl);
#define rtree_node_dalloc JEMALLOC_N(rtree_node_dalloc_impl)
#endif
UNUSED static void
rtree_node_dalloc(tsdn_t *tsdn, rtree_t *rtree, rtree_elm_t *node)
{
rtree_node_dalloc(tsdn_t *tsdn, rtree_t *rtree, rtree_elm_t *node) {
/* Nodes are never deleted during normal operation. */
not_reached();
}
@@ -98,8 +96,7 @@ rtree_node_dalloc_t *rtree_node_dalloc = JEMALLOC_N(rtree_node_dalloc_impl);
#ifdef JEMALLOC_JET
static void
rtree_delete_subtree(tsdn_t *tsdn, rtree_t *rtree, rtree_elm_t *node,
unsigned level)
{
unsigned level) {
if (level + 1 < rtree->height) {
size_t nchildren, i;
@@ -116,22 +113,21 @@ rtree_delete_subtree(tsdn_t *tsdn, rtree_t *rtree, rtree_elm_t *node,
}
void
rtree_delete(tsdn_t *tsdn, rtree_t *rtree)
{
rtree_delete(tsdn_t *tsdn, rtree_t *rtree) {
unsigned i;
for (i = 0; i < rtree->height; i++) {
rtree_elm_t *subtree = rtree->levels[i].subtree;
if (subtree != NULL)
if (subtree != NULL) {
rtree_delete_subtree(tsdn, rtree, subtree, i);
}
}
}
#endif
static rtree_elm_t *
rtree_node_init(tsdn_t *tsdn, rtree_t *rtree, unsigned level,
rtree_elm_t **elmp)
{
rtree_elm_t **elmp) {
rtree_elm_t *node;
malloc_mutex_lock(tsdn, &rtree->init_lock);
@@ -151,23 +147,20 @@ rtree_node_init(tsdn_t *tsdn, rtree_t *rtree, unsigned level,
}
rtree_elm_t *
rtree_subtree_read_hard(tsdn_t *tsdn, rtree_t *rtree, unsigned level)
{
rtree_subtree_read_hard(tsdn_t *tsdn, rtree_t *rtree, unsigned level) {
return (rtree_node_init(tsdn, rtree, level,
&rtree->levels[level].subtree));
}
rtree_elm_t *
rtree_child_read_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_elm_t *elm,
unsigned level)
{
unsigned level) {
return (rtree_node_init(tsdn, rtree, level+1, &elm->child));
}
static int
rtree_elm_witness_comp(const witness_t *a, void *oa, const witness_t *b,
void *ob)
{
void *ob) {
uintptr_t ka = (uintptr_t)oa;
uintptr_t kb = (uintptr_t)ob;
@@ -178,8 +171,7 @@ rtree_elm_witness_comp(const witness_t *a, void *oa, const witness_t *b,
}
static witness_t *
rtree_elm_witness_alloc(tsd_t *tsd, uintptr_t key, const rtree_elm_t *elm)
{
rtree_elm_witness_alloc(tsd_t *tsd, uintptr_t key, const rtree_elm_t *elm) {
witness_t *witness;
size_t i;
rtree_elm_witness_tsd_t *witnesses = tsd_rtree_elm_witnessesp_get(tsd);
@@ -204,8 +196,7 @@ rtree_elm_witness_alloc(tsd_t *tsd, uintptr_t key, const rtree_elm_t *elm)
}
static witness_t *
rtree_elm_witness_find(tsd_t *tsd, const rtree_elm_t *elm)
{
rtree_elm_witness_find(tsd_t *tsd, const rtree_elm_t *elm) {
size_t i;
rtree_elm_witness_tsd_t *witnesses = tsd_rtree_elm_witnessesp_get(tsd);
@@ -213,15 +204,16 @@ rtree_elm_witness_find(tsd_t *tsd, const rtree_elm_t *elm)
i++) {
rtree_elm_witness_t *rew = &witnesses->witnesses[i];
if (rew->elm == elm)
if (rew->elm == elm) {
return (&rew->witness);
}
}
not_reached();
}
static void
rtree_elm_witness_dalloc(tsd_t *tsd, witness_t *witness, const rtree_elm_t *elm)
{
rtree_elm_witness_dalloc(tsd_t *tsd, witness_t *witness,
const rtree_elm_t *elm) {
size_t i;
rtree_elm_witness_tsd_t *witnesses = tsd_rtree_elm_witnessesp_get(tsd);
@@ -242,12 +234,12 @@ rtree_elm_witness_dalloc(tsd_t *tsd, witness_t *witness, const rtree_elm_t *elm)
void
rtree_elm_witness_acquire(tsdn_t *tsdn, const rtree_t *rtree, uintptr_t key,
const rtree_elm_t *elm)
{
const rtree_elm_t *elm) {
witness_t *witness;
if (tsdn_null(tsdn))
if (tsdn_null(tsdn)) {
return;
}
witness = rtree_elm_witness_alloc(tsdn_tsd(tsdn), key, elm);
witness_lock(tsdn, witness);
@@ -255,12 +247,12 @@ rtree_elm_witness_acquire(tsdn_t *tsdn, const rtree_t *rtree, uintptr_t key,
void
rtree_elm_witness_access(tsdn_t *tsdn, const rtree_t *rtree,
const rtree_elm_t *elm)
{
const rtree_elm_t *elm) {
witness_t *witness;
if (tsdn_null(tsdn))
if (tsdn_null(tsdn)) {
return;
}
witness = rtree_elm_witness_find(tsdn_tsd(tsdn), elm);
witness_assert_owner(tsdn, witness);
@@ -268,12 +260,12 @@ rtree_elm_witness_access(tsdn_t *tsdn, const rtree_t *rtree,
void
rtree_elm_witness_release(tsdn_t *tsdn, const rtree_t *rtree,
const rtree_elm_t *elm)
{
const rtree_elm_t *elm) {
witness_t *witness;
if (tsdn_null(tsdn))
if (tsdn_null(tsdn)) {
return;
}
witness = rtree_elm_witness_find(tsdn_tsd(tsdn), elm);
witness_unlock(tsdn, witness);

45
src/stats.c
View File

@@ -34,8 +34,7 @@ bool opt_stats_print = false;
static void
stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, bool large, unsigned i)
{
bool json, bool large, unsigned i) {
size_t page;
bool in_gap, in_gap_prev;
unsigned nbins, j;
@@ -144,8 +143,9 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
} else if (milli < 1000) {
malloc_snprintf(util, sizeof(util), "0.%zu",
milli);
} else
} else {
malloc_snprintf(util, sizeof(util), "1");
}
if (config_tcache) {
malloc_cprintf(write_cb, cbopaque,
@@ -183,8 +183,7 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
static void
stats_arena_lextents_print(void (*write_cb)(void *, const char *),
void *cbopaque, bool json, unsigned i)
{
void *cbopaque, bool json, unsigned i) {
unsigned nbins, nlextents, j;
bool in_gap, in_gap_prev;
@@ -248,8 +247,7 @@ stats_arena_lextents_print(void (*write_cb)(void *, const char *),
static void
stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, unsigned i, bool bins, bool large)
{
bool json, unsigned i, bool bins, bool large) {
unsigned nthreads;
const char *dss;
ssize_t decay_time;
@@ -290,8 +288,9 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
if (decay_time >= 0) {
malloc_cprintf(write_cb, cbopaque, "decay time: %zd\n",
decay_time);
} else
} else {
malloc_cprintf(write_cb, cbopaque, "decay time: N/A\n");
}
}
CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t);
@@ -445,16 +444,17 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
"resident: %12zu\n", resident);
}
if (bins)
if (bins) {
stats_arena_bins_print(write_cb, cbopaque, json, large, i);
if (large)
}
if (large) {
stats_arena_lextents_print(write_cb, cbopaque, json, i);
}
}
static void
stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, bool more)
{
bool json, bool more) {
const char *cpv;
bool bv;
unsigned uv;
@@ -473,8 +473,9 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\"version\": \"%s\",\n", cpv);
} else
} else {
malloc_cprintf(write_cb, cbopaque, "Version: %s\n", cpv);
}
/* config. */
#define CONFIG_WRITE_BOOL_JSON(n, c) \
@@ -655,8 +656,9 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"narenas\": %u,\n", uv);
} else
} else {
malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv);
}
CTL_GET("arenas.decay_time", &ssv, ssize_t);
if (json) {
@@ -672,15 +674,17 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"quantum\": %zu,\n", sv);
} else
} else {
malloc_cprintf(write_cb, cbopaque, "Quantum size: %zu\n", sv);
}
CTL_GET("arenas.page", &sv, size_t);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"page\": %zu,\n", sv);
} else
} else {
malloc_cprintf(write_cb, cbopaque, "Page size: %zu\n", sv);
}
if (je_mallctl("arenas.tcache_max", (void *)&sv, &ssz, NULL, 0) == 0) {
if (json) {
@@ -787,8 +791,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
static void
stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, bool merged, bool destroyed, bool unmerged, bool bins,
bool large)
{
bool large) {
size_t allocated, active, metadata, resident, mapped, retained;
CTL_GET("stats.allocated", &allocated, size_t);
@@ -846,8 +849,9 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
sz = sizeof(bool);
xmallctlbymib(mib, miblen, &initialized[i], &sz,
NULL, 0);
if (initialized[i])
if (initialized[i]) {
ninitialized++;
}
}
mib[1] = MALLCTL_ARENAS_DESTROYED;
sz = sizeof(bool);
@@ -934,8 +938,7 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
void
stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
const char *opts)
{
const char *opts) {
int err;
uint64_t epoch;
size_t u64sz;

120
src/tcache.c
View File

@@ -24,14 +24,12 @@ static tcaches_t *tcaches_avail;
/******************************************************************************/
size_t
tcache_salloc(tsdn_t *tsdn, const void *ptr)
{
tcache_salloc(tsdn_t *tsdn, const void *ptr) {
return (arena_salloc(tsdn, iealloc(tsdn, ptr), ptr));
}
void
tcache_event_hard(tsd_t *tsd, tcache_t *tcache)
{
tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
szind_t binind = tcache->next_gc_bin;
tcache_bin_t *tbin = &tcache->tbins[binind];
tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
@@ -52,33 +50,36 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache)
* Reduce fill count by 2X. Limit lg_fill_div such that the
* fill count is always at least 1.
*/
if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1)
if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1) {
tbin->lg_fill_div++;
}
} else if (tbin->low_water < 0) {
/*
* Increase fill count by 2X. Make sure lg_fill_div stays
* greater than 0.
*/
if (tbin->lg_fill_div > 1)
if (tbin->lg_fill_div > 1) {
tbin->lg_fill_div--;
}
}
tbin->low_water = tbin->ncached;
tcache->next_gc_bin++;
if (tcache->next_gc_bin == nhbins)
if (tcache->next_gc_bin == nhbins) {
tcache->next_gc_bin = 0;
}
}
void *
tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
tcache_bin_t *tbin, szind_t binind, bool *tcache_success)
{
tcache_bin_t *tbin, szind_t binind, bool *tcache_success) {
void *ret;
arena_tcache_fill_small(tsdn, arena, tbin, binind, config_prof ?
tcache->prof_accumbytes : 0);
if (config_prof)
if (config_prof) {
tcache->prof_accumbytes = 0;
}
ret = tcache_alloc_easy(tbin, tcache_success);
return (ret);
@@ -86,8 +87,7 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
void
tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
szind_t binind, unsigned rem)
{
szind_t binind, unsigned rem) {
arena_t *arena;
void *ptr;
unsigned i, nflush, ndeferred;
@@ -106,8 +106,9 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
if (config_prof && bin_arena == arena) {
if (arena_prof_accum(tsd_tsdn(tsd), arena,
tcache->prof_accumbytes))
tcache->prof_accumbytes)) {
prof_idump(tsd_tsdn(tsd));
}
tcache->prof_accumbytes = 0;
}
@@ -158,14 +159,14 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
sizeof(void *));
tbin->ncached = rem;
if ((int)tbin->ncached < tbin->low_water)
if ((int)tbin->ncached < tbin->low_water) {
tbin->low_water = tbin->ncached;
}
}
void
tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
unsigned rem, tcache_t *tcache)
{
unsigned rem, tcache_t *tcache) {
arena_t *arena;
void *ptr;
unsigned i, nflush, ndeferred;
@@ -182,8 +183,9 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
arena_t *locked_arena = extent_arena_get(extent);
UNUSED bool idump;
if (config_prof)
if (config_prof) {
idump = false;
}
malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->lock);
if ((config_prof || config_stats) && locked_arena == arena) {
if (config_prof) {
@@ -220,8 +222,9 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
}
}
malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->lock);
if (config_prof && idump)
if (config_prof && idump) {
prof_idump(tsd_tsdn(tsd));
}
arena_decay_ticks(tsd_tsdn(tsd), locked_arena, nflush -
ndeferred);
}
@@ -241,13 +244,13 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
sizeof(void *));
tbin->ncached = rem;
if ((int)tbin->ncached < tbin->low_water)
if ((int)tbin->ncached < tbin->low_water) {
tbin->low_water = tbin->ncached;
}
}
static void
tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
{
tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
if (config_stats) {
/* Link into list of extant tcaches. */
malloc_mutex_lock(tsdn, &arena->lock);
@@ -258,8 +261,7 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
}
static void
tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
{
tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
if (config_stats) {
/* Unlink from list of extant tcaches. */
malloc_mutex_lock(tsdn, &arena->lock);
@@ -282,31 +284,30 @@ tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
void
tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *oldarena,
arena_t *newarena)
{
arena_t *newarena) {
tcache_arena_dissociate(tsdn, tcache, oldarena);
tcache_arena_associate(tsdn, tcache, newarena);
}
tcache_t *
tcache_get_hard(tsd_t *tsd)
{
tcache_get_hard(tsd_t *tsd) {
arena_t *arena;
if (!tcache_enabled_get()) {
if (tsd_nominal(tsd))
if (tsd_nominal(tsd)) {
tcache_enabled_set(false); /* Memoize. */
}
return (NULL);
}
arena = arena_choose(tsd, NULL);
if (unlikely(arena == NULL))
if (unlikely(arena == NULL)) {
return (NULL);
}
return (tcache_create(tsd_tsdn(tsd), arena));
}
tcache_t *
tcache_create(tsdn_t *tsdn, arena_t *arena)
{
tcache_create(tsdn_t *tsdn, arena_t *arena) {
tcache_t *tcache;
size_t size, stack_offset;
unsigned i;
@@ -321,8 +322,9 @@ tcache_create(tsdn_t *tsdn, arena_t *arena)
tcache = ipallocztm(tsdn, size, CACHELINE, true, NULL, true,
arena_get(TSDN_NULL, 0, true));
if (tcache == NULL)
if (tcache == NULL) {
return (NULL);
}
tcache_arena_associate(tsdn, tcache, arena);
@@ -345,8 +347,7 @@ tcache_create(tsdn_t *tsdn, arena_t *arena)
}
static void
tcache_destroy(tsd_t *tsd, tcache_t *tcache)
{
tcache_destroy(tsd_t *tsd, tcache_t *tcache) {
arena_t *arena;
unsigned i;
@@ -372,20 +373,21 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
}
if (config_prof && tcache->prof_accumbytes > 0 &&
arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes))
arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes)) {
prof_idump(tsd_tsdn(tsd));
}
idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), tcache), tcache, NULL,
true, true);
}
void
tcache_cleanup(tsd_t *tsd)
{
tcache_cleanup(tsd_t *tsd) {
tcache_t *tcache;
if (!config_tcache)
if (!config_tcache) {
return;
}
if ((tcache = tsd_tcache_get(tsd)) != NULL) {
tcache_destroy(tsd, tcache);
@@ -394,8 +396,7 @@ tcache_cleanup(tsd_t *tsd)
}
void
tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
{
tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
unsigned i;
cassert(config_stats);
@@ -422,8 +423,7 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
}
bool
tcaches_create(tsd_t *tsd, unsigned *r_ind)
{
tcaches_create(tsd_t *tsd, unsigned *r_ind) {
arena_t *arena;
tcache_t *tcache;
tcaches_t *elm;
@@ -431,18 +431,22 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
if (tcaches == NULL) {
tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *)
* (MALLOCX_TCACHE_MAX+1), CACHELINE);
if (tcaches == NULL)
if (tcaches == NULL) {
return (true);
}
}
if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX)
if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
return (true);
}
arena = arena_ichoose(tsd, NULL);
if (unlikely(arena == NULL))
if (unlikely(arena == NULL)) {
return (true);
}
tcache = tcache_create(tsd_tsdn(tsd), arena);
if (tcache == NULL)
if (tcache == NULL) {
return (true);
}
if (tcaches_avail != NULL) {
elm = tcaches_avail;
@@ -460,23 +464,21 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
}
static void
tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm)
{
if (elm->tcache == NULL)
tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) {
if (elm->tcache == NULL) {
return;
}
tcache_destroy(tsd, elm->tcache);
elm->tcache = NULL;
}
void
tcaches_flush(tsd_t *tsd, unsigned ind)
{
tcaches_flush(tsd_t *tsd, unsigned ind) {
tcaches_elm_flush(tsd, &tcaches[ind]);
}
void
tcaches_destroy(tsd_t *tsd, unsigned ind)
{
tcaches_destroy(tsd_t *tsd, unsigned ind) {
tcaches_t *elm = &tcaches[ind];
tcaches_elm_flush(tsd, elm);
elm->next = tcaches_avail;
@@ -484,23 +486,25 @@ tcaches_destroy(tsd_t *tsd, unsigned ind)
}
bool
tcache_boot(tsdn_t *tsdn)
{
tcache_boot(tsdn_t *tsdn) {
unsigned i;
/* If necessary, clamp opt_lg_tcache_max. */
if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < SMALL_MAXCLASS)
if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) <
SMALL_MAXCLASS) {
tcache_maxclass = SMALL_MAXCLASS;
else
} else {
tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
}
nhbins = size2index(tcache_maxclass) + 1;
/* Initialize tcache_bin_info. */
tcache_bin_info = (tcache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins
* sizeof(tcache_bin_info_t), CACHELINE);
if (tcache_bin_info == NULL)
if (tcache_bin_info == NULL) {
return (true);
}
stack_nelms = 0;
for (i = 0; i < NBINS; i++) {
if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {

42
src/tsd.c
View File

@@ -12,20 +12,17 @@ malloc_tsd_data(, , tsd_t, TSD_INITIALIZER)
/******************************************************************************/
void *
malloc_tsd_malloc(size_t size)
{
malloc_tsd_malloc(size_t size) {
return (a0malloc(CACHELINE_CEILING(size)));
}
void
malloc_tsd_dalloc(void *wrapper)
{
malloc_tsd_dalloc(void *wrapper) {
a0dalloc(wrapper);
}
void
malloc_tsd_no_cleanup(void *arg)
{
malloc_tsd_no_cleanup(void *arg) {
not_reached();
}
@@ -34,21 +31,22 @@ malloc_tsd_no_cleanup(void *arg)
JEMALLOC_EXPORT
#endif
void
_malloc_thread_cleanup(void)
{
_malloc_thread_cleanup(void) {
bool pending[MALLOC_TSD_CLEANUPS_MAX], again;
unsigned i;
for (i = 0; i < ncleanups; i++)
for (i = 0; i < ncleanups; i++) {
pending[i] = true;
}
do {
again = false;
for (i = 0; i < ncleanups; i++) {
if (pending[i]) {
pending[i] = cleanups[i]();
if (pending[i])
if (pending[i]) {
again = true;
}
}
}
} while (again);
@@ -56,16 +54,14 @@ _malloc_thread_cleanup(void)
#endif
void
malloc_tsd_cleanup_register(bool (*f)(void))
{
malloc_tsd_cleanup_register(bool (*f)(void)) {
assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX);
cleanups[ncleanups] = f;
ncleanups++;
}
void
tsd_cleanup(void *arg)
{
tsd_cleanup(void *arg) {
tsd_t *tsd = (tsd_t *)arg;
switch (tsd->state) {
@@ -108,29 +104,27 @@ MALLOC_TSD
}
tsd_t *
malloc_tsd_boot0(void)
{
malloc_tsd_boot0(void) {
tsd_t *tsd;
ncleanups = 0;
if (tsd_boot0())
if (tsd_boot0()) {
return (NULL);
}
tsd = tsd_fetch();
*tsd_arenas_tdata_bypassp_get(tsd) = true;
return (tsd);
}
void
malloc_tsd_boot1(void)
{
malloc_tsd_boot1(void) {
tsd_boot1();
*tsd_arenas_tdata_bypassp_get(tsd_fetch()) = false;
}
#ifdef _WIN32
static BOOL WINAPI
_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
{
_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
switch (fdwReason) {
#ifdef JEMALLOC_LAZY_LOCK
case DLL_THREAD_ATTACH:
@@ -164,8 +158,7 @@ BOOL (WINAPI *const tls_callback)(HINSTANCE hinstDLL,
#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
!defined(_WIN32))
void *
tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block)
{
tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block) {
pthread_t self = pthread_self();
tsd_init_block_t *iter;
@@ -186,8 +179,7 @@ tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block)
}
void
tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block)
{
tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) {
malloc_mutex_lock(TSDN_NULL, &head->lock);
ql_remove(&head->blocks, block, link);
malloc_mutex_unlock(TSDN_NULL, &head->lock);

96
src/util.c
View File

@@ -46,8 +46,7 @@ static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
/* malloc_message() setup. */
static void
wrtmessage(void *cbopaque, const char *s)
{
wrtmessage(void *cbopaque, const char *s) {
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write)
/*
* Use syscall(2) rather than write(2) when possible in order to avoid
@@ -71,12 +70,12 @@ JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
* je_malloc_message(...) throughout the code.
*/
void
malloc_write(const char *s)
{
if (je_malloc_message != NULL)
malloc_write(const char *s) {
if (je_malloc_message != NULL) {
je_malloc_message(NULL, s);
else
} else {
wrtmessage(NULL, s);
}
}
/*
@@ -84,8 +83,7 @@ malloc_write(const char *s)
* provide a wrapper.
*/
int
buferror(int err, char *buf, size_t buflen)
{
buferror(int err, char *buf, size_t buflen) {
#ifdef _WIN32
FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0,
(LPSTR)buf, (DWORD)buflen, NULL);
@@ -103,8 +101,7 @@ buferror(int err, char *buf, size_t buflen)
}
uintmax_t
malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base)
{
malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) {
uintmax_t ret, digit;
unsigned b;
bool neg;
@@ -149,10 +146,12 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base)
switch (p[1]) {
case '0': case '1': case '2': case '3': case '4': case '5':
case '6': case '7':
if (b == 0)
if (b == 0) {
b = 8;
if (b == 8)
}
if (b == 8) {
p++;
}
break;
case 'X': case 'x':
switch (p[2]) {
@@ -162,10 +161,12 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base)
case 'F':
case 'a': case 'b': case 'c': case 'd': case 'e':
case 'f':
if (b == 0)
if (b == 0) {
b = 16;
if (b == 16)
}
if (b == 16) {
p += 2;
}
break;
default:
break;
@@ -177,8 +178,9 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base)
goto label_return;
}
}
if (b == 0)
if (b == 0) {
b = 10;
}
/* Convert. */
ret = 0;
@@ -196,8 +198,9 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base)
}
p++;
}
if (neg)
if (neg) {
ret = (uintmax_t)(-((intmax_t)ret));
}
if (p == ns) {
/* No conversion performed. */
@@ -211,15 +214,15 @@ label_return:
if (p == ns) {
/* No characters were converted. */
*endptr = (char *)nptr;
} else
} else {
*endptr = (char *)p;
}
}
return (ret);
}
static char *
u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p)
{
u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) {
unsigned i;
i = U2S_BUFSIZE - 1;
@@ -261,19 +264,21 @@ u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p)
}
static char *
d2s(intmax_t x, char sign, char *s, size_t *slen_p)
{
d2s(intmax_t x, char sign, char *s, size_t *slen_p) {
bool neg;
if ((neg = (x < 0)))
if ((neg = (x < 0))) {
x = -x;
}
s = u2s(x, 10, false, s, slen_p);
if (neg)
if (neg) {
sign = '-';
}
switch (sign) {
case '-':
if (!neg)
if (!neg) {
break;
}
/* Fall through. */
case ' ':
case '+':
@@ -287,8 +292,7 @@ d2s(intmax_t x, char sign, char *s, size_t *slen_p)
}
static char *
o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p)
{
o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) {
s = u2s(x, 8, false, s, slen_p);
if (alt_form && *s != '0') {
s--;
@@ -299,8 +303,7 @@ o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p)
}
static char *
x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p)
{
x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
s = u2s(x, 16, uppercase, s, slen_p);
if (alt_form) {
s -= 2;
@@ -311,14 +314,14 @@ x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p)
}
size_t
malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
{
malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
size_t i;
const char *f;
#define APPEND_C(c) do { \
if (i < size) \
if (i < size) { \
str[i] = (c); \
} \
i++; \
} while (0)
#define APPEND_S(s, slen) do { \
@@ -334,16 +337,18 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
(size_t)width - slen : 0); \
if (!left_justify && pad_len != 0) { \
size_t j; \
for (j = 0; j < pad_len; j++) \
for (j = 0; j < pad_len; j++) { \
APPEND_C(' '); \
} \
} \
/* Value. */ \
APPEND_S(s, slen); \
/* Right padding. */ \
if (left_justify && pad_len != 0) { \
size_t j; \
for (j = 0; j < pad_len; j++) \
for (j = 0; j < pad_len; j++) { \
APPEND_C(' '); \
} \
} \
} while (0)
#define GET_ARG_NUMERIC(val, len) do { \
@@ -454,10 +459,11 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
break;
}
/* Width/precision separator. */
if (*f == '.')
if (*f == '.') {
f++;
else
} else {
goto label_length;
}
/* Precision. */
switch (*f) {
case '*':
@@ -484,8 +490,9 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
if (*f == 'l') {
len = 'q';
f++;
} else
} else {
len = 'l';
}
break;
case 'q': case 'j': case 't': case 'z':
len = *f;
@@ -576,10 +583,11 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
}}
}
label_out:
if (i < size)
if (i < size) {
str[i] = '\0';
else
} else {
str[size - 1] = '\0';
}
#undef APPEND_C
#undef APPEND_S
@@ -590,8 +598,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
JEMALLOC_FORMAT_PRINTF(3, 4)
size_t
malloc_snprintf(char *str, size_t size, const char *format, ...)
{
malloc_snprintf(char *str, size_t size, const char *format, ...) {
size_t ret;
va_list ap;
@@ -604,8 +611,7 @@ malloc_snprintf(char *str, size_t size, const char *format, ...)
void
malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
const char *format, va_list ap)
{
const char *format, va_list ap) {
char buf[MALLOC_PRINTF_BUFSIZE];
if (write_cb == NULL) {
@@ -630,8 +636,7 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
JEMALLOC_FORMAT_PRINTF(3, 4)
void
malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
const char *format, ...)
{
const char *format, ...) {
va_list ap;
va_start(ap, format);
@@ -642,8 +647,7 @@ malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
/* Print to stderr in such a way as to avoid memory allocation. */
JEMALLOC_FORMAT_PRINTF(1, 2)
void
malloc_printf(const char *format, ...)
{
malloc_printf(const char *format, ...) {
va_list ap;
va_start(ap, format);

27
src/witness.c
View File

@@ -3,8 +3,7 @@
void
witness_init(witness_t *witness, const char *name, witness_rank_t rank,
witness_comp_t *comp, void *opaque)
{
witness_comp_t *comp, void *opaque) {
witness->name = name;
witness->rank = rank;
witness->comp = comp;
@@ -16,8 +15,7 @@ witness_init(witness_t *witness, const char *name, witness_rank_t rank,
#define witness_lock_error JEMALLOC_N(n_witness_lock_error)
#endif
void
witness_lock_error(const witness_list_t *witnesses, const witness_t *witness)
{
witness_lock_error(const witness_list_t *witnesses, const witness_t *witness) {
witness_t *w;
malloc_printf("<jemalloc>: Lock rank order reversal:");
@@ -38,8 +36,7 @@ witness_lock_error_t *witness_lock_error = JEMALLOC_N(n_witness_lock_error);
#define witness_owner_error JEMALLOC_N(n_witness_owner_error)
#endif
void
witness_owner_error(const witness_t *witness)
{
witness_owner_error(const witness_t *witness) {
malloc_printf("<jemalloc>: Should own %s(%u)\n", witness->name,
witness->rank);
abort();
@@ -55,8 +52,7 @@ witness_owner_error_t *witness_owner_error = JEMALLOC_N(n_witness_owner_error);
#define witness_not_owner_error JEMALLOC_N(n_witness_not_owner_error)
#endif
void
witness_not_owner_error(const witness_t *witness)
{
witness_not_owner_error(const witness_t *witness) {
malloc_printf("<jemalloc>: Should not own %s(%u)\n", witness->name,
witness->rank);
abort();
@@ -73,8 +69,7 @@ witness_not_owner_error_t *witness_not_owner_error =
#define witness_lockless_error JEMALLOC_N(n_witness_lockless_error)
#endif
void
witness_lockless_error(const witness_list_t *witnesses)
{
witness_lockless_error(const witness_list_t *witnesses) {
witness_t *w;
malloc_printf("<jemalloc>: Should not own any locks:");
@@ -92,28 +87,24 @@ witness_lockless_error_t *witness_lockless_error =
#endif
void
witnesses_cleanup(tsd_t *tsd)
{
witnesses_cleanup(tsd_t *tsd) {
witness_assert_lockless(tsd_tsdn(tsd));
/* Do nothing. */
}
void
witness_prefork(tsd_t *tsd)
{
witness_prefork(tsd_t *tsd) {
tsd_witness_fork_set(tsd, true);
}
void
witness_postfork_parent(tsd_t *tsd)
{
witness_postfork_parent(tsd_t *tsd) {
tsd_witness_fork_set(tsd, false);
}
void
witness_postfork_child(tsd_t *tsd)
{
witness_postfork_child(tsd_t *tsd) {
#ifndef JEMALLOC_MUTEX_INIT_CB
witness_list_t *witnesses;

96
src/zone.c
View File

@@ -125,8 +125,7 @@ static void zone_reinit_lock(malloc_zone_t *zone);
*/
static size_t
zone_size(malloc_zone_t *zone, const void *ptr)
{
zone_size(malloc_zone_t *zone, const void *ptr) {
/*
* There appear to be places within Darwin (such as setenv(3)) that
* cause calls to this function with pointers that *no* zone owns. If
@@ -140,20 +139,17 @@ zone_size(malloc_zone_t *zone, const void *ptr)
}
static void *
zone_malloc(malloc_zone_t *zone, size_t size)
{
zone_malloc(malloc_zone_t *zone, size_t size) {
return (je_malloc(size));
}
static void *
zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
{
zone_calloc(malloc_zone_t *zone, size_t num, size_t size) {
return (je_calloc(num, size));
}
static void *
zone_valloc(malloc_zone_t *zone, size_t size)
{
zone_valloc(malloc_zone_t *zone, size_t size) {
void *ret = NULL; /* Assignment avoids useless compiler warning. */
je_posix_memalign(&ret, PAGE, size);
@@ -162,8 +158,7 @@ zone_valloc(malloc_zone_t *zone, size_t size)
}
static void
zone_free(malloc_zone_t *zone, void *ptr)
{
zone_free(malloc_zone_t *zone, void *ptr) {
if (ivsalloc(tsdn_fetch(), ptr) != 0) {
je_free(ptr);
return;
@@ -173,17 +168,16 @@ zone_free(malloc_zone_t *zone, void *ptr)
}
static void *
zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
{
if (ivsalloc(tsdn_fetch(), ptr) != 0)
zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) {
if (ivsalloc(tsdn_fetch(), ptr) != 0) {
return (je_realloc(ptr, size));
}
return (realloc(ptr, size));
}
static void *
zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
{
zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) {
void *ret = NULL; /* Assignment avoids useless compiler warning. */
je_posix_memalign(&ret, alignment, size);
@@ -192,8 +186,7 @@ zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
}
static void
zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
{
zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) {
size_t alloc_size;
alloc_size = ivsalloc(tsdn_fetch(), ptr);
@@ -207,16 +200,14 @@ zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
}
static void
zone_destroy(malloc_zone_t *zone)
{
zone_destroy(malloc_zone_t *zone) {
/* This function should never be called. */
not_reached();
}
static unsigned
zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, void **results,
unsigned num_requested)
{
unsigned num_requested) {
unsigned i;
for (i = 0; i < num_requested; i++) {
@@ -230,8 +221,7 @@ zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, void **results,
static void
zone_batch_free(struct _malloc_zone_t *zone, void **to_be_freed,
unsigned num_to_be_freed)
{
unsigned num_to_be_freed) {
unsigned i;
for (i = 0; i < num_to_be_freed; i++) {
@@ -241,53 +231,47 @@ zone_batch_free(struct _malloc_zone_t *zone, void **to_be_freed,
}
static size_t
zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal)
{
zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal) {
return 0;
}
static size_t
zone_good_size(malloc_zone_t *zone, size_t size)
{
if (size == 0)
zone_good_size(malloc_zone_t *zone, size_t size) {
if (size == 0) {
size = 1;
}
return (s2u(size));
}
static kern_return_t
zone_enumerator(task_t task, void *data, unsigned type_mask,
vm_address_t zone_address, memory_reader_t reader,
vm_range_recorder_t recorder)
{
vm_range_recorder_t recorder) {
return KERN_SUCCESS;
}
static boolean_t
zone_check(malloc_zone_t *zone)
{
zone_check(malloc_zone_t *zone) {
return true;
}
static void
zone_print(malloc_zone_t *zone, boolean_t verbose)
{
zone_print(malloc_zone_t *zone, boolean_t verbose) {
}
static void
zone_log(malloc_zone_t *zone, void *address)
{
zone_log(malloc_zone_t *zone, void *address) {
}
static void
zone_force_lock(malloc_zone_t *zone)
{
if (isthreaded)
zone_force_lock(malloc_zone_t *zone) {
if (isthreaded) {
jemalloc_prefork();
}
}
static void
zone_force_unlock(malloc_zone_t *zone)
{
zone_force_unlock(malloc_zone_t *zone) {
/*
* Call jemalloc_postfork_child() rather than
* jemalloc_postfork_parent(), because this function is executed by both
@@ -295,13 +279,13 @@ zone_force_unlock(malloc_zone_t *zone)
* reinitialized, but the child cannot unlock mutexes that were locked
* by the parent.
*/
if (isthreaded)
if (isthreaded) {
jemalloc_postfork_child();
}
}
static void
zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats)
{
zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) {
/* We make no effort to actually fill the values */
stats->blocks_in_use = 0;
stats->size_in_use = 0;
@@ -310,23 +294,20 @@ zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats)
}
static boolean_t
zone_locked(malloc_zone_t *zone)
{
zone_locked(malloc_zone_t *zone) {
/* Pretend no lock is being held */
return false;
}
static void
zone_reinit_lock(malloc_zone_t *zone)
{
zone_reinit_lock(malloc_zone_t *zone) {
/* As of OSX 10.12, this function is only used when force_unlock would
* be used if the zone version were < 9. So just use force_unlock. */
zone_force_unlock(zone);
}
static void
zone_init(void)
{
zone_init(void) {
jemalloc_zone.size = zone_size;
jemalloc_zone.malloc = zone_malloc;
jemalloc_zone.calloc = zone_calloc;
@@ -364,8 +345,7 @@ zone_init(void)
}
static malloc_zone_t *
zone_default_get(void)
{
zone_default_get(void) {
malloc_zone_t **zones = NULL;
unsigned int num_zones = 0;
@@ -387,16 +367,16 @@ zone_default_get(void)
num_zones = 0;
}
if (num_zones)
if (num_zones) {
return (zones[0]);
}
return (malloc_default_zone());
}
/* As written, this function can only promote jemalloc_zone. */
static void
zone_promote(void)
{
zone_promote(void) {
malloc_zone_t *zone;
do {
@@ -433,16 +413,16 @@ zone_promote(void)
JEMALLOC_ATTR(constructor)
void
zone_register(void)
{
zone_register(void) {
/*
* If something else replaced the system default zone allocator, don't
* register jemalloc's.
*/
default_zone = zone_default_get();
if (!default_zone->zone_name || strcmp(default_zone->zone_name,
"DefaultMallocZone") != 0)
"DefaultMallocZone") != 0) {
return;
}
/*
* The default purgeable zone is created lazily by OSX's libc. It uses