project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add emap, for tracking extent locking.

Browse Source
This commit is contained in:
David Goldblatt
2020-01-27 13:55:46 -08:00
committed by David Goldblatt
parent 0f686e82a3
commit 01f255161c
17 changed files with 257 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/arena.c
View File

@@ -1112,8 +1112,9 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree,
rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind,
&alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES);
if (config_stats || (config_prof && opt_prof)) {
@@ -1601,13 +1602,13 @@ arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
edata_t *edata = rtree_edata_read(tsdn, &extents_rtree, rtree_ctx,
edata_t *edata = rtree_edata_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true);
szind_t szind = sz_size2index(usize);
edata_szind_set(edata, szind);
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
szind, false);
rtree_szind_slab_update(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, szind, false);
prof_idump_rollback(tsdn, usize);
@@ -1622,8 +1623,8 @@ arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
edata_szind_set(edata, SC_NBINS);
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
SC_NBINS, false);
rtree_szind_slab_update(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, SC_NBINS, false);
assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS);

127
src/emap.c Normal file
View File

@@ -0,0 +1,127 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/emap.h"
emap_t emap_global;
enum emap_lock_result_e {
emap_lock_result_success,
emap_lock_result_failure,
emap_lock_result_no_extent
};
typedef enum emap_lock_result_e emap_lock_result_t;
bool
emap_init(emap_t *emap) {
bool err;
err = rtree_new(&emap->rtree, true);
if (err) {
return true;
}
err = mutex_pool_init(&emap->mtx_pool, "emap_mutex_pool",
WITNESS_RANK_EMAP);
if (err) {
return true;
}
return false;
}
void
emap_lock_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
assert(edata != NULL);
mutex_pool_lock(tsdn, &emap->mtx_pool, (uintptr_t)edata);
}
void
emap_unlock_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
assert(edata != NULL);
mutex_pool_unlock(tsdn, &emap->mtx_pool, (uintptr_t)edata);
}
void
emap_lock_edata2(tsdn_t *tsdn, emap_t *emap, edata_t *edata1,
edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_lock2(tsdn, &emap->mtx_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
void
emap_unlock_edata2(tsdn_t *tsdn, emap_t *emap, edata_t *edata1,
edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_unlock2(tsdn, &emap->mtx_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
static inline emap_lock_result_t
emap_try_lock_rtree_leaf_elm(tsdn_t *tsdn, emap_t *emap, rtree_leaf_elm_t *elm,
edata_t **result, bool inactive_only) {
edata_t *edata1 = rtree_leaf_elm_edata_read(tsdn, &emap->rtree,
elm, true);
/* Slab implies active extents and should be skipped. */
if (edata1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn,
&emap->rtree, elm, true))) {
return emap_lock_result_no_extent;
}
/*
* It's possible that the extent changed out from under us, and with it
* the leaf->edata mapping. We have to recheck while holding the lock.
*/
emap_lock_edata(tsdn, emap, edata1);
edata_t *edata2 = rtree_leaf_elm_edata_read(tsdn, &emap->rtree, elm,
true);
if (edata1 == edata2) {
*result = edata1;
return emap_lock_result_success;
} else {
emap_unlock_edata(tsdn, emap, edata1);
return emap_lock_result_failure;
}
}
/*
* Returns a pool-locked edata_t * if there's one associated with the given
* address, and NULL otherwise.
*/
edata_t *
emap_lock_edata_from_addr(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx,
void *addr, bool inactive_only) {
edata_t *ret = NULL;
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &emap->rtree,
rtree_ctx, (uintptr_t)addr, false, false);
if (elm == NULL) {
return NULL;
}
emap_lock_result_t lock_result;
do {
lock_result = emap_try_lock_rtree_leaf_elm(tsdn, emap, elm,
&ret, inactive_only);
} while (lock_result == emap_lock_result_failure);
return ret;
}
bool
emap_rtree_leaf_elms_lookup(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx,
const edata_t *edata, bool dependent, bool init_missing,
rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) {
*r_elm_a = rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), dependent, init_missing);
if (!dependent && *r_elm_a == NULL) {
return true;
}
assert(*r_elm_a != NULL);
*r_elm_b = rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_last_get(edata), dependent, init_missing);
if (!dependent && *r_elm_b == NULL) {
return true;
}
assert(*r_elm_b != NULL);
return false;
}

211
src/extent.c
View File

@@ -2,20 +2,15 @@
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/ph.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_pool.h"
/******************************************************************************/
/* Data. */
rtree_t extents_rtree;
/* Keyed by the address of the edata_t being protected. */
mutex_pool_t extent_mutex_pool;
size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;
static bool extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
@@ -58,88 +53,6 @@ static edata_t *extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
/******************************************************************************/
typedef enum {
lock_result_success,
lock_result_failure,
lock_result_no_extent
} lock_result_t;
static inline void
extent_lock_edata(tsdn_t *tsdn, edata_t *edata) {
assert(edata != NULL);
mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)edata);
}
static inline void
extent_unlock_edata(tsdn_t *tsdn, edata_t *edata) {
assert(edata != NULL);
mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)edata);
}
static inline void
extent_lock_edata2(tsdn_t *tsdn, edata_t *edata1, edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
static inline void
extent_unlock_edata2(tsdn_t *tsdn, edata_t *edata1, edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
static lock_result_t
extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm,
edata_t **result, bool inactive_only) {
edata_t *edata1 = rtree_leaf_elm_edata_read(tsdn, &extents_rtree,
elm, true);
/* Slab implies active extents and should be skipped. */
if (edata1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn,
&extents_rtree, elm, true))) {
return lock_result_no_extent;
}
/*
* It's possible that the extent changed out from under us, and with it
* the leaf->edata mapping. We have to recheck while holding the lock.
*/
extent_lock_edata(tsdn, edata1);
edata_t *edata2 = rtree_leaf_elm_edata_read(tsdn, &extents_rtree, elm,
true);
if (edata1 == edata2) {
*result = edata1;
return lock_result_success;
} else {
extent_unlock_edata(tsdn, edata1);
return lock_result_failure;
}
}
/*
* Returns a pool-locked edata_t * if there's one associated with the given
* address, and NULL otherwise.
*/
static edata_t *
extent_lock_edata_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr,
bool inactive_only) {
edata_t *ret = NULL;
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)addr, false, false);
if (elm == NULL) {
return NULL;
}
lock_result_t lock_result;
do {
lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret,
inactive_only);
} while (lock_result == lock_result_failure);
return ret;
}
static void
extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
size_t alignment) {
@@ -357,34 +270,14 @@ extent_activate_locked(tsdn_t *tsdn, ecache_t *ecache, edata_t *edata) {
edata_state_set(edata, extent_state_active);
}
static bool
extent_rtree_leaf_elms_lookup(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
const edata_t *edata, bool dependent, bool init_missing,
rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) {
*r_elm_a = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), dependent, init_missing);
if (!dependent && *r_elm_a == NULL) {
return true;
}
assert(*r_elm_a != NULL);
*r_elm_b = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)edata_last_get(edata), dependent, init_missing);
if (!dependent && *r_elm_b == NULL) {
return true;
}
assert(*r_elm_b != NULL);
return false;
}
static void
extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a,
rtree_leaf_elm_t *elm_b, edata_t *edata, szind_t szind, bool slab) {
rtree_leaf_elm_write(tsdn, &extents_rtree, elm_a, edata, szind, slab);
rtree_leaf_elm_write(tsdn, &emap_global.rtree, elm_a, edata, szind,
slab);
if (elm_b != NULL) {
rtree_leaf_elm_write(tsdn, &extents_rtree, elm_b, edata, szind,
slab);
rtree_leaf_elm_write(tsdn, &emap_global.rtree, elm_b, edata,
szind, slab);
}
}
@@ -395,7 +288,7 @@ extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, edata_t *edata,
/* Register interior. */
for (size_t i = 1; i < (edata_size_get(edata) >> LG_PAGE) - 1; i++) {
rtree_write(tsdn, &extents_rtree, rtree_ctx,
rtree_write(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata) + (uintptr_t)(i <<
LG_PAGE), edata, szind, true);
}
@@ -448,11 +341,11 @@ extent_register_impl(tsdn_t *tsdn, edata_t *edata, bool gdump_add) {
* We need to hold the lock to protect against a concurrent coalesce
* operation that sees us in a partial state.
*/
extent_lock_edata(tsdn, edata);
emap_lock_edata(tsdn, &emap_global, edata);
if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, edata, false, true,
&elm_a, &elm_b)) {
extent_unlock_edata(tsdn, edata);
if (emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, edata,
false, true, &elm_a, &elm_b)) {
emap_unlock_edata(tsdn, &emap_global, edata);
return true;
}
@@ -463,7 +356,7 @@ extent_register_impl(tsdn_t *tsdn, edata_t *edata, bool gdump_add) {
extent_interior_register(tsdn, rtree_ctx, edata, szind);
}
extent_unlock_edata(tsdn, edata);
emap_unlock_edata(tsdn, &emap_global, edata);
if (config_prof && gdump_add) {
extent_gdump_add(tsdn, edata);
@@ -503,7 +396,7 @@ extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
assert(edata_slab_get(edata));
for (i = 1; i < (edata_size_get(edata) >> LG_PAGE) - 1; i++) {
rtree_clear(tsdn, &extents_rtree, rtree_ctx,
rtree_clear(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata) + (uintptr_t)(i <<
LG_PAGE));
}
@@ -517,10 +410,10 @@ extent_deregister_impl(tsdn_t *tsdn, edata_t *edata, bool gdump) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_leaf_elm_t *elm_a, *elm_b;
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, edata, true, false,
&elm_a, &elm_b);
emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, edata,
true, false, &elm_a, &elm_b);
extent_lock_edata(tsdn, edata);
emap_lock_edata(tsdn, &emap_global, edata);
extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false);
if (edata_slab_get(edata)) {
@@ -528,7 +421,7 @@ extent_deregister_impl(tsdn_t *tsdn, edata_t *edata, bool gdump) {
edata_slab_set(edata, false);
}
extent_unlock_edata(tsdn, edata);
emap_unlock_edata(tsdn, &emap_global, edata);
if (config_prof && gdump) {
extent_gdump_sub(tsdn, edata);
@@ -577,8 +470,8 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
malloc_mutex_lock(tsdn, &ecache->mtx);
edata_t *edata;
if (new_addr != NULL) {
edata = extent_lock_edata_from_addr(tsdn, rtree_ctx, new_addr,
false);
edata = emap_lock_edata_from_addr(tsdn, &emap_global, rtree_ctx,
new_addr, false);
if (edata != NULL) {
/*
* We might null-out edata to report an error, but we
@@ -592,7 +485,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
!= ecache->state) {
edata = NULL;
}
extent_unlock_edata(tsdn, unlock_edata);
emap_unlock_edata(tsdn, &emap_global, unlock_edata);
}
} else {
edata = eset_fit(&ecache->eset, esize, alignment,
@@ -692,11 +585,12 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
*/
edata_szind_set(*edata, szind);
if (szind != SC_NSIZES) {
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)edata_addr_get(*edata), szind, slab);
rtree_szind_slab_update(tsdn, &emap_global.rtree,
rtree_ctx, (uintptr_t)edata_addr_get(*edata), szind,
slab);
if (slab && edata_size_get(*edata) > PAGE) {
rtree_szind_slab_update(tsdn, &extents_rtree,
rtree_ctx,
rtree_szind_slab_update(tsdn,
&emap_global.rtree, rtree_ctx,
(uintptr_t)edata_past_get(*edata) -
(uintptr_t)PAGE, szind, slab);
}
@@ -760,8 +654,8 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, ehooks, ecache, to_leak,
growing_retained);
assert(extent_lock_edata_from_addr(tsdn, rtree_ctx, leak,
false) == NULL);
assert(emap_lock_edata_from_addr(tsdn, &emap_global,
rtree_ctx, leak, false) == NULL);
}
return NULL;
}
@@ -1119,8 +1013,8 @@ extent_try_coalesce_impl(tsdn_t *tsdn, edata_cache_t *edata_cache,
again = false;
/* Try to coalesce forward. */
edata_t *next = extent_lock_edata_from_addr(tsdn, rtree_ctx,
edata_past_get(edata), inactive_only);
edata_t *next = emap_lock_edata_from_addr(tsdn, &emap_global,
rtree_ctx, edata_past_get(edata), inactive_only);
if (next != NULL) {
/*
* ecache->mtx only protects against races for
@@ -1130,7 +1024,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, edata_cache_t *edata_cache,
bool can_coalesce = extent_can_coalesce(ecache,
edata, next);
extent_unlock_edata(tsdn, next);
emap_unlock_edata(tsdn, &emap_global, next);
if (can_coalesce && !extent_coalesce(tsdn, edata_cache,
ehooks, ecache, edata, next, true,
@@ -1145,12 +1039,12 @@ extent_try_coalesce_impl(tsdn_t *tsdn, edata_cache_t *edata_cache,
}
/* Try to coalesce backward. */
edata_t *prev = extent_lock_edata_from_addr(tsdn, rtree_ctx,
edata_before_get(edata), inactive_only);
edata_t *prev = emap_lock_edata_from_addr(tsdn, &emap_global,
rtree_ctx, edata_before_get(edata), inactive_only);
if (prev != NULL) {
bool can_coalesce = extent_can_coalesce(ecache, edata,
prev);
extent_unlock_edata(tsdn, prev);
emap_unlock_edata(tsdn, &emap_global, prev);
if (can_coalesce && !extent_coalesce(tsdn, edata_cache,
ehooks, ecache, edata, prev, false,
@@ -1210,7 +1104,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
edata_slab_set(edata, false);
}
assert(rtree_edata_read(tsdn, &extents_rtree, rtree_ctx,
assert(rtree_edata_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), true) == edata);
if (!ecache->delay_coalesce) {
@@ -1449,19 +1343,19 @@ extent_split_impl(tsdn_t *tsdn, edata_cache_t *edata_cache, ehooks_t *ehooks,
edata_committed_get(edata), edata_dumpable_get(edata),
EXTENT_NOT_HEAD);
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false,
true, &lead_elm_a, &lead_elm_b);
emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx,
&lead, false, true, &lead_elm_a, &lead_elm_b);
}
rtree_leaf_elm_t *trail_elm_a, *trail_elm_b;
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false, true,
&trail_elm_a, &trail_elm_b);
emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, trail, false,
true, &trail_elm_a, &trail_elm_b);
if (lead_elm_a == NULL || lead_elm_b == NULL || trail_elm_a == NULL
|| trail_elm_b == NULL) {
goto label_error_b;
}
extent_lock_edata2(tsdn, edata, trail);
emap_lock_edata2(tsdn, &emap_global, edata, trail);
bool err = ehooks_split(tsdn, ehooks, edata_base_get(edata),
size_a + size_b, size_a, size_b, edata_committed_get(edata));
@@ -1478,11 +1372,11 @@ extent_split_impl(tsdn_t *tsdn, edata_cache_t *edata_cache, ehooks_t *ehooks,
extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail,
szind_b, slab_b);
extent_unlock_edata2(tsdn, edata, trail);
emap_unlock_edata2(tsdn, &emap_global, edata, trail);
return trail;
label_error_c:
extent_unlock_edata2(tsdn, edata, trail);
emap_unlock_edata2(tsdn, &emap_global, edata, trail);
label_error_b:
edata_cache_put(tsdn, edata_cache, trail);
label_error_a:
@@ -1523,19 +1417,19 @@ extent_merge_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_cache_t *edata_cache,
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_leaf_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, a, true, false, &a_elm_a,
&a_elm_b);
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true, false, &b_elm_a,
&b_elm_b);
emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, a, true,
false, &a_elm_a, &a_elm_b);
emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, b, true,
false, &b_elm_a, &b_elm_b);
extent_lock_edata2(tsdn, a, b);
emap_lock_edata2(tsdn, &emap_global, a, b);
if (a_elm_b != NULL) {
rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL,
rtree_leaf_elm_write(tsdn, &emap_global.rtree, a_elm_b, NULL,
SC_NSIZES, false);
}
if (b_elm_b != NULL) {
rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL,
rtree_leaf_elm_write(tsdn, &emap_global.rtree, b_elm_a, NULL,
SC_NSIZES, false);
} else {
b_elm_b = b_elm_a;
@@ -1550,7 +1444,7 @@ extent_merge_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_cache_t *edata_cache,
extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES,
false);
extent_unlock_edata2(tsdn, a, b);
emap_unlock_edata2(tsdn, &emap_global, a, b);
edata_cache_put(tsdn, edata_cache, b);
@@ -1567,15 +1461,6 @@ bool
extent_boot(void) {
assert(sizeof(slab_data_t) >= sizeof(e_prof_info_t));
if (rtree_new(&extents_rtree, true)) {
return true;
}
if (mutex_pool_init(&extent_mutex_pool, "extent_mutex_pool",
WITNESS_RANK_EXTENT_POOL)) {
return true;
}
if (have_dss) {
extent_dss_boot();
}

20
src/jemalloc.c
View File

@@ -6,6 +6,7 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/buf_writer.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/hook.h"
@@ -1571,6 +1572,9 @@ malloc_init_hard_a0_locked() {
if (base_boot(TSDN_NULL)) {
return true;
}
if (emap_init(&emap_global)) {
return true;
}
if (extent_boot()) {
return true;
}
@@ -2565,7 +2569,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES);
@@ -2619,15 +2623,16 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
alloc_ctx_t dbg_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd),
&extents_rtree, rtree_ctx, (uintptr_t)ptr,
true, &dbg_ctx.szind, &dbg_ctx.slab);
&emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &dbg_ctx.szind,
&dbg_ctx.slab);
assert(dbg_ctx.szind == ctx->szind);
assert(dbg_ctx.slab == ctx->slab);
}
} else if (opt_prof) {
ctx = &alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree,
rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree,
rtree_ctx, (uintptr_t)ptr, true, &ctx->szind,
&ctx->slab);
/* Small alloc may have !slab (sampled). */
@@ -2699,7 +2704,8 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
bool slab;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd),
&extents_rtree, rtree_ctx, (uintptr_t)ptr, &szind, &slab);
&emap_global.rtree, rtree_ctx, (uintptr_t)ptr, &szind,
&slab);
/* Note: profiled objects will have alloc_ctx.slab set */
if (unlikely(!res || !slab)) {
@@ -3142,7 +3148,7 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind);
@@ -3421,7 +3427,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind);

2
src/large.c
View File

@@ -179,7 +179,7 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
szind_t szind = sz_size2index(usize);
edata_szind_set(edata, szind);
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,
rtree_szind_slab_update(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_addr_get(edata), szind, false);
if (config_stats && new_mapping) {

2
src/tcache.c
View File

@@ -129,7 +129,7 @@ tbin_edatas_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind,
size_t sz_sum = binind * nflush;
void **bottom_item = cache_bin_bottom_item_get(tbin, binind);
for (unsigned i = 0 ; i < nflush; i++) {
rtree_edata_szind_read(tsdn, &extents_rtree,
rtree_edata_szind_read(tsdn, &emap_global.rtree,
rtree_ctx, (uintptr_t)*(bottom_item - i), true,
&edatas[i], &szind);
sz_sum -= szind;