Add emap, for tracking extent locking.

This commit is contained in:
David Goldblatt 2020-01-27 13:55:46 -08:00 committed by David Goldblatt
parent 0f686e82a3
commit 01f255161c
17 changed files with 257 additions and 197 deletions

View File

@ -110,6 +110,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/edata.c \ $(srcroot)src/edata.c \
$(srcroot)src/edata_cache.c \ $(srcroot)src/edata_cache.c \
$(srcroot)src/ehooks.c \ $(srcroot)src/ehooks.c \
$(srcroot)src/emap.c \
$(srcroot)src/eset.c \ $(srcroot)src/eset.c \
$(srcroot)src/extent.c \ $(srcroot)src/extent.c \
$(srcroot)src/extent_dss.c \ $(srcroot)src/extent_dss.c \

View File

@ -188,7 +188,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) {
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx, szind_t szind = rtree_szind_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true); (uintptr_t)ptr, true);
assert(szind != SC_NSIZES); assert(szind != SC_NSIZES);
@ -211,7 +211,7 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
edata_t *edata; edata_t *edata;
szind_t szind; szind_t szind;
if (rtree_edata_szind_read(tsdn, &extents_rtree, rtree_ctx, if (rtree_edata_szind_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, false, &edata, &szind)) { (uintptr_t)ptr, false, &edata, &szind)) {
return 0; return 0;
} }
@ -247,11 +247,11 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
szind_t szind; szind_t szind;
bool slab; bool slab;
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, rtree_szind_slab_read(tsdn, &emap_global.rtree, rtree_ctx,
true, &szind, &slab); (uintptr_t)ptr, true, &szind, &slab);
if (config_debug) { if (config_debug) {
edata_t *edata = rtree_edata_read(tsdn, &extents_rtree, edata_t *edata = rtree_edata_read(tsdn, &emap_global.rtree,
rtree_ctx, (uintptr_t)ptr, true); rtree_ctx, (uintptr_t)ptr, true);
assert(szind == edata_szind_get(edata)); assert(szind == edata_szind_get(edata));
assert(szind < SC_NSIZES); assert(szind < SC_NSIZES);
@ -302,13 +302,13 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
assert(szind != SC_NSIZES); assert(szind != SC_NSIZES);
} else { } else {
rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, rtree_szind_slab_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &szind, &slab); (uintptr_t)ptr, true, &szind, &slab);
} }
if (config_debug) { if (config_debug) {
rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
edata_t *edata = rtree_edata_read(tsdn, &extents_rtree, edata_t *edata = rtree_edata_read(tsdn, &emap_global.rtree,
rtree_ctx, (uintptr_t)ptr, true); rtree_ctx, (uintptr_t)ptr, true);
assert(szind == edata_szind_get(edata)); assert(szind == edata_szind_get(edata));
assert(szind < SC_NSIZES); assert(szind < SC_NSIZES);
@ -345,7 +345,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
&rtree_ctx_fallback); &rtree_ctx_fallback);
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, rtree_szind_slab_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &szind, &slab); (uintptr_t)ptr, true, &szind, &slab);
assert(szind == sz_size2index(size)); assert(szind == sz_size2index(size));
@ -353,7 +353,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
if (config_debug) { if (config_debug) {
edata_t *edata = rtree_edata_read(tsdn, edata_t *edata = rtree_edata_read(tsdn,
&extents_rtree, rtree_ctx, (uintptr_t)ptr, true); &emap_global.rtree, rtree_ctx, (uintptr_t)ptr, true);
assert(szind == edata_szind_get(edata)); assert(szind == edata_szind_get(edata));
assert(slab == edata_slab_get(edata)); assert(slab == edata_slab_get(edata));
} }
@ -388,8 +388,8 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
&rtree_ctx_fallback); &rtree_ctx_fallback);
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, rtree_szind_slab_read(tsdn, &emap_global.rtree,
(uintptr_t)ptr, true, &local_ctx.szind, rtree_ctx, (uintptr_t)ptr, true, &local_ctx.szind,
&local_ctx.slab); &local_ctx.slab);
assert(local_ctx.szind == sz_size2index(size)); assert(local_ctx.szind == sz_size2index(size));
alloc_ctx = &local_ctx; alloc_ctx = &local_ctx;
@ -407,10 +407,10 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
if (config_debug) { if (config_debug) {
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, rtree_szind_slab_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &szind, &slab); (uintptr_t)ptr, true, &szind, &slab);
edata_t *edata = rtree_edata_read(tsdn, edata_t *edata = rtree_edata_read(tsdn,
&extents_rtree, rtree_ctx, (uintptr_t)ptr, true); &emap_global.rtree, rtree_ctx, (uintptr_t)ptr, true);
assert(szind == edata_szind_get(edata)); assert(szind == edata_szind_get(edata));
assert(slab == edata_slab_get(edata)); assert(slab == edata_slab_get(edata));
} }

View File

@ -0,0 +1,33 @@
#ifndef JEMALLOC_INTERNAL_EMAP_H
#define JEMALLOC_INTERNAL_EMAP_H
#include "jemalloc/internal/mutex_pool.h"
#include "jemalloc/internal/rtree.h"
typedef struct emap_s emap_t;
struct emap_s {
rtree_t rtree;
/* Keyed by the address of the edata_t being protected. */
mutex_pool_t mtx_pool;
};
extern emap_t emap_global;
bool emap_init(emap_t *emap);
void emap_lock_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata);
void emap_unlock_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata);
void emap_lock_edata2(tsdn_t *tsdn, emap_t *emap, edata_t *edata1,
edata_t *edata2);
void emap_unlock_edata2(tsdn_t *tsdn, emap_t *emap, edata_t *edata1,
edata_t *edata2);
edata_t *emap_lock_edata_from_addr(tsdn_t *tsdn, emap_t *emap,
rtree_ctx_t *rtree_ctx, void *addr, bool inactive_only);
bool emap_rtree_leaf_elms_lookup(tsdn_t *tsdn, emap_t *emap,
rtree_ctx_t *rtree_ctx, const edata_t *edata, bool dependent,
bool init_missing, rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b);
#endif /* JEMALLOC_INTERNAL_EMAP_H */

View File

@ -19,8 +19,6 @@
#define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6 #define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6
extern size_t opt_lg_extent_max_active_fit; extern size_t opt_lg_extent_max_active_fit;
extern rtree_t extents_rtree;
edata_t *ecache_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *ecache_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
ecache_t *ecache, void *new_addr, size_t size, size_t pad, size_t alignment, ecache_t *ecache, void *new_addr, size_t size, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero); bool slab, szind_t szind, bool *zero);

View File

@ -1,6 +1,7 @@
#ifndef JEMALLOC_INTERNAL_INLINES_B_H #ifndef JEMALLOC_INTERNAL_INLINES_B_H
#define JEMALLOC_INTERNAL_INLINES_B_H #define JEMALLOC_INTERNAL_INLINES_B_H
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent.h" #include "jemalloc/internal/extent.h"
#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/rtree.h"
@ -81,7 +82,7 @@ iealloc(tsdn_t *tsdn, const void *ptr) {
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
return rtree_edata_read(tsdn, &extents_rtree, rtree_ctx, return rtree_edata_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true); (uintptr_t)ptr, true);
} }

View File

@ -45,7 +45,7 @@
#define WITNESS_RANK_EXTENTS 15U #define WITNESS_RANK_EXTENTS 15U
#define WITNESS_RANK_EDATA_CACHE 16U #define WITNESS_RANK_EDATA_CACHE 16U
#define WITNESS_RANK_EXTENT_POOL 17U #define WITNESS_RANK_EMAP 17U
#define WITNESS_RANK_RTREE 18U #define WITNESS_RANK_RTREE 18U
#define WITNESS_RANK_BASE 19U #define WITNESS_RANK_BASE 19U
#define WITNESS_RANK_ARENA_LARGE 20U #define WITNESS_RANK_ARENA_LARGE 20U

View File

@ -50,6 +50,7 @@
<ClCompile Include="..\..\..\..\src\edata.c" /> <ClCompile Include="..\..\..\..\src\edata.c" />
<ClCompile Include="..\..\..\..\src\edata_cache.c" /> <ClCompile Include="..\..\..\..\src\edata_cache.c" />
<ClCompile Include="..\..\..\..\src\ehooks.c" /> <ClCompile Include="..\..\..\..\src\ehooks.c" />
<ClCompile Include="..\..\..\..\src\emap.c" />
<ClCompile Include="..\..\..\..\src\eset.c" /> <ClCompile Include="..\..\..\..\src\eset.c" />
<ClCompile Include="..\..\..\..\src\extent.c" /> <ClCompile Include="..\..\..\..\src\extent.c" />
<ClCompile Include="..\..\..\..\src\extent_dss.c" /> <ClCompile Include="..\..\..\..\src\extent_dss.c" />

View File

@ -37,6 +37,9 @@
<ClCompile Include="..\..\..\..\src\div.c"> <ClCompile Include="..\..\..\..\src\div.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\..\..\..\src\emap.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\extent.c"> <ClCompile Include="..\..\..\..\src\extent.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>

View File

@ -50,6 +50,7 @@
<ClCompile Include="..\..\..\..\src\edata.c" /> <ClCompile Include="..\..\..\..\src\edata.c" />
<ClCompile Include="..\..\..\..\src\edata_cache.c" /> <ClCompile Include="..\..\..\..\src\edata_cache.c" />
<ClCompile Include="..\..\..\..\src\ehooks.c" /> <ClCompile Include="..\..\..\..\src\ehooks.c" />
<ClCompile Include="..\..\..\..\src\emap.c" />
<ClCompile Include="..\..\..\..\src\eset.c" /> <ClCompile Include="..\..\..\..\src\eset.c" />
<ClCompile Include="..\..\..\..\src\extent.c" /> <ClCompile Include="..\..\..\..\src\extent.c" />
<ClCompile Include="..\..\..\..\src\extent_dss.c" /> <ClCompile Include="..\..\..\..\src\extent_dss.c" />

View File

@ -37,6 +37,9 @@
<ClCompile Include="..\..\..\..\src\div.c"> <ClCompile Include="..\..\..\..\src\div.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\..\..\..\src\emap.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\extent.c"> <ClCompile Include="..\..\..\..\src\extent.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>

View File

@ -1112,8 +1112,9 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind,
&alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES); assert(alloc_ctx.szind != SC_NSIZES);
if (config_stats || (config_prof && opt_prof)) { if (config_stats || (config_prof && opt_prof)) {
@ -1601,13 +1602,13 @@ arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize) {
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
edata_t *edata = rtree_edata_read(tsdn, &extents_rtree, rtree_ctx, edata_t *edata = rtree_edata_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true); (uintptr_t)ptr, true);
szind_t szind = sz_size2index(usize); szind_t szind = sz_size2index(usize);
edata_szind_set(edata, szind); edata_szind_set(edata, szind);
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, rtree_szind_slab_update(tsdn, &emap_global.rtree, rtree_ctx,
szind, false); (uintptr_t)ptr, szind, false);
prof_idump_rollback(tsdn, usize); prof_idump_rollback(tsdn, usize);
@ -1622,8 +1623,8 @@ arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
edata_szind_set(edata, SC_NBINS); edata_szind_set(edata, SC_NBINS);
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, rtree_szind_slab_update(tsdn, &emap_global.rtree, rtree_ctx,
SC_NBINS, false); (uintptr_t)ptr, SC_NBINS, false);
assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS);

127
src/emap.c Normal file
View File

@ -0,0 +1,127 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/emap.h"
emap_t emap_global;
enum emap_lock_result_e {
emap_lock_result_success,
emap_lock_result_failure,
emap_lock_result_no_extent
};
typedef enum emap_lock_result_e emap_lock_result_t;
bool
emap_init(emap_t *emap) {
bool err;
err = rtree_new(&emap->rtree, true);
if (err) {
return true;
}
err = mutex_pool_init(&emap->mtx_pool, "emap_mutex_pool",
WITNESS_RANK_EMAP);
if (err) {
return true;
}
return false;
}
void
emap_lock_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
assert(edata != NULL);
mutex_pool_lock(tsdn, &emap->mtx_pool, (uintptr_t)edata);
}
void
emap_unlock_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
assert(edata != NULL);
mutex_pool_unlock(tsdn, &emap->mtx_pool, (uintptr_t)edata);
}
void
emap_lock_edata2(tsdn_t *tsdn, emap_t *emap, edata_t *edata1,
edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_lock2(tsdn, &emap->mtx_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
void
emap_unlock_edata2(tsdn_t *tsdn, emap_t *emap, edata_t *edata1,
edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_unlock2(tsdn, &emap->mtx_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
static inline emap_lock_result_t
emap_try_lock_rtree_leaf_elm(tsdn_t *tsdn, emap_t *emap, rtree_leaf_elm_t *elm,
edata_t **result, bool inactive_only) {
edata_t *edata1 = rtree_leaf_elm_edata_read(tsdn, &emap->rtree,
elm, true);
/* Slab implies active extents and should be skipped. */
if (edata1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn,
&emap->rtree, elm, true))) {
return emap_lock_result_no_extent;
}
/*
* It's possible that the extent changed out from under us, and with it
* the leaf->edata mapping. We have to recheck while holding the lock.
*/
emap_lock_edata(tsdn, emap, edata1);
edata_t *edata2 = rtree_leaf_elm_edata_read(tsdn, &emap->rtree, elm,
true);
if (edata1 == edata2) {
*result = edata1;
return emap_lock_result_success;
} else {
emap_unlock_edata(tsdn, emap, edata1);
return emap_lock_result_failure;
}
}
/*
* Returns a pool-locked edata_t * if there's one associated with the given
* address, and NULL otherwise.
*/
edata_t *
emap_lock_edata_from_addr(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx,
void *addr, bool inactive_only) {
edata_t *ret = NULL;
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &emap->rtree,
rtree_ctx, (uintptr_t)addr, false, false);
if (elm == NULL) {
return NULL;
}
emap_lock_result_t lock_result;
do {
lock_result = emap_try_lock_rtree_leaf_elm(tsdn, emap, elm,
&ret, inactive_only);
} while (lock_result == emap_lock_result_failure);
return ret;
}
bool
emap_rtree_leaf_elms_lookup(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx,
const edata_t *edata, bool dependent, bool init_missing,
rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) {
*r_elm_a = rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), dependent, init_missing);
if (!dependent && *r_elm_a == NULL) {
return true;
}
assert(*r_elm_a != NULL);
*r_elm_b = rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_last_get(edata), dependent, init_missing);
if (!dependent && *r_elm_b == NULL) {
return true;
}
assert(*r_elm_b != NULL);
return false;
}

View File

@ -2,20 +2,15 @@
#include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/assert.h" #include "jemalloc/internal/assert.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/ph.h" #include "jemalloc/internal/ph.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_pool.h"
/******************************************************************************/ /******************************************************************************/
/* Data. */ /* Data. */
rtree_t extents_rtree;
/* Keyed by the address of the edata_t being protected. */
mutex_pool_t extent_mutex_pool;
size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT; size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;
static bool extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, static bool extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
@ -58,88 +53,6 @@ static edata_t *extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
/******************************************************************************/ /******************************************************************************/
typedef enum {
lock_result_success,
lock_result_failure,
lock_result_no_extent
} lock_result_t;
static inline void
extent_lock_edata(tsdn_t *tsdn, edata_t *edata) {
assert(edata != NULL);
mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)edata);
}
static inline void
extent_unlock_edata(tsdn_t *tsdn, edata_t *edata) {
assert(edata != NULL);
mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)edata);
}
static inline void
extent_lock_edata2(tsdn_t *tsdn, edata_t *edata1, edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
static inline void
extent_unlock_edata2(tsdn_t *tsdn, edata_t *edata1, edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
static lock_result_t
extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm,
edata_t **result, bool inactive_only) {
edata_t *edata1 = rtree_leaf_elm_edata_read(tsdn, &extents_rtree,
elm, true);
/* Slab implies active extents and should be skipped. */
if (edata1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn,
&extents_rtree, elm, true))) {
return lock_result_no_extent;
}
/*
* It's possible that the extent changed out from under us, and with it
* the leaf->edata mapping. We have to recheck while holding the lock.
*/
extent_lock_edata(tsdn, edata1);
edata_t *edata2 = rtree_leaf_elm_edata_read(tsdn, &extents_rtree, elm,
true);
if (edata1 == edata2) {
*result = edata1;
return lock_result_success;
} else {
extent_unlock_edata(tsdn, edata1);
return lock_result_failure;
}
}
/*
* Returns a pool-locked edata_t * if there's one associated with the given
* address, and NULL otherwise.
*/
static edata_t *
extent_lock_edata_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr,
bool inactive_only) {
edata_t *ret = NULL;
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)addr, false, false);
if (elm == NULL) {
return NULL;
}
lock_result_t lock_result;
do {
lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret,
inactive_only);
} while (lock_result == lock_result_failure);
return ret;
}
static void static void
extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata, extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
size_t alignment) { size_t alignment) {
@ -357,34 +270,14 @@ extent_activate_locked(tsdn_t *tsdn, ecache_t *ecache, edata_t *edata) {
edata_state_set(edata, extent_state_active); edata_state_set(edata, extent_state_active);
} }
static bool
extent_rtree_leaf_elms_lookup(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
const edata_t *edata, bool dependent, bool init_missing,
rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) {
*r_elm_a = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), dependent, init_missing);
if (!dependent && *r_elm_a == NULL) {
return true;
}
assert(*r_elm_a != NULL);
*r_elm_b = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)edata_last_get(edata), dependent, init_missing);
if (!dependent && *r_elm_b == NULL) {
return true;
}
assert(*r_elm_b != NULL);
return false;
}
static void static void
extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a, extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a,
rtree_leaf_elm_t *elm_b, edata_t *edata, szind_t szind, bool slab) { rtree_leaf_elm_t *elm_b, edata_t *edata, szind_t szind, bool slab) {
rtree_leaf_elm_write(tsdn, &extents_rtree, elm_a, edata, szind, slab); rtree_leaf_elm_write(tsdn, &emap_global.rtree, elm_a, edata, szind,
slab);
if (elm_b != NULL) { if (elm_b != NULL) {
rtree_leaf_elm_write(tsdn, &extents_rtree, elm_b, edata, szind, rtree_leaf_elm_write(tsdn, &emap_global.rtree, elm_b, edata,
slab); szind, slab);
} }
} }
@ -395,7 +288,7 @@ extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, edata_t *edata,
/* Register interior. */ /* Register interior. */
for (size_t i = 1; i < (edata_size_get(edata) >> LG_PAGE) - 1; i++) { for (size_t i = 1; i < (edata_size_get(edata) >> LG_PAGE) - 1; i++) {
rtree_write(tsdn, &extents_rtree, rtree_ctx, rtree_write(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata) + (uintptr_t)(i << (uintptr_t)edata_base_get(edata) + (uintptr_t)(i <<
LG_PAGE), edata, szind, true); LG_PAGE), edata, szind, true);
} }
@ -448,11 +341,11 @@ extent_register_impl(tsdn_t *tsdn, edata_t *edata, bool gdump_add) {
* We need to hold the lock to protect against a concurrent coalesce * We need to hold the lock to protect against a concurrent coalesce
* operation that sees us in a partial state. * operation that sees us in a partial state.
*/ */
extent_lock_edata(tsdn, edata); emap_lock_edata(tsdn, &emap_global, edata);
if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, edata, false, true, if (emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, edata,
&elm_a, &elm_b)) { false, true, &elm_a, &elm_b)) {
extent_unlock_edata(tsdn, edata); emap_unlock_edata(tsdn, &emap_global, edata);
return true; return true;
} }
@ -463,7 +356,7 @@ extent_register_impl(tsdn_t *tsdn, edata_t *edata, bool gdump_add) {
extent_interior_register(tsdn, rtree_ctx, edata, szind); extent_interior_register(tsdn, rtree_ctx, edata, szind);
} }
extent_unlock_edata(tsdn, edata); emap_unlock_edata(tsdn, &emap_global, edata);
if (config_prof && gdump_add) { if (config_prof && gdump_add) {
extent_gdump_add(tsdn, edata); extent_gdump_add(tsdn, edata);
@ -503,7 +396,7 @@ extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
assert(edata_slab_get(edata)); assert(edata_slab_get(edata));
for (i = 1; i < (edata_size_get(edata) >> LG_PAGE) - 1; i++) { for (i = 1; i < (edata_size_get(edata) >> LG_PAGE) - 1; i++) {
rtree_clear(tsdn, &extents_rtree, rtree_ctx, rtree_clear(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata) + (uintptr_t)(i << (uintptr_t)edata_base_get(edata) + (uintptr_t)(i <<
LG_PAGE)); LG_PAGE));
} }
@ -517,10 +410,10 @@ extent_deregister_impl(tsdn_t *tsdn, edata_t *edata, bool gdump) {
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_leaf_elm_t *elm_a, *elm_b; rtree_leaf_elm_t *elm_a, *elm_b;
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, edata, true, false, emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, edata,
&elm_a, &elm_b); true, false, &elm_a, &elm_b);
extent_lock_edata(tsdn, edata); emap_lock_edata(tsdn, &emap_global, edata);
extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false); extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false);
if (edata_slab_get(edata)) { if (edata_slab_get(edata)) {
@ -528,7 +421,7 @@ extent_deregister_impl(tsdn_t *tsdn, edata_t *edata, bool gdump) {
edata_slab_set(edata, false); edata_slab_set(edata, false);
} }
extent_unlock_edata(tsdn, edata); emap_unlock_edata(tsdn, &emap_global, edata);
if (config_prof && gdump) { if (config_prof && gdump) {
extent_gdump_sub(tsdn, edata); extent_gdump_sub(tsdn, edata);
@ -577,8 +470,8 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
malloc_mutex_lock(tsdn, &ecache->mtx); malloc_mutex_lock(tsdn, &ecache->mtx);
edata_t *edata; edata_t *edata;
if (new_addr != NULL) { if (new_addr != NULL) {
edata = extent_lock_edata_from_addr(tsdn, rtree_ctx, new_addr, edata = emap_lock_edata_from_addr(tsdn, &emap_global, rtree_ctx,
false); new_addr, false);
if (edata != NULL) { if (edata != NULL) {
/* /*
* We might null-out edata to report an error, but we * We might null-out edata to report an error, but we
@ -592,7 +485,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
!= ecache->state) { != ecache->state) {
edata = NULL; edata = NULL;
} }
extent_unlock_edata(tsdn, unlock_edata); emap_unlock_edata(tsdn, &emap_global, unlock_edata);
} }
} else { } else {
edata = eset_fit(&ecache->eset, esize, alignment, edata = eset_fit(&ecache->eset, esize, alignment,
@ -692,11 +585,12 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
*/ */
edata_szind_set(*edata, szind); edata_szind_set(*edata, szind);
if (szind != SC_NSIZES) { if (szind != SC_NSIZES) {
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, rtree_szind_slab_update(tsdn, &emap_global.rtree,
(uintptr_t)edata_addr_get(*edata), szind, slab); rtree_ctx, (uintptr_t)edata_addr_get(*edata), szind,
slab);
if (slab && edata_size_get(*edata) > PAGE) { if (slab && edata_size_get(*edata) > PAGE) {
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_szind_slab_update(tsdn,
rtree_ctx, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_past_get(*edata) - (uintptr_t)edata_past_get(*edata) -
(uintptr_t)PAGE, szind, slab); (uintptr_t)PAGE, szind, slab);
} }
@ -760,8 +654,8 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_deregister_no_gdump_sub(tsdn, to_leak); extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, ehooks, ecache, to_leak, extents_abandon_vm(tsdn, arena, ehooks, ecache, to_leak,
growing_retained); growing_retained);
assert(extent_lock_edata_from_addr(tsdn, rtree_ctx, leak, assert(emap_lock_edata_from_addr(tsdn, &emap_global,
false) == NULL); rtree_ctx, leak, false) == NULL);
} }
return NULL; return NULL;
} }
@ -1119,8 +1013,8 @@ extent_try_coalesce_impl(tsdn_t *tsdn, edata_cache_t *edata_cache,
again = false; again = false;
/* Try to coalesce forward. */ /* Try to coalesce forward. */
edata_t *next = extent_lock_edata_from_addr(tsdn, rtree_ctx, edata_t *next = emap_lock_edata_from_addr(tsdn, &emap_global,
edata_past_get(edata), inactive_only); rtree_ctx, edata_past_get(edata), inactive_only);
if (next != NULL) { if (next != NULL) {
/* /*
* ecache->mtx only protects against races for * ecache->mtx only protects against races for
@ -1130,7 +1024,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, edata_cache_t *edata_cache,
bool can_coalesce = extent_can_coalesce(ecache, bool can_coalesce = extent_can_coalesce(ecache,
edata, next); edata, next);
extent_unlock_edata(tsdn, next); emap_unlock_edata(tsdn, &emap_global, next);
if (can_coalesce && !extent_coalesce(tsdn, edata_cache, if (can_coalesce && !extent_coalesce(tsdn, edata_cache,
ehooks, ecache, edata, next, true, ehooks, ecache, edata, next, true,
@ -1145,12 +1039,12 @@ extent_try_coalesce_impl(tsdn_t *tsdn, edata_cache_t *edata_cache,
} }
/* Try to coalesce backward. */ /* Try to coalesce backward. */
edata_t *prev = extent_lock_edata_from_addr(tsdn, rtree_ctx, edata_t *prev = emap_lock_edata_from_addr(tsdn, &emap_global,
edata_before_get(edata), inactive_only); rtree_ctx, edata_before_get(edata), inactive_only);
if (prev != NULL) { if (prev != NULL) {
bool can_coalesce = extent_can_coalesce(ecache, edata, bool can_coalesce = extent_can_coalesce(ecache, edata,
prev); prev);
extent_unlock_edata(tsdn, prev); emap_unlock_edata(tsdn, &emap_global, prev);
if (can_coalesce && !extent_coalesce(tsdn, edata_cache, if (can_coalesce && !extent_coalesce(tsdn, edata_cache,
ehooks, ecache, edata, prev, false, ehooks, ecache, edata, prev, false,
@ -1210,7 +1104,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
edata_slab_set(edata, false); edata_slab_set(edata, false);
} }
assert(rtree_edata_read(tsdn, &extents_rtree, rtree_ctx, assert(rtree_edata_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), true) == edata); (uintptr_t)edata_base_get(edata), true) == edata);
if (!ecache->delay_coalesce) { if (!ecache->delay_coalesce) {
@ -1449,19 +1343,19 @@ extent_split_impl(tsdn_t *tsdn, edata_cache_t *edata_cache, ehooks_t *ehooks,
edata_committed_get(edata), edata_dumpable_get(edata), edata_committed_get(edata), edata_dumpable_get(edata),
EXTENT_NOT_HEAD); EXTENT_NOT_HEAD);
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false, emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx,
true, &lead_elm_a, &lead_elm_b); &lead, false, true, &lead_elm_a, &lead_elm_b);
} }
rtree_leaf_elm_t *trail_elm_a, *trail_elm_b; rtree_leaf_elm_t *trail_elm_a, *trail_elm_b;
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false, true, emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, trail, false,
&trail_elm_a, &trail_elm_b); true, &trail_elm_a, &trail_elm_b);
if (lead_elm_a == NULL || lead_elm_b == NULL || trail_elm_a == NULL if (lead_elm_a == NULL || lead_elm_b == NULL || trail_elm_a == NULL
|| trail_elm_b == NULL) { || trail_elm_b == NULL) {
goto label_error_b; goto label_error_b;
} }
extent_lock_edata2(tsdn, edata, trail); emap_lock_edata2(tsdn, &emap_global, edata, trail);
bool err = ehooks_split(tsdn, ehooks, edata_base_get(edata), bool err = ehooks_split(tsdn, ehooks, edata_base_get(edata),
size_a + size_b, size_a, size_b, edata_committed_get(edata)); size_a + size_b, size_a, size_b, edata_committed_get(edata));
@ -1478,11 +1372,11 @@ extent_split_impl(tsdn_t *tsdn, edata_cache_t *edata_cache, ehooks_t *ehooks,
extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail, extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail,
szind_b, slab_b); szind_b, slab_b);
extent_unlock_edata2(tsdn, edata, trail); emap_unlock_edata2(tsdn, &emap_global, edata, trail);
return trail; return trail;
label_error_c: label_error_c:
extent_unlock_edata2(tsdn, edata, trail); emap_unlock_edata2(tsdn, &emap_global, edata, trail);
label_error_b: label_error_b:
edata_cache_put(tsdn, edata_cache, trail); edata_cache_put(tsdn, edata_cache, trail);
label_error_a: label_error_a:
@ -1523,19 +1417,19 @@ extent_merge_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_cache_t *edata_cache,
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_leaf_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; rtree_leaf_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, a, true, false, &a_elm_a, emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, a, true,
&a_elm_b); false, &a_elm_a, &a_elm_b);
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true, false, &b_elm_a, emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, b, true,
&b_elm_b); false, &b_elm_a, &b_elm_b);
extent_lock_edata2(tsdn, a, b); emap_lock_edata2(tsdn, &emap_global, a, b);
if (a_elm_b != NULL) { if (a_elm_b != NULL) {
rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL, rtree_leaf_elm_write(tsdn, &emap_global.rtree, a_elm_b, NULL,
SC_NSIZES, false); SC_NSIZES, false);
} }
if (b_elm_b != NULL) { if (b_elm_b != NULL) {
rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL, rtree_leaf_elm_write(tsdn, &emap_global.rtree, b_elm_a, NULL,
SC_NSIZES, false); SC_NSIZES, false);
} else { } else {
b_elm_b = b_elm_a; b_elm_b = b_elm_a;
@ -1550,7 +1444,7 @@ extent_merge_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_cache_t *edata_cache,
extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES, extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES,
false); false);
extent_unlock_edata2(tsdn, a, b); emap_unlock_edata2(tsdn, &emap_global, a, b);
edata_cache_put(tsdn, edata_cache, b); edata_cache_put(tsdn, edata_cache, b);
@ -1567,15 +1461,6 @@ bool
extent_boot(void) { extent_boot(void) {
assert(sizeof(slab_data_t) >= sizeof(e_prof_info_t)); assert(sizeof(slab_data_t) >= sizeof(e_prof_info_t));
if (rtree_new(&extents_rtree, true)) {
return true;
}
if (mutex_pool_init(&extent_mutex_pool, "extent_mutex_pool",
WITNESS_RANK_EXTENT_POOL)) {
return true;
}
if (have_dss) { if (have_dss) {
extent_dss_boot(); extent_dss_boot();
} }

View File

@ -6,6 +6,7 @@
#include "jemalloc/internal/atomic.h" #include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/buf_writer.h" #include "jemalloc/internal/buf_writer.h"
#include "jemalloc/internal/ctl.h" #include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/hook.h" #include "jemalloc/internal/hook.h"
@ -1571,6 +1572,9 @@ malloc_init_hard_a0_locked() {
if (base_boot(TSDN_NULL)) { if (base_boot(TSDN_NULL)) {
return true; return true;
} }
if (emap_init(&emap_global)) {
return true;
}
if (extent_boot()) { if (extent_boot()) {
return true; return true;
} }
@ -2565,7 +2569,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES); assert(alloc_ctx.szind != SC_NSIZES);
@ -2619,15 +2623,16 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
alloc_ctx_t dbg_ctx; alloc_ctx_t dbg_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), rtree_szind_slab_read(tsd_tsdn(tsd),
&extents_rtree, rtree_ctx, (uintptr_t)ptr, &emap_global.rtree, rtree_ctx,
true, &dbg_ctx.szind, &dbg_ctx.slab); (uintptr_t)ptr, true, &dbg_ctx.szind,
&dbg_ctx.slab);
assert(dbg_ctx.szind == ctx->szind); assert(dbg_ctx.szind == ctx->szind);
assert(dbg_ctx.slab == ctx->slab); assert(dbg_ctx.slab == ctx->slab);
} }
} else if (opt_prof) { } else if (opt_prof) {
ctx = &alloc_ctx; ctx = &alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree,
rtree_ctx, (uintptr_t)ptr, true, &ctx->szind, rtree_ctx, (uintptr_t)ptr, true, &ctx->szind,
&ctx->slab); &ctx->slab);
/* Small alloc may have !slab (sampled). */ /* Small alloc may have !slab (sampled). */
@ -2699,7 +2704,8 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
bool slab; bool slab;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd),
&extents_rtree, rtree_ctx, (uintptr_t)ptr, &szind, &slab); &emap_global.rtree, rtree_ctx, (uintptr_t)ptr, &szind,
&slab);
/* Note: profiled objects will have alloc_ctx.slab set */ /* Note: profiled objects will have alloc_ctx.slab set */
if (unlikely(!res || !slab)) { if (unlikely(!res || !slab)) {
@ -3142,7 +3148,7 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES); assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind); old_usize = sz_index2size(alloc_ctx.szind);
@ -3421,7 +3427,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, rtree_szind_slab_read(tsd_tsdn(tsd), &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES); assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind); old_usize = sz_index2size(alloc_ctx.szind);

View File

@ -179,7 +179,7 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
szind_t szind = sz_size2index(usize); szind_t szind = sz_size2index(usize);
edata_szind_set(edata, szind); edata_szind_set(edata, szind);
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, rtree_szind_slab_update(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)edata_addr_get(edata), szind, false); (uintptr_t)edata_addr_get(edata), szind, false);
if (config_stats && new_mapping) { if (config_stats && new_mapping) {

View File

@ -129,7 +129,7 @@ tbin_edatas_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind,
size_t sz_sum = binind * nflush; size_t sz_sum = binind * nflush;
void **bottom_item = cache_bin_bottom_item_get(tbin, binind); void **bottom_item = cache_bin_bottom_item_get(tbin, binind);
for (unsigned i = 0 ; i < nflush; i++) { for (unsigned i = 0 ; i < nflush; i++) {
rtree_edata_szind_read(tsdn, &extents_rtree, rtree_edata_szind_read(tsdn, &emap_global.rtree,
rtree_ctx, (uintptr_t)*(bottom_item - i), true, rtree_ctx, (uintptr_t)*(bottom_item - i), true,
&edatas[i], &szind); &edatas[i], &szind);
sz_sum -= szind; sz_sum -= szind;

View File

@ -65,7 +65,7 @@ vsalloc(tsdn_t *tsdn, const void *ptr) {
edata_t *edata; edata_t *edata;
szind_t szind; szind_t szind;
if (rtree_edata_szind_read(tsdn, &extents_rtree, rtree_ctx, if (rtree_edata_szind_read(tsdn, &emap_global.rtree, rtree_ctx,
(uintptr_t)ptr, false, &edata, &szind)) { (uintptr_t)ptr, false, &edata, &szind)) {
return 0; return 0;
} }