project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix/refactor tcaches synchronization.

Browse Source 
Synchronize tcaches with tcaches_mtx rather than ctl_mtx.  Add missing
synchronization for tcache flushing.  This bug was introduced by
1cb181ed63 (Implement explicit tcache
support.), which was first released in 4.0.0.
This commit is contained in:
Jason Evans 2017-01-29 21:32:39 -08:00
parent d0e93ada51
commit 1b6e43507e
6 changed files with 102 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/jemalloc/internal/private_symbols.txt
View File

@ -469,6 +469,9 @@ tcache_flush
tcache_get
tcache_get_hard
tcache_maxclass
tcache_prefork
tcache_postfork_child
tcache_postfork_parent
tcache_salloc
tcache_stats_merge
tcaches

3
include/jemalloc/internal/tcache_externs.h
View File

@ -43,5 +43,8 @@ bool tcaches_create(tsd_t *tsd, unsigned *r_ind);
void tcaches_flush(tsd_t *tsd, unsigned ind);
void tcaches_destroy(tsd_t *tsd, unsigned ind);
bool tcache_boot(tsdn_t *tsdn);
void tcache_prefork(tsdn_t *tsdn);
void tcache_postfork_parent(tsdn_t *tsdn);
void tcache_postfork_child(tsdn_t *tsdn);
#endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */

25
include/jemalloc/internal/witness_types.h
View File

@ -17,21 +17,22 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
#define WITNESS_RANK_INIT 1U
#define WITNESS_RANK_CTL 1U
#define WITNESS_RANK_ARENAS 2U
#define WITNESS_RANK_TCACHES 2U
#define WITNESS_RANK_ARENAS 3U
#define WITNESS_RANK_PROF_DUMP 3U
#define WITNESS_RANK_PROF_BT2GCTX 4U
#define WITNESS_RANK_PROF_TDATAS 5U
#define WITNESS_RANK_PROF_TDATA 6U
#define WITNESS_RANK_PROF_GCTX 7U
#define WITNESS_RANK_PROF_DUMP 4U
#define WITNESS_RANK_PROF_BT2GCTX 5U
#define WITNESS_RANK_PROF_TDATAS 6U
#define WITNESS_RANK_PROF_TDATA 7U
#define WITNESS_RANK_PROF_GCTX 8U
#define WITNESS_RANK_ARENA 8U
#define WITNESS_RANK_ARENA_EXTENTS 9U
#define WITNESS_RANK_ARENA_EXTENT_CACHE 10
#define WITNESS_RANK_ARENA 9U
#define WITNESS_RANK_ARENA_EXTENTS 10U
#define WITNESS_RANK_ARENA_EXTENT_CACHE 11U
#define WITNESS_RANK_RTREE_ELM 11U
#define WITNESS_RANK_RTREE 12U
#define WITNESS_RANK_BASE 13U
#define WITNESS_RANK_RTREE_ELM 12U
#define WITNESS_RANK_RTREE 13U
#define WITNESS_RANK_BASE 14U
#define WITNESS_RANK_LEAF 0xffffffffU
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF

2
src/ctl.c
View File

@ -1477,7 +1477,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
return ENOENT;
}
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
READONLY();
if (tcaches_create(tsd, &tcache_ind)) {
ret = EFAULT;
@ -1487,7 +1486,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = 0;
label_return:
malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
return ret;
}

3
src/jemalloc.c
View File

@ -2765,6 +2765,7 @@ _malloc_prefork(void)
witness_prefork(tsd);
/* Acquire all mutexes in a safe order. */
ctl_prefork(tsd_tsdn(tsd));
tcache_prefork(tsd_tsdn(tsd));
malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
prof_prefork0(tsd_tsdn(tsd));
for (i = 0; i < 3; i++) {
@ -2825,6 +2826,7 @@ _malloc_postfork(void)
}
prof_postfork_parent(tsd_tsdn(tsd));
malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
tcache_postfork_parent(tsd_tsdn(tsd));
ctl_postfork_parent(tsd_tsdn(tsd));
}
@ -2848,6 +2850,7 @@ jemalloc_postfork_child(void) {
}
prof_postfork_child(tsd_tsdn(tsd));
malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
tcache_postfork_child(tsd_tsdn(tsd));
ctl_postfork_child(tsd_tsdn(tsd));
}

96
src/tcache.c
View File

@ -21,6 +21,9 @@ static unsigned tcaches_past;
/* Head of singly linked list tracking available tcaches elements. */
static tcaches_t *tcaches_avail;
/* Protects tcaches{,_past,_avail}. */
static malloc_mutex_t tcaches_mtx;
/******************************************************************************/
size_t
@ -422,32 +425,56 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
}
}
bool
tcaches_create(tsd_t *tsd, unsigned *r_ind) {
arena_t *arena;
tcache_t *tcache;
tcaches_t *elm;
static bool
tcaches_create_prep(tsd_t *tsd) {
bool err;
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcaches == NULL) {
tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *)
* (MALLOCX_TCACHE_MAX+1), CACHELINE);
if (tcaches == NULL) {
return true;
err = true;
goto label_return;
}
}
if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
return true;
}
arena = arena_ichoose(tsd, NULL);
if (unlikely(arena == NULL)) {
return true;
}
tcache = tcache_create(tsd_tsdn(tsd), arena);
if (tcache == NULL) {
return true;
err = true;
goto label_return;
}
err = false;
label_return:
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
return err;
}
bool
tcaches_create(tsd_t *tsd, unsigned *r_ind) {
witness_assert_depth(tsd_tsdn(tsd), 0);
bool err;
if (tcaches_create_prep(tsd)) {
err = true;
goto label_return;
}
arena_t *arena = arena_ichoose(tsd, NULL);
if (unlikely(arena == NULL)) {
err = true;
goto label_return;
}
tcache_t *tcache = tcache_create(tsd_tsdn(tsd), arena);
if (tcache == NULL) {
err = true;
goto label_return;
}
tcaches_t *elm;
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcaches_avail != NULL) {
elm = tcaches_avail;
tcaches_avail = tcaches_avail->next;
@ -459,12 +486,18 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) {
*r_ind = tcaches_past;
tcaches_past++;
}
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
return false;
err = false;
label_return:
witness_assert_depth(tsd_tsdn(tsd), 0);
return err;
}
static void
tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) {
malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
if (elm->tcache == NULL) {
return;
}
@ -474,19 +507,25 @@ tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) {
void
tcaches_flush(tsd_t *tsd, unsigned ind) {
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcaches_elm_flush(tsd, &tcaches[ind]);
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
}
void
tcaches_destroy(tsd_t *tsd, unsigned ind) {
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcaches_t *elm = &tcaches[ind];
tcaches_elm_flush(tsd, elm);
elm->next = tcaches_avail;
tcaches_avail = elm;
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
}
bool
tcache_boot(tsdn_t *tsdn) {
cassert(config_tcache);
unsigned i;
/* If necessary, clamp opt_lg_tcache_max. */
@ -497,6 +536,10 @@ tcache_boot(tsdn_t *tsdn) {
tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
}
if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES)) {
return true;
}
nhbins = size2index(tcache_maxclass) + 1;
/* Initialize tcache_bin_info. */
@ -527,3 +570,24 @@ tcache_boot(tsdn_t *tsdn) {
return false;
}
void
tcache_prefork(tsdn_t *tsdn) {
if (!config_prof && opt_tcache) {
malloc_mutex_prefork(tsdn, &tcaches_mtx);
}
}
void
tcache_postfork_parent(tsdn_t *tsdn) {
if (!config_prof && opt_tcache) {
malloc_mutex_postfork_parent(tsdn, &tcaches_mtx);
}
}
void
tcache_postfork_child(tsdn_t *tsdn) {
if (!config_prof && opt_tcache) {
malloc_mutex_postfork_child(tsdn, &tcaches_mtx);
}
}