project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix/refactor tcaches synchronization.

Browse Source 
Synchronize tcaches with tcaches_mtx rather than ctl_mtx.  Add missing
synchronization for tcache flushing.  This bug was introduced by
1cb181ed63 (Implement explicit tcache
support.), which was first released in 4.0.0.
This commit is contained in:
Jason Evans 2017-01-29 21:32:39 -08:00
parent d0e93ada51
commit 1b6e43507e
6 changed files with 102 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/jemalloc/internal/private_symbols.txt
View File

@ -469,6 +469,9 @@ tcache_flush
tcache_get tcache_get
tcache_get_hard tcache_get_hard
tcache_maxclass tcache_maxclass
tcache_prefork
tcache_postfork_child
tcache_postfork_parent
tcache_salloc tcache_salloc
tcache_stats_merge tcache_stats_merge
tcaches tcaches

3
include/jemalloc/internal/tcache_externs.h
View File

@ -43,5 +43,8 @@ bool tcaches_create(tsd_t *tsd, unsigned *r_ind);
void tcaches_flush(tsd_t *tsd, unsigned ind); void tcaches_flush(tsd_t *tsd, unsigned ind);
void tcaches_destroy(tsd_t *tsd, unsigned ind); void tcaches_destroy(tsd_t *tsd, unsigned ind);
bool tcache_boot(tsdn_t *tsdn); bool tcache_boot(tsdn_t *tsdn);
void tcache_prefork(tsdn_t *tsdn);
void tcache_postfork_parent(tsdn_t *tsdn);
void tcache_postfork_child(tsdn_t *tsdn);
#endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */ #endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */

25
include/jemalloc/internal/witness_types.h
View File

@ -17,21 +17,22 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
#define WITNESS_RANK_INIT 1U #define WITNESS_RANK_INIT 1U
#define WITNESS_RANK_CTL 1U #define WITNESS_RANK_CTL 1U
#define WITNESS_RANK_ARENAS 2U #define WITNESS_RANK_TCACHES 2U
#define WITNESS_RANK_ARENAS 3U
#define WITNESS_RANK_PROF_DUMP 3U #define WITNESS_RANK_PROF_DUMP 4U
#define WITNESS_RANK_PROF_BT2GCTX 4U #define WITNESS_RANK_PROF_BT2GCTX 5U
#define WITNESS_RANK_PROF_TDATAS 5U #define WITNESS_RANK_PROF_TDATAS 6U
#define WITNESS_RANK_PROF_TDATA 6U #define WITNESS_RANK_PROF_TDATA 7U
#define WITNESS_RANK_PROF_GCTX 7U #define WITNESS_RANK_PROF_GCTX 8U
#define WITNESS_RANK_ARENA 8U #define WITNESS_RANK_ARENA 9U
#define WITNESS_RANK_ARENA_EXTENTS 9U #define WITNESS_RANK_ARENA_EXTENTS 10U
#define WITNESS_RANK_ARENA_EXTENT_CACHE 10 #define WITNESS_RANK_ARENA_EXTENT_CACHE 11U
#define WITNESS_RANK_RTREE_ELM 11U #define WITNESS_RANK_RTREE_ELM 12U
#define WITNESS_RANK_RTREE 12U #define WITNESS_RANK_RTREE 13U
#define WITNESS_RANK_BASE 13U #define WITNESS_RANK_BASE 14U
#define WITNESS_RANK_LEAF 0xffffffffU #define WITNESS_RANK_LEAF 0xffffffffU
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF #define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF

2
src/ctl.c
View File

@ -1477,7 +1477,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
return ENOENT; return ENOENT;
} }
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
READONLY(); READONLY();
if (tcaches_create(tsd, &tcache_ind)) { if (tcaches_create(tsd, &tcache_ind)) {
ret = EFAULT; ret = EFAULT;
@ -1487,7 +1486,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = 0; ret = 0;
label_return: label_return:
malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
return ret; return ret;
} }

3
src/jemalloc.c
View File

@ -2765,6 +2765,7 @@ _malloc_prefork(void)
witness_prefork(tsd); witness_prefork(tsd);
/* Acquire all mutexes in a safe order. */ /* Acquire all mutexes in a safe order. */
ctl_prefork(tsd_tsdn(tsd)); ctl_prefork(tsd_tsdn(tsd));
tcache_prefork(tsd_tsdn(tsd));
malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
prof_prefork0(tsd_tsdn(tsd)); prof_prefork0(tsd_tsdn(tsd));
for (i = 0; i < 3; i++) { for (i = 0; i < 3; i++) {
@ -2825,6 +2826,7 @@ _malloc_postfork(void)
} }
prof_postfork_parent(tsd_tsdn(tsd)); prof_postfork_parent(tsd_tsdn(tsd));
malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
tcache_postfork_parent(tsd_tsdn(tsd));
ctl_postfork_parent(tsd_tsdn(tsd)); ctl_postfork_parent(tsd_tsdn(tsd));
} }
@ -2848,6 +2850,7 @@ jemalloc_postfork_child(void) {
} }
prof_postfork_child(tsd_tsdn(tsd)); prof_postfork_child(tsd_tsdn(tsd));
malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
tcache_postfork_child(tsd_tsdn(tsd));
ctl_postfork_child(tsd_tsdn(tsd)); ctl_postfork_child(tsd_tsdn(tsd));
} }

96
src/tcache.c
View File

@ -21,6 +21,9 @@ static unsigned tcaches_past;
/* Head of singly linked list tracking available tcaches elements. */ /* Head of singly linked list tracking available tcaches elements. */
static tcaches_t *tcaches_avail; static tcaches_t *tcaches_avail;
/* Protects tcaches{,_past,_avail}. */
static malloc_mutex_t tcaches_mtx;
/******************************************************************************/ /******************************************************************************/
size_t size_t
@ -422,32 +425,56 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
} }
} }
bool static bool
tcaches_create(tsd_t *tsd, unsigned *r_ind) { tcaches_create_prep(tsd_t *tsd) {
arena_t *arena; bool err;
tcache_t *tcache;
tcaches_t *elm; malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcaches == NULL) { if (tcaches == NULL) {
tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *) tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *)
* (MALLOCX_TCACHE_MAX+1), CACHELINE); * (MALLOCX_TCACHE_MAX+1), CACHELINE);
if (tcaches == NULL) { if (tcaches == NULL) {
return true; err = true;
goto label_return;
} }
} }
if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) { if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
return true; err = true;
} goto label_return;
arena = arena_ichoose(tsd, NULL);
if (unlikely(arena == NULL)) {
return true;
}
tcache = tcache_create(tsd_tsdn(tsd), arena);
if (tcache == NULL) {
return true;
} }
err = false;
label_return:
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
return err;
}
bool
tcaches_create(tsd_t *tsd, unsigned *r_ind) {
witness_assert_depth(tsd_tsdn(tsd), 0);
bool err;
if (tcaches_create_prep(tsd)) {
err = true;
goto label_return;
}
arena_t *arena = arena_ichoose(tsd, NULL);
if (unlikely(arena == NULL)) {
err = true;
goto label_return;
}
tcache_t *tcache = tcache_create(tsd_tsdn(tsd), arena);
if (tcache == NULL) {
err = true;
goto label_return;
}
tcaches_t *elm;
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcaches_avail != NULL) { if (tcaches_avail != NULL) {
elm = tcaches_avail; elm = tcaches_avail;
tcaches_avail = tcaches_avail->next; tcaches_avail = tcaches_avail->next;
@ -459,12 +486,18 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) {
*r_ind = tcaches_past; *r_ind = tcaches_past;
tcaches_past++; tcaches_past++;
} }
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
return false; err = false;
label_return:
witness_assert_depth(tsd_tsdn(tsd), 0);
return err;
} }
static void static void
tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) { tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) {
malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
if (elm->tcache == NULL) { if (elm->tcache == NULL) {
return; return;
} }
@ -474,19 +507,25 @@ tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) {
void void
tcaches_flush(tsd_t *tsd, unsigned ind) { tcaches_flush(tsd_t *tsd, unsigned ind) {
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcaches_elm_flush(tsd, &tcaches[ind]); tcaches_elm_flush(tsd, &tcaches[ind]);
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
} }
void void
tcaches_destroy(tsd_t *tsd, unsigned ind) { tcaches_destroy(tsd_t *tsd, unsigned ind) {
malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcaches_t *elm = &tcaches[ind]; tcaches_t *elm = &tcaches[ind];
tcaches_elm_flush(tsd, elm); tcaches_elm_flush(tsd, elm);
elm->next = tcaches_avail; elm->next = tcaches_avail;
tcaches_avail = elm; tcaches_avail = elm;
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
} }
bool bool
tcache_boot(tsdn_t *tsdn) { tcache_boot(tsdn_t *tsdn) {
cassert(config_tcache);
unsigned i; unsigned i;
/* If necessary, clamp opt_lg_tcache_max. */ /* If necessary, clamp opt_lg_tcache_max. */
@ -497,6 +536,10 @@ tcache_boot(tsdn_t *tsdn) {
tcache_maxclass = (ZU(1) << opt_lg_tcache_max); tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
} }
if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES)) {
return true;
}
nhbins = size2index(tcache_maxclass) + 1; nhbins = size2index(tcache_maxclass) + 1;
/* Initialize tcache_bin_info. */ /* Initialize tcache_bin_info. */
@ -527,3 +570,24 @@ tcache_boot(tsdn_t *tsdn) {
return false; return false;
} }
void
tcache_prefork(tsdn_t *tsdn) {
if (!config_prof && opt_tcache) {
malloc_mutex_prefork(tsdn, &tcaches_mtx);
}
}
void
tcache_postfork_parent(tsdn_t *tsdn) {
if (!config_prof && opt_tcache) {
malloc_mutex_postfork_parent(tsdn, &tcaches_mtx);
}
}
void
tcache_postfork_child(tsdn_t *tsdn) {
if (!config_prof && opt_tcache) {
malloc_mutex_postfork_child(tsdn, &tcaches_mtx);
}
}