From 26c792e61a163b38b373023bca2947283dcd1fc8 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Mon, 15 May 2017 15:38:15 -0700 Subject: [PATCH] Allow mutexes to take a lock ordering enum at construction. This lets us specify whether and how mutexes of the same rank are allowed to be acquired. Currently, we only allow two polices (only a single mutex at a given rank at a time, and mutexes acquired in ascending order), but we can plausibly allow more (e.g. the "release uncontended mutexes before blocking"). --- include/jemalloc/internal/mutex_externs.h | 2 +- include/jemalloc/internal/mutex_structs.h | 6 +++-- include/jemalloc/internal/mutex_types.h | 10 ++++++++ src/arena.c | 13 ++++++----- src/base.c | 3 ++- src/ctl.c | 3 ++- src/extent.c | 3 ++- src/jemalloc.c | 6 +++-- src/mutex.c | 28 ++++++++++++++++++++--- src/prof.c | 25 +++++++++++--------- src/rtree.c | 3 ++- src/tcache.c | 3 ++- 12 files changed, 75 insertions(+), 30 deletions(-) diff --git a/include/jemalloc/internal/mutex_externs.h b/include/jemalloc/internal/mutex_externs.h index 8e40cb34..c9a817fb 100644 --- a/include/jemalloc/internal/mutex_externs.h +++ b/include/jemalloc/internal/mutex_externs.h @@ -11,7 +11,7 @@ extern bool isthreaded; #endif bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, - witness_rank_t rank); + witness_rank_t rank, malloc_mutex_lock_order_t lock_order); void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); diff --git a/include/jemalloc/internal/mutex_structs.h b/include/jemalloc/internal/mutex_structs.h index 2691852d..a8b16a16 100644 --- a/include/jemalloc/internal/mutex_structs.h +++ b/include/jemalloc/internal/mutex_structs.h @@ -40,12 +40,14 @@ struct malloc_mutex_s { * memory cost. */ #if !defined(JEMALLOC_DEBUG) - witness_t witness; + witness_t witness; + malloc_mutex_lock_order_t lock_order; #endif }; #if defined(JEMALLOC_DEBUG) - witness_t witness; + witness_t witness; + malloc_mutex_lock_order_t lock_order; #endif }; diff --git a/include/jemalloc/internal/mutex_types.h b/include/jemalloc/internal/mutex_types.h index 5af8d099..65a9938d 100644 --- a/include/jemalloc/internal/mutex_types.h +++ b/include/jemalloc/internal/mutex_types.h @@ -3,6 +3,16 @@ typedef struct malloc_mutex_s malloc_mutex_t; +typedef enum { + /* Can only acquire one mutex of a given witness rank at a time. */ + malloc_mutex_rank_exclusive, + /* + * Can acquire multiple mutexes of the same witness rank, but in + * address-ascending order only. + */ + malloc_mutex_address_ordered +} malloc_mutex_lock_order_t; + /* * Based on benchmark results, a fixed spin with this amount of retries works * well for our critical sections. diff --git a/src/arena.c b/src/arena.c index 42bfc6b1..67e1b2f5 100644 --- a/src/arena.c +++ b/src/arena.c @@ -64,7 +64,7 @@ arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) { } #ifndef JEMALLOC_ATOMIC_U64 if (malloc_mutex_init(&arena_stats->mtx, "arena_stats", - WITNESS_RANK_ARENA_STATS)) { + WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) { return true; } #endif @@ -734,7 +734,8 @@ arena_decay_init(arena_decay_t *decay, extents_t *extents, ssize_t decay_ms, assert(((char *)decay)[i] == 0); } } - if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY)) { + if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY, + malloc_mutex_rank_exclusive)) { return true; } decay->purging = false; @@ -1869,7 +1870,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { ql_new(&arena->tcache_ql); if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql", - WITNESS_RANK_TCACHE_QL)) { + WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) { goto label_error; } } @@ -1901,7 +1902,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { extent_list_init(&arena->large); if (malloc_mutex_init(&arena->large_mtx, "arena_large", - WITNESS_RANK_ARENA_LARGE)) { + WITNESS_RANK_ARENA_LARGE, malloc_mutex_rank_exclusive)) { goto label_error; } @@ -1950,7 +1951,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { extent_avail_new(&arena->extent_avail); if (malloc_mutex_init(&arena->extent_avail_mtx, "extent_avail", - WITNESS_RANK_EXTENT_FREELIST)) { + WITNESS_RANK_EXTENT_FREELIST, malloc_mutex_rank_exclusive)) { goto label_error; } @@ -1958,7 +1959,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { for (i = 0; i < NBINS; i++) { arena_bin_t *bin = &arena->bins[i]; if (malloc_mutex_init(&bin->lock, "arena_bin", - WITNESS_RANK_ARENA_BIN)) { + WITNESS_RANK_ARENA_BIN, malloc_mutex_rank_exclusive)) { goto label_error; } bin->slabcur = NULL; diff --git a/src/base.c b/src/base.c index 3de6e3b0..7502a657 100644 --- a/src/base.c +++ b/src/base.c @@ -238,7 +238,8 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { &gap_size, base_size, base_alignment); base->ind = ind; atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELAXED); - if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE)) { + if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE, + malloc_mutex_rank_exclusive)) { base_unmap(extent_hooks, ind, block, block->size); return NULL; } diff --git a/src/ctl.c b/src/ctl.c index 296e74f2..7f69f151 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1199,7 +1199,8 @@ label_return: bool ctl_boot(void) { - if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL)) { + if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL, + malloc_mutex_rank_exclusive)) { return true; } diff --git a/src/extent.c b/src/extent.c index 1b284535..513d16d5 100644 --- a/src/extent.c +++ b/src/extent.c @@ -195,7 +195,8 @@ ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp) bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, bool delay_coalesce) { - if (malloc_mutex_init(&extents->mtx, "extents", WITNESS_RANK_EXTENTS)) { + if (malloc_mutex_init(&extents->mtx, "extents", WITNESS_RANK_EXTENTS, + malloc_mutex_rank_exclusive)) { return true; } for (unsigned i = 0; i < NPSIZES+1; i++) { diff --git a/src/jemalloc.c b/src/jemalloc.c index 47133edf..56aef5b0 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -235,7 +235,8 @@ _init_init_lock(void) { * doing anything. */ if (!init_lock_initialized) { - malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT); + malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT, + malloc_mutex_rank_exclusive); } init_lock_initialized = true; } @@ -1237,7 +1238,8 @@ malloc_init_hard_a0_locked() { if (tcache_boot(TSDN_NULL)) { return true; } - if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS)) { + if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS, + malloc_mutex_rank_exclusive)) { return true; } /* diff --git a/src/mutex.c b/src/mutex.c index 3eec970f..b15bbf6e 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -138,9 +138,25 @@ malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex) { mutex_prof_data_init(&mutex->prof_data); } +static int +mutex_addr_comp(const witness_t *witness1, void *mutex1, + const witness_t *witness2, void *mutex2) { + assert(mutex1 != NULL); + assert(mutex2 != NULL); + uintptr_t mu1int = (uintptr_t)mutex1; + uintptr_t mu2int = (uintptr_t)mutex2; + if (mu1int < mu2int) { + return -1; + } else if (mu1int == mu2int) { + return 0; + } else { + return 1; + } +} + bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, - witness_rank_t rank) { + witness_rank_t rank, malloc_mutex_lock_order_t lock_order) { mutex_prof_data_init(&mutex->prof_data); #ifdef _WIN32 # if _WIN32_WINNT >= 0x0600 @@ -179,7 +195,13 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, pthread_mutexattr_destroy(&attr); #endif if (config_debug) { - witness_init(&mutex->witness, name, rank, NULL, NULL); + mutex->lock_order = lock_order; + if (lock_order == malloc_mutex_address_ordered) { + witness_init(&mutex->witness, name, rank, + mutex_addr_comp, &mutex); + } else { + witness_init(&mutex->witness, name, rank, NULL, NULL); + } } return false; } @@ -200,7 +222,7 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex) { malloc_mutex_unlock(tsdn, mutex); #else if (malloc_mutex_init(mutex, mutex->witness.name, - mutex->witness.rank)) { + mutex->witness.rank, mutex->lock_order)) { malloc_printf(": Error re-initializing mutex in " "child\n"); if (opt_abort) { diff --git a/src/prof.c b/src/prof.c index 470d926f..18978810 100644 --- a/src/prof.c +++ b/src/prof.c @@ -1754,7 +1754,7 @@ prof_accum_init(tsdn_t *tsdn, prof_accum_t *prof_accum) { #ifndef JEMALLOC_ATOMIC_U64 if (malloc_mutex_init(&prof_accum->mtx, "prof_accum", - WITNESS_RANK_PROF_ACCUM)) { + WITNESS_RANK_PROF_ACCUM, malloc_mutex_rank_exclusive)) { return true; } prof_accum->accumbytes = 0; @@ -2289,20 +2289,21 @@ prof_boot2(tsd_t *tsd) { prof_active = opt_prof_active; if (malloc_mutex_init(&prof_active_mtx, "prof_active", - WITNESS_RANK_PROF_ACTIVE)) { + WITNESS_RANK_PROF_ACTIVE, malloc_mutex_rank_exclusive)) { return true; } prof_gdump_val = opt_prof_gdump; if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump", - WITNESS_RANK_PROF_GDUMP)) { + WITNESS_RANK_PROF_GDUMP, malloc_mutex_rank_exclusive)) { return true; } prof_thread_active_init = opt_prof_thread_active_init; if (malloc_mutex_init(&prof_thread_active_init_mtx, "prof_thread_active_init", - WITNESS_RANK_PROF_THREAD_ACTIVE_INIT)) { + WITNESS_RANK_PROF_THREAD_ACTIVE_INIT, + malloc_mutex_rank_exclusive)) { return true; } @@ -2311,28 +2312,28 @@ prof_boot2(tsd_t *tsd) { return true; } if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx", - WITNESS_RANK_PROF_BT2GCTX)) { + WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) { return true; } tdata_tree_new(&tdatas); if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas", - WITNESS_RANK_PROF_TDATAS)) { + WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) { return true; } next_thr_uid = 0; if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid", - WITNESS_RANK_PROF_NEXT_THR_UID)) { + WITNESS_RANK_PROF_NEXT_THR_UID, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq", - WITNESS_RANK_PROF_DUMP_SEQ)) { + WITNESS_RANK_PROF_DUMP_SEQ, malloc_mutex_rank_exclusive)) { return true; } if (malloc_mutex_init(&prof_dump_mtx, "prof_dump", - WITNESS_RANK_PROF_DUMP)) { + WITNESS_RANK_PROF_DUMP, malloc_mutex_rank_exclusive)) { return true; } @@ -2352,7 +2353,8 @@ prof_boot2(tsd_t *tsd) { } for (i = 0; i < PROF_NCTX_LOCKS; i++) { if (malloc_mutex_init(&gctx_locks[i], "prof_gctx", - WITNESS_RANK_PROF_GCTX)) { + WITNESS_RANK_PROF_GCTX, + malloc_mutex_rank_exclusive)) { return true; } } @@ -2365,7 +2367,8 @@ prof_boot2(tsd_t *tsd) { } for (i = 0; i < PROF_NTDATA_LOCKS; i++) { if (malloc_mutex_init(&tdata_locks[i], "prof_tdata", - WITNESS_RANK_PROF_TDATA)) { + WITNESS_RANK_PROF_TDATA, + malloc_mutex_rank_exclusive)) { return true; } } diff --git a/src/rtree.c b/src/rtree.c index 62df0143..6d4a71a2 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -18,7 +18,8 @@ rtree_new(rtree_t *rtree, bool zeroed) { assert(zeroed); #endif - if (malloc_mutex_init(&rtree->init_lock, "rtree", WITNESS_RANK_RTREE)) { + if (malloc_mutex_init(&rtree->init_lock, "rtree", WITNESS_RANK_RTREE, + malloc_mutex_rank_exclusive)) { return true; } diff --git a/src/tcache.c b/src/tcache.c index ee5e816f..d9f5e7cb 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -649,7 +649,8 @@ tcache_boot(tsdn_t *tsdn) { tcache_maxclass = (ZU(1) << opt_lg_tcache_max); } - if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES)) { + if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES, + malloc_mutex_rank_exclusive)) { return true; }