Add/use adaptive spinning.
Add spin_t and spin_{init,adaptive}(), which provide a simple abstraction for adaptive spinning. Adaptively spin during busy waits in bootstrapping and rtree node initialization.
This commit is contained in:
parent
a2539fab95
commit
9737685943
@ -101,6 +101,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
|
|||||||
$(srcroot)src/quarantine.c \
|
$(srcroot)src/quarantine.c \
|
||||||
$(srcroot)src/rtree.c \
|
$(srcroot)src/rtree.c \
|
||||||
$(srcroot)src/stats.c \
|
$(srcroot)src/stats.c \
|
||||||
|
$(srcroot)src/spin.c \
|
||||||
$(srcroot)src/tcache.c \
|
$(srcroot)src/tcache.c \
|
||||||
$(srcroot)src/ticker.c \
|
$(srcroot)src/ticker.c \
|
||||||
$(srcroot)src/tsd.c \
|
$(srcroot)src/tsd.c \
|
||||||
|
@ -369,6 +369,7 @@ typedef unsigned szind_t;
|
|||||||
#include "jemalloc/internal/valgrind.h"
|
#include "jemalloc/internal/valgrind.h"
|
||||||
#include "jemalloc/internal/util.h"
|
#include "jemalloc/internal/util.h"
|
||||||
#include "jemalloc/internal/atomic.h"
|
#include "jemalloc/internal/atomic.h"
|
||||||
|
#include "jemalloc/internal/spin.h"
|
||||||
#include "jemalloc/internal/prng.h"
|
#include "jemalloc/internal/prng.h"
|
||||||
#include "jemalloc/internal/ticker.h"
|
#include "jemalloc/internal/ticker.h"
|
||||||
#include "jemalloc/internal/ckh.h"
|
#include "jemalloc/internal/ckh.h"
|
||||||
@ -401,6 +402,7 @@ typedef unsigned szind_t;
|
|||||||
#include "jemalloc/internal/valgrind.h"
|
#include "jemalloc/internal/valgrind.h"
|
||||||
#include "jemalloc/internal/util.h"
|
#include "jemalloc/internal/util.h"
|
||||||
#include "jemalloc/internal/atomic.h"
|
#include "jemalloc/internal/atomic.h"
|
||||||
|
#include "jemalloc/internal/spin.h"
|
||||||
#include "jemalloc/internal/prng.h"
|
#include "jemalloc/internal/prng.h"
|
||||||
#include "jemalloc/internal/ticker.h"
|
#include "jemalloc/internal/ticker.h"
|
||||||
#include "jemalloc/internal/ckh.h"
|
#include "jemalloc/internal/ckh.h"
|
||||||
@ -502,6 +504,7 @@ void jemalloc_postfork_child(void);
|
|||||||
#include "jemalloc/internal/valgrind.h"
|
#include "jemalloc/internal/valgrind.h"
|
||||||
#include "jemalloc/internal/util.h"
|
#include "jemalloc/internal/util.h"
|
||||||
#include "jemalloc/internal/atomic.h"
|
#include "jemalloc/internal/atomic.h"
|
||||||
|
#include "jemalloc/internal/spin.h"
|
||||||
#include "jemalloc/internal/prng.h"
|
#include "jemalloc/internal/prng.h"
|
||||||
#include "jemalloc/internal/ticker.h"
|
#include "jemalloc/internal/ticker.h"
|
||||||
#include "jemalloc/internal/ckh.h"
|
#include "jemalloc/internal/ckh.h"
|
||||||
@ -534,6 +537,7 @@ void jemalloc_postfork_child(void);
|
|||||||
#include "jemalloc/internal/valgrind.h"
|
#include "jemalloc/internal/valgrind.h"
|
||||||
#include "jemalloc/internal/util.h"
|
#include "jemalloc/internal/util.h"
|
||||||
#include "jemalloc/internal/atomic.h"
|
#include "jemalloc/internal/atomic.h"
|
||||||
|
#include "jemalloc/internal/spin.h"
|
||||||
#include "jemalloc/internal/prng.h"
|
#include "jemalloc/internal/prng.h"
|
||||||
#include "jemalloc/internal/ticker.h"
|
#include "jemalloc/internal/ticker.h"
|
||||||
#include "jemalloc/internal/ckh.h"
|
#include "jemalloc/internal/ckh.h"
|
||||||
|
51
include/jemalloc/internal/spin.h
Normal file
51
include/jemalloc/internal/spin.h
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
/******************************************************************************/
|
||||||
|
#ifdef JEMALLOC_H_TYPES
|
||||||
|
|
||||||
|
typedef struct spin_s spin_t;
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_H_TYPES */
|
||||||
|
/******************************************************************************/
|
||||||
|
#ifdef JEMALLOC_H_STRUCTS
|
||||||
|
|
||||||
|
struct spin_s {
|
||||||
|
unsigned iteration;
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_H_STRUCTS */
|
||||||
|
/******************************************************************************/
|
||||||
|
#ifdef JEMALLOC_H_EXTERNS
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_H_EXTERNS */
|
||||||
|
/******************************************************************************/
|
||||||
|
#ifdef JEMALLOC_H_INLINES
|
||||||
|
|
||||||
|
#ifndef JEMALLOC_ENABLE_INLINE
|
||||||
|
void spin_init(spin_t *spin);
|
||||||
|
void spin_adaptive(spin_t *spin);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_SPIN_C_))
|
||||||
|
JEMALLOC_INLINE void
|
||||||
|
spin_init(spin_t *spin)
|
||||||
|
{
|
||||||
|
|
||||||
|
spin->iteration = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
JEMALLOC_INLINE void
|
||||||
|
spin_adaptive(spin_t *spin)
|
||||||
|
{
|
||||||
|
volatile uint64_t i;
|
||||||
|
|
||||||
|
for (i = 0; i < (KQU(1) << spin->iteration); i++)
|
||||||
|
CPU_SPINWAIT;
|
||||||
|
|
||||||
|
if (spin->iteration < 63)
|
||||||
|
spin->iteration++;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_H_INLINES */
|
||||||
|
/******************************************************************************/
|
||||||
|
|
@ -1283,10 +1283,13 @@ malloc_init_hard_needed(void)
|
|||||||
}
|
}
|
||||||
#ifdef JEMALLOC_THREADED_INIT
|
#ifdef JEMALLOC_THREADED_INIT
|
||||||
if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
|
if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
|
||||||
|
spin_t spinner;
|
||||||
|
|
||||||
/* Busy-wait until the initializing thread completes. */
|
/* Busy-wait until the initializing thread completes. */
|
||||||
|
spin_init(&spinner);
|
||||||
do {
|
do {
|
||||||
malloc_mutex_unlock(TSDN_NULL, &init_lock);
|
malloc_mutex_unlock(TSDN_NULL, &init_lock);
|
||||||
CPU_SPINWAIT;
|
spin_adaptive(&spinner);
|
||||||
malloc_mutex_lock(TSDN_NULL, &init_lock);
|
malloc_mutex_lock(TSDN_NULL, &init_lock);
|
||||||
} while (!malloc_initialized());
|
} while (!malloc_initialized());
|
||||||
return (false);
|
return (false);
|
||||||
|
@ -96,12 +96,15 @@ rtree_node_init(rtree_t *rtree, unsigned level, rtree_node_elm_t **elmp)
|
|||||||
rtree_node_elm_t *node;
|
rtree_node_elm_t *node;
|
||||||
|
|
||||||
if (atomic_cas_p((void **)elmp, NULL, RTREE_NODE_INITIALIZING)) {
|
if (atomic_cas_p((void **)elmp, NULL, RTREE_NODE_INITIALIZING)) {
|
||||||
|
spin_t spinner;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Another thread is already in the process of initializing.
|
* Another thread is already in the process of initializing.
|
||||||
* Spin-wait until initialization is complete.
|
* Spin-wait until initialization is complete.
|
||||||
*/
|
*/
|
||||||
|
spin_init(&spinner);
|
||||||
do {
|
do {
|
||||||
CPU_SPINWAIT;
|
spin_adaptive(&spinner);
|
||||||
node = atomic_read_p((void **)elmp);
|
node = atomic_read_p((void **)elmp);
|
||||||
} while (node == RTREE_NODE_INITIALIZING);
|
} while (node == RTREE_NODE_INITIALIZING);
|
||||||
} else {
|
} else {
|
||||||
|
2
src/spin.c
Normal file
2
src/spin.c
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
#define JEMALLOC_SPIN_C_
|
||||||
|
#include "jemalloc/internal/jemalloc_internal.h"
|
Loading…
Reference in New Issue
Block a user