project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement tsd.

Browse Source 
Implement tsd, which is a TLS/TSD abstraction that uses one or both
internally.  Modify bootstrapping such that no tsd's are utilized until
allocation is safe.

Remove malloc_[v]tprintf(), and use malloc_snprintf() instead.

Fix %p argument size handling in malloc_vsnprintf().

Fix a long-standing statistics-related bug in the "thread.arena"
mallctl that could cause crashes due to linked list corruption.
This commit is contained in:
Jason Evans
2012-03-21 18:33:03 -07:00
parent 154829d256
commit cd9a1346e9
19 changed files with 705 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/chunk.c
View File

@@ -100,7 +100,7 @@ chunk_dealloc(void *chunk, size_t size, bool unmap)
}
bool
chunk_boot(void)
chunk_boot0(void)
{
/* Set variables according to the value of opt_lg_chunk. */
@@ -114,8 +114,6 @@ chunk_boot(void)
return (true);
memset(&stats_chunks, 0, sizeof(chunk_stats_t));
}
if (chunk_mmap_boot())
return (true);
if (config_dss && chunk_dss_boot())
return (true);
if (config_ivsalloc) {
@@ -127,3 +125,13 @@ chunk_boot(void)
return (false);
}
bool
chunk_boot1(void)
{
if (chunk_mmap_boot())
return (true);
return (false);
}

40
src/chunk_mmap.c
View File

@@ -8,20 +8,9 @@
* Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
* potentially avoid some system calls.
*/
#ifdef JEMALLOC_TLS
static __thread bool mmap_unaligned_tls
JEMALLOC_ATTR(tls_model("initial-exec"));
#define MMAP_UNALIGNED_GET() mmap_unaligned_tls
#define MMAP_UNALIGNED_SET(v) do { \
mmap_unaligned_tls = (v); \
} while (0)
#else
static pthread_key_t mmap_unaligned_tsd;
#define MMAP_UNALIGNED_GET() ((bool)pthread_getspecific(mmap_unaligned_tsd))
#define MMAP_UNALIGNED_SET(v) do { \
pthread_setspecific(mmap_unaligned_tsd, (void *)(v)); \
} while (0)
#endif
malloc_tsd_data(static, mmap_unaligned, bool, false)
malloc_tsd_funcs(JEMALLOC_INLINE, mmap_unaligned, bool, false,
malloc_tsd_no_cleanup)
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
@@ -128,8 +117,10 @@ chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve)
* the next chunk_alloc_mmap() execution tries the fast allocation
* method.
*/
if (unaligned == false)
MMAP_UNALIGNED_SET(false);
if (unaligned == false && mmap_unaligned_booted) {
bool mu = false;
mmap_unaligned_tsd_set(&mu);
}
return (ret);
}
@@ -167,7 +158,7 @@ chunk_alloc_mmap_internal(size_t size, bool noreserve)
* fast method next time.
*/
if (MMAP_UNALIGNED_GET() == false) {
if (mmap_unaligned_booted && *mmap_unaligned_tsd_get() == false) {
size_t offset;
ret = pages_map(NULL, size, noreserve);
@@ -176,7 +167,8 @@ chunk_alloc_mmap_internal(size_t size, bool noreserve)
offset = CHUNK_ADDR2OFFSET(ret);
if (offset != 0) {
MMAP_UNALIGNED_SET(true);
bool mu = true;
mmap_unaligned_tsd_set(&mu);
/* Try to extend chunk boundary. */
if (pages_map((void *)((uintptr_t)ret + size),
chunksize - offset, noreserve) == NULL) {
@@ -225,11 +217,15 @@ bool
chunk_mmap_boot(void)
{
#ifndef JEMALLOC_TLS
if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
malloc_write("<jemalloc>: Error in pthread_key_create()\n");
/*
* XXX For the non-TLS implementation of tsd, the first access from
* each thread causes memory allocation. The result is a bootstrapping
* problem for this particular use case, so for now just disable it by
* leaving it in an unbooted state.
*/
#ifdef JEMALLOC_TLS
if (mmap_unaligned_tsd_boot())
return (true);
}
#endif
return (false);

42
src/ctl.c
View File

@@ -978,13 +978,13 @@ thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
VOID();
tcache = TCACHE_GET();
if (tcache == NULL) {
if ((tcache = *tcache_tsd_get()) == NULL) {
ret = 0;
goto RETURN;
}
tcache_destroy(tcache);
TCACHE_SET(NULL);
tcache = NULL;
tcache_tsd_set(&tcache);
ret = 0;
RETURN:
@@ -1012,23 +1012,26 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
/* Initialize arena if necessary. */
malloc_mutex_lock(&arenas_lock);
if ((arena = arenas[newind]) == NULL)
arena = arenas_extend(newind);
arenas[oldind]->nthreads--;
arenas[newind]->nthreads++;
malloc_mutex_unlock(&arenas_lock);
if (arena == NULL) {
if ((arena = arenas[newind]) == NULL && (arena =
arenas_extend(newind)) == NULL) {
malloc_mutex_unlock(&arenas_lock);
ret = EAGAIN;
goto RETURN;
}
assert(arena == arenas[newind]);
arenas[oldind]->nthreads--;
arenas[newind]->nthreads++;
malloc_mutex_unlock(&arenas_lock);
/* Set new arena association. */
ARENA_SET(arena);
if (config_tcache) {
tcache_t *tcache = TCACHE_GET();
if (tcache != NULL)
tcache->arena = arena;
tcache_t *tcache;
if ((tcache = *tcache_tsd_get()) != NULL) {
tcache_arena_dissociate(tcache);
tcache_arena_associate(tcache, arena);
}
}
arenas_tsd_set(&arena);
}
ret = 0;
@@ -1036,11 +1039,14 @@ RETURN:
return (ret);
}
CTL_RO_NL_CGEN(config_stats, thread_allocated, ALLOCATED_GET(), uint64_t)
CTL_RO_NL_CGEN(config_stats, thread_allocatedp, ALLOCATEDP_GET(), uint64_t *)
CTL_RO_NL_CGEN(config_stats, thread_deallocated, DEALLOCATED_GET(), uint64_t)
CTL_RO_NL_CGEN(config_stats, thread_deallocatedp, DEALLOCATEDP_GET(),
uint64_t *)
CTL_RO_NL_CGEN(config_stats, thread_allocated,
thread_allocated_tsd_get()->allocated, uint64_t)
CTL_RO_NL_CGEN(config_stats, thread_allocatedp,
&thread_allocated_tsd_get()->allocated, uint64_t *)
CTL_RO_NL_CGEN(config_stats, thread_deallocated,
thread_allocated_tsd_get()->deallocated, uint64_t)
CTL_RO_NL_CGEN(config_stats, thread_deallocatedp,
&thread_allocated_tsd_get()->deallocated, uint64_t *)
/******************************************************************************/

171
src/jemalloc.c
View File

@@ -4,36 +4,9 @@
/******************************************************************************/
/* Data. */
malloc_mutex_t arenas_lock;
arena_t **arenas;
unsigned narenas;
pthread_key_t arenas_tsd;
#ifdef JEMALLOC_TLS
__thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec"));
#endif
#ifdef JEMALLOC_TLS
__thread thread_allocated_t thread_allocated_tls;
#endif
pthread_key_t thread_allocated_tsd;
/* Set to true once the allocator has been initialized. */
static bool malloc_initialized = false;
/* Used to let the initializing thread recursively allocate. */
static pthread_t malloc_initializer = (unsigned long)0;
/* Used to avoid initialization races. */
static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
#ifdef DYNAMIC_PAGE_SHIFT
size_t pagesize;
size_t pagesize_mask;
size_t lg_pagesize;
#endif
unsigned ncpus;
malloc_tsd_data(, arenas, arena_t *, NULL)
malloc_tsd_data(, thread_allocated, thread_allocated_t,
THREAD_ALLOCATED_INITIALIZER)
/* Runtime configuration options. */
const char *je_malloc_conf JEMALLOC_ATTR(visibility("default"));
@@ -52,15 +25,32 @@ bool opt_xmalloc = false;
bool opt_zero = false;
size_t opt_narenas = 0;
#ifdef DYNAMIC_PAGE_SHIFT
size_t pagesize;
size_t pagesize_mask;
size_t lg_pagesize;
#endif
unsigned ncpus;
malloc_mutex_t arenas_lock;
arena_t **arenas;
unsigned narenas;
/* Set to true once the allocator has been initialized. */
static bool malloc_initialized = false;
/* Used to let the initializing thread recursively allocate. */
static pthread_t malloc_initializer = (unsigned long)0;
/* Used to avoid initialization races. */
static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static void stats_print_atexit(void);
static unsigned malloc_ncpus(void);
static void arenas_cleanup(void *arg);
#ifndef JEMALLOC_TLS
static void thread_allocated_cleanup(void *arg);
#endif
static bool malloc_conf_next(char const **opts_p, char const **k_p,
size_t *klen_p, char const **v_p, size_t *vlen_p);
static void malloc_conf_error(const char *msg, const char *k, size_t klen,
@@ -156,7 +146,7 @@ choose_arena_hard(void)
malloc_mutex_unlock(&arenas_lock);
}
ARENA_SET(ret);
arenas_tsd_set(&ret);
return (ret);
}
@@ -197,26 +187,6 @@ stats_print_atexit(void)
je_malloc_stats_print(NULL, NULL, NULL);
}
thread_allocated_t *
thread_allocated_get_hard(void)
{
thread_allocated_t *thread_allocated = (thread_allocated_t *)
imalloc(sizeof(thread_allocated_t));
if (thread_allocated == NULL) {
static thread_allocated_t static_thread_allocated = {0, 0};
malloc_write("<jemalloc>: Error allocating TSD;"
" mallctl(\"thread.{de,}allocated[p]\", ...)"
" will be inaccurate\n");
if (opt_abort)
abort();
return (&static_thread_allocated);
}
pthread_setspecific(thread_allocated_tsd, thread_allocated);
thread_allocated->allocated = 0;
thread_allocated->deallocated = 0;
return (thread_allocated);
}
/*
* End miscellaneous support functions.
*/
@@ -241,32 +211,16 @@ malloc_ncpus(void)
return (ret);
}
static void
void
arenas_cleanup(void *arg)
{
arena_t *arena = (arena_t *)arg;
arena_t *arena = *(arena_t **)arg;
malloc_mutex_lock(&arenas_lock);
arena->nthreads--;
malloc_mutex_unlock(&arenas_lock);
}
#ifndef JEMALLOC_TLS
static void
thread_allocated_cleanup(void *arg)
{
uint64_t *allocated = (uint64_t *)arg;
if (allocated != NULL)
idalloc(allocated);
}
#endif
/*
* FreeBSD's pthreads implementation calls malloc(3), so the malloc
* implementation has to take pains to avoid infinite recursion during
* initialization.
*/
static inline bool
malloc_init(void)
{
@@ -604,6 +558,7 @@ malloc_init_hard(void)
}
#endif
malloc_tsd_boot();
if (config_prof)
prof_boot0();
@@ -631,7 +586,7 @@ malloc_init_hard(void)
}
}
if (chunk_boot()) {
if (chunk_boot0()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
@@ -646,7 +601,7 @@ malloc_init_hard(void)
arena_boot();
if (config_tcache && tcache_boot()) {
if (config_tcache && tcache_boot0()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
@@ -656,23 +611,9 @@ malloc_init_hard(void)
return (true);
}
#ifndef JEMALLOC_TLS
/* Initialize allocation counters before any allocations can occur. */
if (config_stats && pthread_key_create(&thread_allocated_tsd,
thread_allocated_cleanup) != 0) {
malloc_mutex_unlock(&init_lock);
return (true);
}
#endif
if (malloc_mutex_init(&arenas_lock))
return (true);
if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) {
malloc_mutex_unlock(&init_lock);
return (true);
}
/*
* Create enough scaffolding to allow recursive allocation in
* malloc_ncpus().
@@ -691,25 +632,38 @@ malloc_init_hard(void)
return (true);
}
/*
* Assign the initial arena to the initial thread, in order to avoid
* spurious creation of an extra arena if the application switches to
* threaded mode.
*/
ARENA_SET(arenas[0]);
arenas[0]->nthreads++;
/* Initialize allocation counters before any allocations can occur. */
if (config_stats && thread_allocated_tsd_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_prof && prof_boot2()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (arenas_tsd_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_tcache && tcache_boot1()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
/* Get number of CPUs. */
malloc_initializer = pthread_self();
malloc_mutex_unlock(&init_lock);
ncpus = malloc_ncpus();
malloc_mutex_lock(&init_lock);
if (chunk_boot1()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (opt_narenas == 0) {
/*
* For SMP systems, create more than one arena per CPU by
@@ -844,7 +798,7 @@ OOM:
prof_malloc(ret, usize, cnt);
if (config_stats && ret != NULL) {
assert(usize == isalloc(ret));
ALLOCATED_ADD(usize, 0);
thread_allocated_tsd_get()->allocated += usize;
}
return (ret);
}
@@ -939,7 +893,7 @@ imemalign(void **memptr, size_t alignment, size_t size,
RETURN:
if (config_stats && result != NULL) {
assert(usize == isalloc(result));
ALLOCATED_ADD(usize, 0);
thread_allocated_tsd_get()->allocated += usize;
}
if (config_prof && opt_prof && result != NULL)
prof_malloc(result, usize, cnt);
@@ -1044,7 +998,7 @@ RETURN:
prof_malloc(ret, usize, cnt);
if (config_stats && ret != NULL) {
assert(usize == isalloc(ret));
ALLOCATED_ADD(usize, 0);
thread_allocated_tsd_get()->allocated += usize;
}
return (ret);
}
@@ -1173,8 +1127,11 @@ RETURN:
if (config_prof && opt_prof)
prof_realloc(ret, usize, cnt, old_size, old_ctx);
if (config_stats && ret != NULL) {
thread_allocated_t *ta;
assert(usize == isalloc(ret));
ALLOCATED_ADD(usize, old_size);
ta = thread_allocated_tsd_get();
ta->allocated += usize;
ta->deallocated += old_size;
}
return (ret);
}
@@ -1197,7 +1154,7 @@ je_free(void *ptr)
usize = isalloc(ptr);
}
if (config_stats)
ALLOCATED_ADD(0, usize);
thread_allocated_tsd_get()->deallocated += usize;
idalloc(ptr);
}
}
@@ -1412,7 +1369,7 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
*ptr = p;
if (config_stats) {
assert(usize == isalloc(p));
ALLOCATED_ADD(usize, 0);
thread_allocated_tsd_get()->allocated += usize;
}
return (ALLOCM_SUCCESS);
OOM:
@@ -1502,8 +1459,12 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
}
*ptr = q;
if (config_stats)
ALLOCATED_ADD(usize, old_size);
if (config_stats) {
thread_allocated_t *ta;
ta = thread_allocated_tsd_get();
ta->allocated += usize;
ta->deallocated += old_size;
}
return (ALLOCM_SUCCESS);
ERR:
if (no_move)
@@ -1556,7 +1517,7 @@ je_dallocm(void *ptr, int flags)
prof_free(ptr, usize);
}
if (config_stats)
ALLOCATED_ADD(0, usize);
thread_allocated_tsd_get()->deallocated += usize;
idalloc(ptr);
return (ALLOCM_SUCCESS);

45
src/prof.c
View File

@@ -14,6 +14,8 @@
/******************************************************************************/
/* Data. */
malloc_tsd_data(, prof_tdata, prof_tdata_t *, NULL)
bool opt_prof = false;
bool opt_prof_active = true;
size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
@@ -26,12 +28,6 @@ char opt_prof_prefix[PATH_MAX + 1];
uint64_t prof_interval;
bool prof_promote;
#ifdef JEMALLOC_TLS
__thread prof_tdata_t *prof_tdata_tls
JEMALLOC_ATTR(tls_model("initial-exec"));
#endif
pthread_key_t prof_tdata_tsd;
/*
* Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data
* structure that knows about all backtraces currently captured.
@@ -50,7 +46,7 @@ static uint64_t prof_dump_useq;
* all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since
* it must be locked anyway during dumping.
*/
static char prof_dump_buf[PROF_DUMP_BUF_SIZE];
static char prof_dump_buf[PROF_DUMP_BUFSIZE];
static unsigned prof_dump_buf_end;
static int prof_dump_fd;
@@ -91,7 +87,6 @@ static void prof_fdump(void);
static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1,
size_t *hash2);
static bool prof_bt_keycomp(const void *k1, const void *k2);
static void prof_tdata_cleanup(void *arg);
/******************************************************************************/
@@ -439,7 +434,7 @@ prof_lookup(prof_bt_t *bt)
cassert(config_prof);
prof_tdata = PROF_TCACHE_GET();
prof_tdata = *prof_tdata_tsd_get();
if (prof_tdata == NULL) {
prof_tdata = prof_tdata_init();
if (prof_tdata == NULL)
@@ -599,16 +594,16 @@ prof_write(bool propagate_err, const char *s)
slen = strlen(s);
while (i < slen) {
/* Flush the buffer if it is full. */
if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE)
if (prof_dump_buf_end == PROF_DUMP_BUFSIZE)
if (prof_flush(propagate_err) && propagate_err)
return (true);
if (prof_dump_buf_end + slen <= PROF_DUMP_BUF_SIZE) {
if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
/* Finish writing. */
n = slen - i;
} else {
/* Write as much of s as will fit. */
n = PROF_DUMP_BUF_SIZE - prof_dump_buf_end;
n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
}
memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
prof_dump_buf_end += n;
@@ -624,10 +619,12 @@ prof_printf(bool propagate_err, const char *format, ...)
{
bool ret;
va_list ap;
char buf[PROF_PRINTF_BUFSIZE];
va_start(ap, format);
ret = prof_write(propagate_err, malloc_vtprintf(format, ap));
malloc_snprintf(buf, sizeof(buf), format, ap);
va_end(ap);
ret = prof_write(propagate_err, buf);
return (ret);
}
@@ -795,11 +792,13 @@ static bool
prof_dump_maps(bool propagate_err)
{
int mfd;
char filename[PATH_MAX + 1];
cassert(config_prof);
mfd = open(malloc_tprintf("/proc/%d/maps", (int)getpid()),
O_RDONLY);
malloc_snprintf(filename, sizeof(filename), "/proc/%d/maps",
(int)getpid());
mfd = open(filename, O_RDONLY);
if (mfd != -1) {
ssize_t nread;
@@ -809,13 +808,13 @@ prof_dump_maps(bool propagate_err)
nread = 0;
do {
prof_dump_buf_end += nread;
if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) {
if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
/* Make space in prof_dump_buf before read(). */
if (prof_flush(propagate_err) && propagate_err)
return (true);
}
nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
PROF_DUMP_BUF_SIZE - prof_dump_buf_end);
PROF_DUMP_BUFSIZE - prof_dump_buf_end);
} while (nread > 0);
close(mfd);
} else
@@ -1098,16 +1097,16 @@ prof_tdata_init(void)
prof_tdata->threshold = 0;
prof_tdata->accum = 0;
PROF_TCACHE_SET(prof_tdata);
prof_tdata_tsd_set(&prof_tdata);
return (prof_tdata);
}
static void
void
prof_tdata_cleanup(void *arg)
{
prof_thr_cnt_t *cnt;
prof_tdata_t *prof_tdata = (prof_tdata_t *)arg;
prof_tdata_t *prof_tdata = *(prof_tdata_t **)arg;
cassert(config_prof);
@@ -1127,7 +1126,8 @@ prof_tdata_cleanup(void *arg)
idalloc(prof_tdata->vec);
idalloc(prof_tdata);
PROF_TCACHE_SET(NULL);
prof_tdata = NULL;
prof_tdata_tsd_set(&prof_tdata);
}
void
@@ -1182,8 +1182,7 @@ prof_boot2(void)
return (true);
if (malloc_mutex_init(&bt2ctx_mtx))
return (true);
if (pthread_key_create(&prof_tdata_tsd, prof_tdata_cleanup)
!= 0) {
if (prof_tdata_tsd_boot()) {
malloc_write(
"<jemalloc>: Error in pthread_key_create()\n");
abort();

103
src/tcache.c
View File

@@ -4,30 +4,16 @@
/******************************************************************************/
/* Data. */
malloc_tsd_data(, tcache, tcache_t *, NULL)
bool opt_tcache = true;
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
tcache_bin_info_t *tcache_bin_info;
static unsigned stack_nelms; /* Total stack elms per tcache. */
/* Map of thread-specific caches. */
#ifdef JEMALLOC_TLS
__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec"));
#endif
/*
* Same contents as tcache, but initialized such that the TSD destructor is
* called when a thread exits, so that the cache can be cleaned up.
*/
pthread_key_t tcache_tsd;
size_t nhbins;
size_t tcache_maxclass;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static void tcache_thread_cleanup(void *arg);
size_t nhbins;
size_t tcache_maxclass;
/******************************************************************************/
@@ -196,6 +182,33 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
tbin->low_water = tbin->ncached;
}
void
tcache_arena_associate(tcache_t *tcache, arena_t *arena)
{
if (config_stats) {
/* Link into list of extant tcaches. */
malloc_mutex_lock(&arena->lock);
ql_elm_new(tcache, link);
ql_tail_insert(&arena->tcache_ql, tcache, link);
malloc_mutex_unlock(&arena->lock);
}
tcache->arena = arena;
}
void
tcache_arena_dissociate(tcache_t *tcache)
{
if (config_stats) {
/* Unlink from list of extant tcaches. */
malloc_mutex_lock(&tcache->arena->lock);
ql_remove(&tcache->arena->tcache_ql, tcache, link);
malloc_mutex_unlock(&tcache->arena->lock);
tcache_stats_merge(tcache, tcache->arena);
}
}
tcache_t *
tcache_create(arena_t *arena)
{
@@ -228,15 +241,8 @@ tcache_create(arena_t *arena)
if (tcache == NULL)
return (NULL);
if (config_stats) {
/* Link into list of extant tcaches. */
malloc_mutex_lock(&arena->lock);
ql_elm_new(tcache, link);
ql_tail_insert(&arena->tcache_ql, tcache, link);
malloc_mutex_unlock(&arena->lock);
}
tcache_arena_associate(tcache, arena);
tcache->arena = arena;
assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
for (i = 0; i < nhbins; i++) {
tcache->tbins[i].lg_fill_div = 1;
@@ -245,7 +251,7 @@ tcache_create(arena_t *arena)
stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
}
TCACHE_SET(tcache);
tcache_tsd_set(&tcache);
return (tcache);
}
@@ -256,13 +262,7 @@ tcache_destroy(tcache_t *tcache)
unsigned i;
size_t tcache_size;
if (config_stats) {
/* Unlink from list of extant tcaches. */
malloc_mutex_lock(&tcache->arena->lock);
ql_remove(&tcache->arena->tcache_ql, tcache, link);
malloc_mutex_unlock(&tcache->arena->lock);
tcache_stats_merge(tcache, tcache->arena);
}
tcache_arena_dissociate(tcache);
for (i = 0; i < NBINS; i++) {
tcache_bin_t *tbin = &tcache->tbins[i];
@@ -323,10 +323,10 @@ tcache_destroy(tcache_t *tcache)
idalloc(tcache);
}
static void
void
tcache_thread_cleanup(void *arg)
{
tcache_t *tcache = (tcache_t *)arg;
tcache_t *tcache = *(tcache_t **)arg;
if (tcache == (void *)(uintptr_t)1) {
/*
@@ -341,11 +341,13 @@ tcache_thread_cleanup(void *arg)
* destructor was called. Reset tcache to 1 in order to
* receive another callback.
*/
TCACHE_SET((uintptr_t)1);
tcache = (tcache_t *)(uintptr_t)1;
tcache_tsd_set(&tcache);
} else if (tcache != NULL) {
assert(tcache != (void *)(uintptr_t)1);
tcache_destroy(tcache);
TCACHE_SET((uintptr_t)1);
tcache = (tcache_t *)(uintptr_t)1;
tcache_tsd_set(&tcache);
}
}
@@ -374,7 +376,7 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
}
bool
tcache_boot(void)
tcache_boot0(void)
{
if (opt_tcache) {
@@ -385,8 +387,8 @@ tcache_boot(void)
* SMALL_MAXCLASS and arena_maxclass are known.
* XXX Can this be done earlier?
*/
if (opt_lg_tcache_max < 0 || (1U <<
opt_lg_tcache_max) < SMALL_MAXCLASS)
if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) <
SMALL_MAXCLASS)
tcache_maxclass = SMALL_MAXCLASS;
else if ((1U << opt_lg_tcache_max) > arena_maxclass)
tcache_maxclass = arena_maxclass;
@@ -416,13 +418,18 @@ tcache_boot(void)
tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
stack_nelms += tcache_bin_info[i].ncached_max;
}
if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) !=
0) {
malloc_write(
"<jemalloc>: Error in pthread_key_create()\n");
abort();
}
}
return (false);
}
bool
tcache_boot1(void)
{
if (opt_tcache) {
if (tcache_tsd_boot())
return (true);
}
return (false);

72
src/tsd.c Normal file
View File

@@ -0,0 +1,72 @@
#define JEMALLOC_TSD_C_
#include "jemalloc/internal/jemalloc_internal.h"
/******************************************************************************/
/* Data. */
static unsigned ncleanups;
static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
/******************************************************************************/
void *
malloc_tsd_malloc(size_t size)
{
/* Avoid choose_arena() in order to dodge bootstrapping issues. */
return arena_malloc_prechosen(arenas[0], size, false);
}
void
malloc_tsd_dalloc(void *wrapper)
{
idalloc(wrapper);
}
void
malloc_tsd_no_cleanup(void *arg)
{
not_reached();
}
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
void
_malloc_thread_cleanup(void)
{
bool pending[ncleanups], again;
unsigned i;
for (i = 0; i < ncleanups; i++)
pending[i] = true;
do {
again = false;
for (i = 0; i < ncleanups; i++) {
if (pending[i]) {
pending[i] = cleanups[i].f(cleanups[i].arg);
if (pending[i])
again = true;
}
}
} while (again);
}
#endif
void
malloc_tsd_cleanup_register(bool (*f)(void *), void *arg)
{
assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX);
cleanups[ncleanups].f = f;
cleanups[ncleanups].arg = arg;
ncleanups++;
}
void
malloc_tsd_boot(void)
{
ncleanups = 0;
}

33
src/util.c
View File

@@ -222,6 +222,9 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
case 'z': \
val = va_arg(ap, size_t); \
break; \
case 'p': /* Synthetic; used for %p. */ \
val = va_arg(ap, uintptr_t); \
break; \
default: not_reached(); \
} \
} while (0)
@@ -410,7 +413,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
uintmax_t val;
char buf[X2S_BUFSIZE];
GET_ARG_NUMERIC(val, len);
GET_ARG_NUMERIC(val, 'p');
s = x2s(val, true, false, buf, &slen);
APPEND_PADDED_S(s, slen, width, left_justify);
f++;
@@ -466,34 +469,11 @@ malloc_snprintf(char *str, size_t size, const char *format, ...)
return (ret);
}
const char *
malloc_vtprintf(const char *format, va_list ap)
{
static __thread char buf[MALLOC_PRINTF_BUFSIZE];
malloc_vsnprintf(buf, sizeof(buf), format, ap);
return (buf);
}
JEMALLOC_ATTR(format(printf, 1, 2))
const char *
malloc_tprintf(const char *format, ...)
{
const char *ret;
va_list ap;
va_start(ap, format);
ret = malloc_vtprintf(format, ap);
va_end(ap);
return (ret);
}
void
malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
const char *format, va_list ap)
{
char buf[MALLOC_PRINTF_BUFSIZE];
if (write_cb == NULL) {
/*
@@ -505,7 +485,8 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
cbopaque = NULL;
}
write_cb(cbopaque, malloc_vtprintf(format, ap));
malloc_vsnprintf(buf, sizeof(buf), format, ap);
write_cb(cbopaque, buf);
}
/*