project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Convert all tsd variables to reside in a single tsd structure.

Browse Source
This commit is contained in:
Jason Evans 2014-09-22 21:09:23 -07:00
parent 42f5955938
commit 5460aa6f66
22 changed files with 1027 additions and 935 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
include/jemalloc/internal/arena.h
View File

@ -419,9 +419,9 @@ extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
#endif
bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero);
void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc);
void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t extra, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc);
dss_prec_t arena_dss_prec_get(arena_t *arena);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive,
@ -485,10 +485,12 @@ unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
const void *ptr);
prof_tctx_t *arena_prof_tctx_get(const void *ptr);
void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache);
void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
bool try_tcache);
size_t arena_salloc(const void *ptr, bool demote);
void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache);
void arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size,
void arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr,
bool try_tcache);
void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
bool try_tcache);
#endif
@ -1053,7 +1055,8 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
}
JEMALLOC_ALWAYS_INLINE void *
arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
bool try_tcache)
{
tcache_t *tcache;
@ -1061,12 +1064,12 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
assert(size <= arena_maxclass);
if (likely(size <= SMALL_MAXCLASS)) {
if (likely(try_tcache) && likely((tcache = tcache_get(true)) !=
NULL))
if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
true)) != NULL))
return (tcache_alloc_small(tcache, size, zero));
else {
return (arena_malloc_small(choose_arena(arena), size,
zero));
return (arena_malloc_small(choose_arena(tsd, arena),
size, zero));
}
} else {
/*
@ -1074,11 +1077,11 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
* infinite recursion during tcache initialization.
*/
if (try_tcache && size <= tcache_maxclass && likely((tcache =
tcache_get(true)) != NULL))
tcache_get(tsd, true)) != NULL))
return (tcache_alloc_large(tcache, size, zero));
else {
return (arena_malloc_large(choose_arena(arena), size,
zero));
return (arena_malloc_large(choose_arena(tsd, arena),
size, zero));
}
}
}
@ -1128,7 +1131,7 @@ arena_salloc(const void *ptr, bool demote)
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache)
{
size_t pageind, mapbits;
tcache_t *tcache;
@ -1141,8 +1144,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) {
/* Small allocation. */
if (likely(try_tcache) && likely((tcache = tcache_get(false)) !=
NULL)) {
if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
false)) != NULL)) {
size_t binind = arena_ptr_small_binind_get(ptr,
mapbits);
tcache_dalloc_small(tcache, ptr, binind);
@ -1154,7 +1157,7 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
if (try_tcache && size <= tcache_maxclass && likely((tcache =
tcache_get(false)) != NULL)) {
tcache_get(tsd, false)) != NULL)) {
tcache_dalloc_large(tcache, ptr, size);
} else
arena_dalloc_large(chunk->arena, chunk, ptr);
@ -1162,7 +1165,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
}
JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
bool try_tcache)
{
tcache_t *tcache;
@ -1171,8 +1175,8 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
if (likely(size <= SMALL_MAXCLASS)) {
/* Small allocation. */
if (likely(try_tcache) && likely((tcache = tcache_get(false)) !=
NULL)) {
if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
false)) != NULL)) {
size_t binind = small_size2bin(size);
tcache_dalloc_small(tcache, ptr, binind);
} else {
@ -1184,7 +1188,7 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
if (try_tcache && size <= tcache_maxclass && (tcache =
tcache_get(false)) != NULL) {
tcache_get(tsd, false)) != NULL) {
tcache_dalloc_large(tcache, ptr, size);
} else
arena_dalloc_large(chunk->arena, chunk, ptr);

8
include/jemalloc/internal/ckh.h
View File

@ -66,13 +66,13 @@ struct ckh_s {
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
bool ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh_keycomp_t *keycomp);
void ckh_delete(ckh_t *ckh);
void ckh_delete(tsd_t *tsd, ckh_t *ckh);
size_t ckh_count(ckh_t *ckh);
bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
bool ckh_insert(ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key,
bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
void **data);
bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data);
void ckh_string_hash(const void *key, size_t r_hash[2]);

10
include/jemalloc/internal/huge.h
View File

@ -9,12 +9,14 @@
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *huge_malloc(arena_t *arena, size_t size, bool zero);
void *huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero);
void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
bool zero);
bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra);
void *huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc);
void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t extra, size_t alignment, bool zero,
bool try_tcache_dalloc);
#ifdef JEMALLOC_JET
typedef void (huge_dalloc_junk_t)(void *, size_t);
extern huge_dalloc_junk_t *huge_dalloc_junk;

168
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -350,7 +350,6 @@ static const bool config_ivsalloc =
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/tsd.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
@ -364,15 +363,7 @@ static const bool config_ivsalloc =
#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
typedef struct {
uint64_t allocated;
uint64_t deallocated;
} thread_allocated_t;
/*
* The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro
* argument.
*/
#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_ARG_CONCAT({0, 0})
#include "jemalloc/internal/tsd.h"
#undef JEMALLOC_H_STRUCTS
/******************************************************************************/
@ -407,8 +398,10 @@ extern unsigned narenas_total;
extern unsigned narenas_auto; /* Read-only after initialization. */
arena_t *arenas_extend(unsigned ind);
void arenas_cleanup(void *arg);
arena_t *choose_arena_hard(void);
arena_t *choose_arena_hard(tsd_t *tsd);
void thread_allocated_cleanup(tsd_t *tsd);
void thread_deallocated_cleanup(tsd_t *tsd);
void arena_cleanup(tsd_t *tsd);
void jemalloc_prefork(void);
void jemalloc_postfork_parent(void);
void jemalloc_postfork_child(void);
@ -422,7 +415,6 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/tsd.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
@ -435,6 +427,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/hash.h"
#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
#include "jemalloc/internal/tsd.h"
#undef JEMALLOC_H_EXTERNS
/******************************************************************************/
@ -465,23 +458,13 @@ void jemalloc_postfork_child(void);
#undef JEMALLOC_ARENA_INLINE_A
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
size_t s2u(size_t size);
size_t sa2u(size_t size, size_t alignment);
unsigned narenas_total_get(void);
arena_t *choose_arena(arena_t *arena);
arena_t *choose_arena(tsd_t *tsd, arena_t *arena);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
/*
* Map of pthread_self() --> arenas[???], used for selecting an arena to use
* for allocations.
*/
malloc_tsd_externs(arenas, arena_t *)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL,
arenas_cleanup)
/*
* Compute usable size that would result from allocating an object with the
* specified size.
@ -589,15 +572,15 @@ narenas_total_get(void)
/* Choose an arena based on a per-thread value. */
JEMALLOC_INLINE arena_t *
choose_arena(arena_t *arena)
choose_arena(tsd_t *tsd, arena_t *arena)
{
arena_t *ret;
if (arena != NULL)
return (arena);
if ((ret = *arenas_tsd_get()) == NULL) {
ret = choose_arena_hard();
if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) {
ret = choose_arena_hard(tsd);
assert(ret != NULL);
}
@ -622,72 +605,72 @@ choose_arena(arena_t *arena)
#include "jemalloc/internal/quarantine.h"
#ifndef JEMALLOC_ENABLE_INLINE
void *imalloct(size_t size, bool try_tcache, arena_t *arena);
void *imalloc(size_t size);
void *icalloct(size_t size, bool try_tcache, arena_t *arena);
void *icalloc(size_t size);
void *ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena);
void *ipalloc(size_t usize, size_t alignment, bool zero);
void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
void *imalloc(tsd_t *tsd, size_t size);
void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
void *icalloc(tsd_t *tsd, size_t size);
void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
bool try_tcache, arena_t *arena);
void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
size_t isalloc(const void *ptr, bool demote);
size_t ivsalloc(const void *ptr, bool demote);
size_t u2rz(size_t usize);
size_t p2rz(const void *ptr);
void idalloct(void *ptr, bool try_tcache);
void isdalloct(void *ptr, size_t size, bool try_tcache);
void idalloc(void *ptr);
void iqalloc(void *ptr, bool try_tcache);
void isqalloc(void *ptr, size_t size, bool try_tcache);
void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena);
void *iralloct(void *ptr, size_t size, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
void *iralloc(void *ptr, size_t size, size_t alignment, bool zero);
void idalloct(tsd_t *tsd, void *ptr, bool try_tcache);
void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
void idalloc(tsd_t *tsd, void *ptr);
void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache);
void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc, arena_t *arena);
void *iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
void *iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
bool zero);
bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero);
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE void *
imalloct(size_t size, bool try_tcache, arena_t *arena)
imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
{
assert(size != 0);
if (size <= arena_maxclass)
return (arena_malloc(arena, size, false, try_tcache));
return (arena_malloc(tsd, arena, size, false, try_tcache));
else
return (huge_malloc(arena, size, false));
return (huge_malloc(tsd, arena, size, false));
}
JEMALLOC_ALWAYS_INLINE void *
imalloc(size_t size)
imalloc(tsd_t *tsd, size_t size)
{
return (imalloct(size, true, NULL));
return (imalloct(tsd, size, true, NULL));
}
JEMALLOC_ALWAYS_INLINE void *
icalloct(size_t size, bool try_tcache, arena_t *arena)
icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
{
if (size <= arena_maxclass)
return (arena_malloc(arena, size, true, try_tcache));
return (arena_malloc(tsd, arena, size, true, try_tcache));
else
return (huge_malloc(arena, size, true));
return (huge_malloc(tsd, arena, size, true));
}
JEMALLOC_ALWAYS_INLINE void *
icalloc(size_t size)
icalloc(tsd_t *tsd, size_t size)
{
return (icalloct(size, true, NULL));
return (icalloct(tsd, size, true, NULL));
}
JEMALLOC_ALWAYS_INLINE void *
ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena)
{
void *ret;
@ -696,15 +679,15 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
assert(usize == sa2u(usize, alignment));
if (usize <= arena_maxclass && alignment <= PAGE)
ret = arena_malloc(arena, usize, zero, try_tcache);
ret = arena_malloc(tsd, arena, usize, zero, try_tcache);
else {
if (usize <= arena_maxclass) {
ret = arena_palloc(choose_arena(arena), usize,
ret = arena_palloc(choose_arena(tsd, arena), usize,
alignment, zero);
} else if (alignment <= chunksize)
ret = huge_malloc(arena, usize, zero);
ret = huge_malloc(tsd, arena, usize, zero);
else
ret = huge_palloc(arena, usize, alignment, zero);
ret = huge_palloc(tsd, arena, usize, alignment, zero);
}
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
@ -712,10 +695,10 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
}
JEMALLOC_ALWAYS_INLINE void *
ipalloc(size_t usize, size_t alignment, bool zero)
ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero)
{
return (ipalloct(usize, alignment, zero, true, NULL));
return (ipalloct(tsd, usize, alignment, zero, true, NULL));
}
/*
@ -776,7 +759,7 @@ p2rz(const void *ptr)
}
JEMALLOC_ALWAYS_INLINE void
idalloct(void *ptr, bool try_tcache)
idalloct(tsd_t *tsd, void *ptr, bool try_tcache)
{
arena_chunk_t *chunk;
@ -784,13 +767,13 @@ idalloct(void *ptr, bool try_tcache)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
arena_dalloc(chunk, ptr, try_tcache);
arena_dalloc(tsd, chunk, ptr, try_tcache);
else
huge_dalloc(ptr);
}
JEMALLOC_ALWAYS_INLINE void
isdalloct(void *ptr, size_t size, bool try_tcache)
isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
{
arena_chunk_t *chunk;
@ -798,42 +781,42 @@ isdalloct(void *ptr, size_t size, bool try_tcache)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
arena_sdalloc(chunk, ptr, size, try_tcache);
arena_sdalloc(tsd, chunk, ptr, size, try_tcache);
else
huge_dalloc(ptr);
}
JEMALLOC_ALWAYS_INLINE void
idalloc(void *ptr)
idalloc(tsd_t *tsd, void *ptr)
{
idalloct(ptr, true);
idalloct(tsd, ptr, true);
}
JEMALLOC_ALWAYS_INLINE void
iqalloc(void *ptr, bool try_tcache)
iqalloc(tsd_t *tsd, void *ptr, bool try_tcache)
{
if (config_fill && unlikely(opt_quarantine))
quarantine(ptr);
quarantine(tsd, ptr);
else
idalloct(ptr, try_tcache);
idalloct(tsd, ptr, try_tcache);
}
JEMALLOC_ALWAYS_INLINE void
isqalloc(void *ptr, size_t size, bool try_tcache)
isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
{
if (config_fill && unlikely(opt_quarantine))
quarantine(ptr);
quarantine(tsd, ptr);
else
isdalloct(ptr, size, try_tcache);
isdalloct(tsd, ptr, size, try_tcache);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena)
iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc, arena_t *arena)
{
void *p;
size_t usize, copysize;
@ -841,7 +824,7 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
usize = sa2u(size + extra, alignment);
if (usize == 0)
return (NULL);
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, arena);
if (p == NULL) {
if (extra == 0)
return (NULL);
@ -849,7 +832,8 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
usize = sa2u(size, alignment);
if (usize == 0)
return (NULL);
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc,
arena);
if (p == NULL)
return (NULL);
}
@ -859,12 +843,12 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(p, ptr, copysize);
iqalloc(ptr, try_tcache_dalloc);
iqalloc(tsd, ptr, try_tcache_dalloc);
return (p);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct(void *ptr, size_t size, size_t alignment, bool zero,
iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
{
size_t oldsize;
@ -880,24 +864,24 @@ iralloct(void *ptr, size_t size, size_t alignment, bool zero,
* Existing object alignment is inadequate; allocate new space
* and copy.
*/
return (iralloct_realign(ptr, oldsize, size, 0, alignment, zero,
try_tcache_alloc, try_tcache_dalloc, arena));
return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment,
zero, try_tcache_alloc, try_tcache_dalloc, arena));
}
if (size <= arena_maxclass) {
return (arena_ralloc(arena, ptr, oldsize, size, 0, alignment,
zero, try_tcache_alloc, try_tcache_dalloc));
return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0,
alignment, zero, try_tcache_alloc, try_tcache_dalloc));
} else {
return (huge_ralloc(arena, ptr, oldsize, size, 0, alignment,
zero, try_tcache_dalloc));
return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0,
alignment, zero, try_tcache_dalloc));
}
}
JEMALLOC_ALWAYS_INLINE void *
iralloc(void *ptr, size_t size, size_t alignment, bool zero)
iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero)
{
return (iralloct(ptr, size, alignment, zero, true, true, NULL));
return (iralloct(tsd, ptr, size, alignment, zero, true, true, NULL));
}
JEMALLOC_ALWAYS_INLINE bool
@ -920,10 +904,6 @@ ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
else
return (huge_ralloc_no_move(ptr, oldsize, size, extra));
}
malloc_tsd_externs(thread_allocated, thread_allocated_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t,
THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
#endif
#include "jemalloc/internal/prof.h"

96
include/jemalloc/internal/private_symbols.txt
View File

@ -8,6 +8,7 @@ arena_bitselm_get
arena_boot
arena_chunk_alloc_huge
arena_chunk_dalloc_huge
arena_cleanup
arena_dalloc
arena_dalloc_bin
arena_dalloc_bin_locked
@ -65,19 +66,9 @@ arena_sdalloc
arena_stats_merge
arena_tcache_fill_small
arenas
arenas_booted
arenas_cleanup
arenas_extend
arenas_initialized
arenas_lock
arenas_tls
arenas_tsd
arenas_tsd_boot
arenas_tsd_cleanup_wrapper
arenas_tsd_get
arenas_tsd_get_wrapper
arenas_tsd_init_head
arenas_tsd_set
atomic_add_u
atomic_add_uint32
atomic_add_uint64
@ -317,37 +308,17 @@ prof_sample_accum_update
prof_sample_threshold_update
prof_tctx_get
prof_tctx_set
prof_tdata_booted
prof_tdata_cleanup
prof_tdata_get
prof_tdata_init
prof_tdata_initialized
prof_tdata_tls
prof_tdata_tsd
prof_tdata_tsd_boot
prof_tdata_tsd_cleanup_wrapper
prof_tdata_tsd_get
prof_tdata_tsd_get_wrapper
prof_tdata_tsd_init_head
prof_tdata_tsd_set
prof_thread_active_get
prof_thread_active_set
prof_thread_name_get
prof_thread_name_set
quarantine
quarantine_alloc_hook
quarantine_boot
quarantine_booted
quarantine_cleanup
quarantine_init
quarantine_tls
quarantine_tsd
quarantine_tsd_boot
quarantine_tsd_cleanup_wrapper
quarantine_tsd_get
quarantine_tsd_get_wrapper
quarantine_tsd_init_head
quarantine_tsd_set
register_zone
rtree_delete
rtree_get
@ -386,55 +357,52 @@ tcache_arena_dissociate
tcache_bin_flush_large
tcache_bin_flush_small
tcache_bin_info
tcache_boot0
tcache_boot1
tcache_booted
tcache_boot
tcache_cleanup
tcache_create
tcache_dalloc_large
tcache_dalloc_small
tcache_destroy
tcache_enabled_booted
tcache_enabled_cleanup
tcache_enabled_get
tcache_enabled_initialized
tcache_enabled_set
tcache_enabled_tls
tcache_enabled_tsd
tcache_enabled_tsd_boot
tcache_enabled_tsd_cleanup_wrapper
tcache_enabled_tsd_get
tcache_enabled_tsd_get_wrapper
tcache_enabled_tsd_init_head
tcache_enabled_tsd_set
tcache_event
tcache_event_hard
tcache_flush
tcache_get
tcache_get_hard
tcache_initialized
tcache_maxclass
tcache_salloc
tcache_stats_merge
tcache_thread_cleanup
tcache_tls
tcache_tsd
tcache_tsd_boot
tcache_tsd_cleanup_wrapper
tcache_tsd_get
tcache_tsd_get_wrapper
tcache_tsd_init_head
tcache_tsd_set
thread_allocated_booted
thread_allocated_initialized
thread_allocated_tls
thread_allocated_tsd
thread_allocated_tsd_boot
thread_allocated_tsd_cleanup_wrapper
thread_allocated_tsd_get
thread_allocated_tsd_get_wrapper
thread_allocated_tsd_init_head
thread_allocated_tsd_set
thread_allocated_cleanup
thread_deallocated_cleanup
tsd_booted
tsd_arena_get
tsd_arena_set
tsd_boot
tsd_cleanup
tsd_cleanup_wrapper
tsd_get
tsd_get_wrapper
tsd_initialized
tsd_init_check_recursion
tsd_init_finish
tsd_init_head
tsd_quarantine_get
tsd_quarantine_set
tsd_set
tsd_tcache_enabled_get
tsd_tcache_enabled_set
tsd_tcache_get
tsd_tcache_set
tsd_tls
tsd_tsd
tsd_prof_tdata_get
tsd_prof_tdata_set
tsd_thread_allocated_get
tsd_thread_allocated_set
tsd_thread_deallocated_get
tsd_thread_deallocated_set
tsd_tryget
u2rz
valgrind_freelike_block
valgrind_make_mem_defined

78
include/jemalloc/internal/prof.h
View File

@ -248,13 +248,13 @@ extern uint64_t prof_interval;
*/
extern size_t lg_prof_sample;
void prof_alloc_rollback(prof_tctx_t *tctx, bool updated);
void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
void prof_malloc_sample_object(const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_free_sampled_object(size_t usize, prof_tctx_t *tctx);
void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
void bt_init(prof_bt_t *bt, void **vec);
void prof_backtrace(prof_bt_t *bt);
prof_tctx_t *prof_lookup(prof_bt_t *bt);
prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt);
#ifdef JEMALLOC_JET
size_t prof_bt_count(void);
typedef int (prof_dump_open_t)(bool, const char *);
@ -263,12 +263,12 @@ extern prof_dump_open_t *prof_dump_open;
void prof_idump(void);
bool prof_mdump(const char *filename);
void prof_gdump(void);
prof_tdata_t *prof_tdata_init(void);
prof_tdata_t *prof_tdata_reinit(prof_tdata_t *tdata);
void prof_reset(size_t lg_sample);
void prof_tdata_cleanup(void *arg);
prof_tdata_t *prof_tdata_init(tsd_t *tsd);
prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
void prof_reset(tsd_t *tsd, size_t lg_sample);
void prof_tdata_cleanup(tsd_t *tsd);
const char *prof_thread_name_get(void);
bool prof_thread_name_set(const char *thread_name);
bool prof_thread_name_set(tsd_t *tsd, const char *thread_name);
bool prof_thread_active_get(void);
bool prof_thread_active_set(bool active);
void prof_boot0(void);
@ -284,43 +284,38 @@ void prof_sample_threshold_update(prof_tdata_t *tdata);
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
prof_tdata_t *prof_tdata_get(bool create);
bool prof_sample_accum_update(size_t usize, bool commit,
prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create);
bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
prof_tdata_t **tdata_out);
prof_tctx_t *prof_alloc_prep(size_t usize, bool update);
prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool update);
prof_tctx_t *prof_tctx_get(const void *ptr);
void prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
void prof_malloc_sample_object(const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx);
void prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx,
bool updated, size_t old_usize, prof_tctx_t *old_tctx);
void prof_free(const void *ptr, size_t usize);
void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize,
prof_tctx_t *tctx, bool updated, size_t old_usize, prof_tctx_t *old_tctx);
void prof_free(tsd_t *tsd, const void *ptr, size_t usize);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
/* Thread-specific backtrace cache, used to reduce bt2gctx contention. */
malloc_tsd_externs(prof_tdata, prof_tdata_t *)
malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
prof_tdata_cleanup)
JEMALLOC_INLINE prof_tdata_t *
prof_tdata_get(bool create)
prof_tdata_get(tsd_t *tsd, bool create)
{
prof_tdata_t *tdata;
cassert(config_prof);
tdata = *prof_tdata_tsd_get();
tdata = tsd_prof_tdata_get(tsd);
if (create) {
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {
if (tdata == NULL)
tdata = prof_tdata_init();
} else if (tdata->state == prof_tdata_state_expired)
tdata = prof_tdata_reinit(tdata);
assert((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX ||
if (unlikely(tdata == NULL)) {
tdata = prof_tdata_init(tsd);
tsd_prof_tdata_set(tsd, tdata);
} else if (unlikely(tdata->state == prof_tdata_state_expired)) {
tdata = prof_tdata_reinit(tsd, tdata);
tsd_prof_tdata_set(tsd, tdata);
}
assert(tdata == NULL ||
tdata->state == prof_tdata_state_attached);
}
@ -363,13 +358,14 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
}
JEMALLOC_INLINE bool
prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out)
prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
prof_tdata_t **tdata_out)
{
prof_tdata_t *tdata;
cassert(config_prof);
tdata = prof_tdata_get(true);
tdata = prof_tdata_get(tsd, true);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
tdata = NULL;
@ -392,7 +388,7 @@ prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out)
}
JEMALLOC_INLINE prof_tctx_t *
prof_alloc_prep(size_t usize, bool update)
prof_alloc_prep(tsd_t *tsd, size_t usize, bool update)
{
prof_tctx_t *ret;
prof_tdata_t *tdata;
@ -400,13 +396,13 @@ prof_alloc_prep(size_t usize, bool update)
assert(usize == s2u(usize));
if (!opt_prof_active || likely(prof_sample_accum_update(usize, update,
&tdata)))
if (!opt_prof_active || likely(prof_sample_accum_update(tsd, usize,
update, &tdata)))
ret = (prof_tctx_t *)(uintptr_t)1U;
else {
bt_init(&bt, tdata->vec);
prof_backtrace(&bt);
ret = prof_lookup(&bt);
ret = prof_lookup(tsd, &bt);
}
return (ret);
@ -427,8 +423,8 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx)
}
JEMALLOC_INLINE void
prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
size_t old_usize, prof_tctx_t *old_tctx)
prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
bool updated, size_t old_usize, prof_tctx_t *old_tctx)
{
cassert(config_prof);
@ -436,7 +432,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
if (!updated && ptr != NULL) {
assert(usize == isalloc(ptr, true));
if (prof_sample_accum_update(usize, true, NULL)) {
if (prof_sample_accum_update(tsd, usize, true, NULL)) {
/*
* Don't sample. The usize passed to PROF_ALLOC_PREP()
* was larger than what actually got allocated, so a
@ -449,7 +445,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
}
if (unlikely((uintptr_t)old_tctx > (uintptr_t)1U))
prof_free_sampled_object(old_usize, old_tctx);
prof_free_sampled_object(tsd, old_usize, old_tctx);
if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
prof_malloc_sample_object(ptr, usize, tctx);
else
@ -457,7 +453,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
}
JEMALLOC_INLINE void
prof_free(const void *ptr, size_t usize)
prof_free(tsd_t *tsd, const void *ptr, size_t usize)
{
prof_tctx_t *tctx = prof_tctx_get(ptr);
@ -465,7 +461,7 @@ prof_free(const void *ptr, size_t usize)
assert(usize == isalloc(ptr, true));
if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
prof_free_sampled_object(usize, tctx);
prof_free_sampled_object(tsd, usize, tctx);
}
#endif

21
include/jemalloc/internal/quarantine.h
View File

@ -29,36 +29,29 @@ struct quarantine_s {
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
quarantine_t *quarantine_init(size_t lg_maxobjs);
void quarantine(void *ptr);
void quarantine_cleanup(void *arg);
bool quarantine_boot(void);
quarantine_t *quarantine_init(tsd_t *tsd, size_t lg_maxobjs);
void quarantine(tsd_t *tsd, void *ptr);
void quarantine_cleanup(tsd_t *tsd);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *)
void quarantine_alloc_hook(void);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_))
malloc_tsd_externs(quarantine, quarantine_t *)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL,
quarantine_cleanup)
JEMALLOC_ALWAYS_INLINE void
quarantine_alloc_hook(void)
{
quarantine_t *quarantine;
tsd_t *tsd;
assert(config_fill && opt_quarantine);
quarantine = *quarantine_tsd_get();
if (quarantine == NULL)
quarantine_init(LG_MAXOBJS_INIT);
tsd = tsd_tryget();
if (tsd != NULL && tsd_quarantine_get(tsd) == NULL)
tsd_quarantine_set(tsd, quarantine_init(tsd, LG_MAXOBJS_INIT));
}
#endif

88
include/jemalloc/internal/tcache.h
View File

@ -110,26 +110,22 @@ void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
tcache_t *tcache);
void tcache_arena_associate(tcache_t *tcache, arena_t *arena);
void tcache_arena_dissociate(tcache_t *tcache);
tcache_t *tcache_get_hard(tcache_t *tcache, bool create);
tcache_t *tcache_get_hard(tsd_t *tsd);
tcache_t *tcache_create(arena_t *arena);
void tcache_destroy(tcache_t *tcache);
void tcache_thread_cleanup(void *arg);
void tcache_cleanup(tsd_t *tsd);
void tcache_enabled_cleanup(tsd_t *tsd);
void tcache_stats_merge(tcache_t *tcache, arena_t *arena);
bool tcache_boot0(void);
bool tcache_boot1(void);
bool tcache_boot(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *)
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t)
void tcache_event(tcache_t *tcache);
void tcache_flush(void);
bool tcache_enabled_get(void);
tcache_t *tcache_get(bool create);
tcache_t *tcache_get(tsd_t *tsd, bool create);
void tcache_enabled_set(bool enabled);
void *tcache_alloc_easy(tcache_bin_t *tbin);
void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
@ -139,41 +135,33 @@ void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
/* Map of thread-specific caches. */
malloc_tsd_externs(tcache, tcache_t *)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL,
tcache_thread_cleanup)
/* Per thread flag that allows thread caches to be disabled. */
malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t,
tcache_enabled_default, malloc_tsd_no_cleanup)
JEMALLOC_INLINE void
tcache_flush(void)
{
tcache_t *tcache;
tsd_t *tsd;
cassert(config_tcache);
tcache = *tcache_tsd_get();
if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)
return;
tcache_destroy(tcache);
tcache = NULL;
tcache_tsd_set(&tcache);
tsd = tsd_tryget();
if (tsd != NULL)
tcache_cleanup(tsd);
}
JEMALLOC_INLINE bool
tcache_enabled_get(void)
{
tsd_t *tsd;
tcache_enabled_t tcache_enabled;
cassert(config_tcache);
tcache_enabled = *tcache_enabled_tsd_get();
tsd = tsd_tryget();
if (tsd == NULL)
return (false);
tcache_enabled = tsd_tcache_enabled_get(tsd);
if (tcache_enabled == tcache_enabled_default) {
tcache_enabled = (tcache_enabled_t)opt_tcache;
tcache_enabled_tsd_set(&tcache_enabled);
tsd_tcache_enabled_set(tsd, tcache_enabled);
}
return ((bool)tcache_enabled);
@ -182,33 +170,24 @@ tcache_enabled_get(void)
JEMALLOC_INLINE void
tcache_enabled_set(bool enabled)
{
tsd_t *tsd;
tcache_enabled_t tcache_enabled;
tcache_t *tcache;
cassert(config_tcache);
tsd = tsd_tryget();
if (tsd == NULL)
return;
tcache_enabled = (tcache_enabled_t)enabled;
tcache_enabled_tsd_set(&tcache_enabled);
tcache = *tcache_tsd_get();
if (enabled) {
if (tcache == TCACHE_STATE_DISABLED) {
tcache = NULL;
tcache_tsd_set(&tcache);
}
} else /* disabled */ {
if (tcache > TCACHE_STATE_MAX) {
tcache_destroy(tcache);
tcache = NULL;
}
if (tcache == NULL) {
tcache = TCACHE_STATE_DISABLED;
tcache_tsd_set(&tcache);
}
}
tsd_tcache_enabled_set(tsd, tcache_enabled);
if (!enabled)
tcache_cleanup(tsd);
}
JEMALLOC_ALWAYS_INLINE tcache_t *
tcache_get(bool create)
tcache_get(tsd_t *tsd, bool create)
{
tcache_t *tcache;
@ -216,12 +195,19 @@ tcache_get(bool create)
return (NULL);
if (config_lazy_lock && isthreaded == false)
return (NULL);
tcache = *tcache_tsd_get();
if (unlikely((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)) {
if (tcache == TCACHE_STATE_DISABLED)
/*
* If create is true, the caller has already assured that tsd is
* non-NULL.
*/
if (!create && unlikely(tsd == NULL))
return (NULL);
tcache = tcache_get_hard(tcache, create);
tcache = tsd_tcache_get(tsd);
if (!create)
return (tcache);
if (unlikely(tcache == NULL)) {
tcache = tcache_get_hard(tsd);
tsd_tcache_set(tsd, tcache);
}
return (tcache);

341
include/jemalloc/internal/tsd.h
View File

@ -12,6 +12,15 @@ typedef struct tsd_init_block_s tsd_init_block_t;
typedef struct tsd_init_head_s tsd_init_head_t;
#endif
typedef struct tsd_s tsd_t;
typedef enum {
tsd_state_uninitialized,
tsd_state_nominal,
tsd_state_purgatory,
tsd_state_reincarnated
} tsd_state_t;
/*
* TLS/TSD-agnostic macro-based implementation of thread-specific data. There
* are four macros that support (at least) three use cases: file-private,
@ -24,11 +33,11 @@ typedef struct tsd_init_head_s tsd_init_head_t;
* int y;
* } example_t;
* #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
* malloc_tsd_protos(, example, example_t *)
* malloc_tsd_externs(example, example_t *)
* malloc_tsd_protos(, example_, example_t *)
* malloc_tsd_externs(example_, example_t *)
* In example.c:
* malloc_tsd_data(, example, example_t *, EX_INITIALIZER)
* malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER,
* malloc_tsd_data(, example_, example_t *, EX_INITIALIZER)
* malloc_tsd_funcs(, example_, example_t *, EX_INITIALIZER,
* example_tsd_cleanup)
*
* The result is a set of generated functions, e.g.:
@ -43,15 +52,13 @@ typedef struct tsd_init_head_s tsd_init_head_t;
* cast to (void *). This means that the cleanup function needs to cast *and*
* dereference the function argument, e.g.:
*
* void
* bool
* example_tsd_cleanup(void *arg)
* {
* example_t *example = *(example_t **)arg;
*
* [...]
* if ([want the cleanup function to be called again]) {
* example_tsd_set(&example);
* }
* return ([want the cleanup function to be called again]);
* }
*
* If example_tsd_set() is called within example_tsd_cleanup(), it will be
@ -63,60 +70,60 @@ typedef struct tsd_init_head_s tsd_init_head_t;
/* malloc_tsd_protos(). */
#define malloc_tsd_protos(a_attr, a_name, a_type) \
a_attr bool \
a_name##_tsd_boot(void); \
a_name##tsd_boot(void); \
a_attr a_type * \
a_name##_tsd_get(void); \
a_name##tsd_get(void); \
a_attr void \
a_name##_tsd_set(a_type *val);
a_name##tsd_set(a_type *val);
/* malloc_tsd_externs(). */
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
#define malloc_tsd_externs(a_name, a_type) \
extern __thread a_type a_name##_tls; \
extern __thread bool a_name##_initialized; \
extern bool a_name##_booted;
extern __thread a_type a_name##tsd_tls; \
extern __thread bool a_name##tsd_initialized; \
extern bool a_name##tsd_booted;
#elif (defined(JEMALLOC_TLS))
#define malloc_tsd_externs(a_name, a_type) \
extern __thread a_type a_name##_tls; \
extern pthread_key_t a_name##_tsd; \
extern bool a_name##_booted;
extern __thread a_type a_name##tsd_tls; \
extern pthread_key_t a_name##tsd_tsd; \
extern bool a_name##tsd_booted;
#elif (defined(_WIN32))
#define malloc_tsd_externs(a_name, a_type) \
extern DWORD a_name##_tsd; \
extern bool a_name##_booted;
extern DWORD a_name##tsd_tsd; \
extern bool a_name##tsd_booted;
#else
#define malloc_tsd_externs(a_name, a_type) \
extern pthread_key_t a_name##_tsd; \
extern tsd_init_head_t a_name##_tsd_init_head; \
extern bool a_name##_booted;
extern pthread_key_t a_name##tsd_tsd; \
extern tsd_init_head_t a_name##tsd_init_head; \
extern bool a_name##tsd_booted;
#endif
/* malloc_tsd_data(). */
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr __thread a_type JEMALLOC_TLS_MODEL \
a_name##_tls = a_initializer; \
a_name##tsd_tls = a_initializer; \
a_attr __thread bool JEMALLOC_TLS_MODEL \
a_name##_initialized = false; \
a_attr bool a_name##_booted = false;
a_name##tsd_initialized = false; \
a_attr bool a_name##tsd_booted = false;
#elif (defined(JEMALLOC_TLS))
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr __thread a_type JEMALLOC_TLS_MODEL \
a_name##_tls = a_initializer; \
a_attr pthread_key_t a_name##_tsd; \
a_attr bool a_name##_booted = false;
a_name##tsd_tls = a_initializer; \
a_attr pthread_key_t a_name##tsd_tsd; \
a_attr bool a_name##tsd_booted = false;
#elif (defined(_WIN32))
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr DWORD a_name##_tsd; \
a_attr bool a_name##_booted = false;
a_attr DWORD a_name##tsd_tsd; \
a_attr bool a_name##tsd_booted = false;
#else
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr pthread_key_t a_name##_tsd; \
a_attr tsd_init_head_t a_name##_tsd_init_head = { \
a_attr pthread_key_t a_name##tsd_tsd; \
a_attr tsd_init_head_t a_name##tsd_init_head = { \
ql_head_initializer(blocks), \
MALLOC_MUTEX_INITIALIZER \
}; \
a_attr bool a_name##_booted = false;
a_attr bool a_name##tsd_booted = false;
#endif
/* malloc_tsd_funcs(). */
@ -125,75 +132,76 @@ a_attr bool a_name##_booted = false;
a_cleanup) \
/* Initialization/cleanup. */ \
a_attr bool \
a_name##_tsd_cleanup_wrapper(void) \
a_name##tsd_cleanup_wrapper(void) \
{ \
\
if (a_name##_initialized) { \
a_name##_initialized = false; \
a_cleanup(&a_name##_tls); \
if (a_name##tsd_initialized) { \
a_name##tsd_initialized = false; \
a_cleanup(&a_name##tsd_tls); \
} \
return (a_name##_initialized); \
return (a_name##tsd_initialized); \
} \
a_attr bool \
a_name##_tsd_boot(void) \
a_name##tsd_boot(void) \
{ \
\
if (a_cleanup != malloc_tsd_no_cleanup) { \
malloc_tsd_cleanup_register( \
&a_name##_tsd_cleanup_wrapper); \
&a_name##tsd_cleanup_wrapper); \
} \
a_name##_booted = true; \
a_name##tsd_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_type * \
a_name##_tsd_get(void) \
a_name##tsd_get(void) \
{ \
\
assert(a_name##_booted); \
return (&a_name##_tls); \
assert(a_name##tsd_booted); \
return (&a_name##tsd_tls); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
a_name##tsd_set(a_type *val) \
{ \
\
assert(a_name##_booted); \
a_name##_tls = (*val); \
assert(a_name##tsd_booted); \
a_name##tsd_tls = (*val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
a_name##_initialized = true; \
a_name##tsd_initialized = true; \
}
#elif (defined(JEMALLOC_TLS))
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
a_cleanup) \
/* Initialization/cleanup. */ \
a_attr bool \
a_name##_tsd_boot(void) \
a_name##tsd_boot(void) \
{ \
\
if (a_cleanup != malloc_tsd_no_cleanup) { \
if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0) \
if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \
0) \
return (true); \
} \
a_name##_booted = true; \
a_name##tsd_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_type * \
a_name##_tsd_get(void) \
a_name##tsd_get(void) \
{ \
\
assert(a_name##_booted); \
return (&a_name##_tls); \
assert(a_name##tsd_booted); \
return (&a_name##tsd_tls); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
a_name##tsd_set(a_type *val) \
{ \
\
assert(a_name##_booted); \
a_name##_tls = (*val); \
assert(a_name##tsd_booted); \
a_name##tsd_tls = (*val); \
if (a_cleanup != malloc_tsd_no_cleanup) { \
if (pthread_setspecific(a_name##_tsd, \
(void *)(&a_name##_tls))) { \
if (pthread_setspecific(a_name##tsd_tsd, \
(void *)(&a_name##tsd_tls))) { \
malloc_write("<jemalloc>: Error" \
" setting TSD for "#a_name"\n"); \
if (opt_abort) \
@ -208,23 +216,20 @@ a_name##_tsd_set(a_type *val) \
typedef struct { \
bool initialized; \
a_type val; \
} a_name##_tsd_wrapper_t; \
} a_name##tsd_wrapper_t; \
/* Initialization/cleanup. */ \
a_attr bool \
a_name##_tsd_cleanup_wrapper(void) \
a_name##tsd_cleanup_wrapper(void) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd); \
wrapper = (a_name##tsd_wrapper_t *)TlsGetValue(a_name##tsd_tsd);\
if (wrapper == NULL) \
return (false); \
if (a_cleanup != malloc_tsd_no_cleanup && \
wrapper->initialized) { \
a_type val = wrapper->val; \
a_type tsd_static_data = a_initializer; \
wrapper->initialized = false; \
wrapper->val = tsd_static_data; \
a_cleanup(&val); \
a_cleanup(&wrapper->val); \
if (wrapper->initialized) { \
/* Trigger another cleanup round. */ \
return (true); \
@ -234,39 +239,38 @@ a_name##_tsd_cleanup_wrapper(void) \
return (false); \
} \
a_attr bool \
a_name##_tsd_boot(void) \
a_name##tsd_boot(void) \
{ \
\
a_name##_tsd = TlsAlloc(); \
if (a_name##_tsd == TLS_OUT_OF_INDEXES) \
a_name##tsd_tsd = TlsAlloc(); \
if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \
return (true); \
if (a_cleanup != malloc_tsd_no_cleanup) { \
malloc_tsd_cleanup_register( \
&a_name##_tsd_cleanup_wrapper); \
&a_name##tsd_cleanup_wrapper); \
} \
a_name##_booted = true; \
a_name##tsd_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_name##_tsd_wrapper_t * \
a_name##_tsd_get_wrapper(void) \
a_attr a_name##tsd_wrapper_t * \
a_name##tsd_get_wrapper(void) \
{ \
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \
TlsGetValue(a_name##_tsd); \
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
TlsGetValue(a_name##tsd_tsd); \
\
if (wrapper == NULL) { \
wrapper = (a_name##_tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \
if (unlikely(wrapper == NULL)) { \
wrapper = (a_name##tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
if (wrapper == NULL) { \
malloc_write("<jemalloc>: Error allocating" \
" TSD for "#a_name"\n"); \
abort(); \
} else { \
static a_type tsd_static_data = a_initializer; \
wrapper->initialized = false; \
wrapper->val = tsd_static_data; \
wrapper->val = a_initializer; \
} \
if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) { \
if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \
malloc_write("<jemalloc>: Error setting" \
" TSD for "#a_name"\n"); \
abort(); \
@ -275,21 +279,21 @@ a_name##_tsd_get_wrapper(void) \
return (wrapper); \
} \
a_attr a_type * \
a_name##_tsd_get(void) \
a_name##tsd_get(void) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_get_wrapper(); \
return (&wrapper->val); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
a_name##tsd_set(a_type *val) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_get_wrapper(); \
wrapper->val = *(val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
wrapper->initialized = true; \
@ -301,12 +305,12 @@ a_name##_tsd_set(a_type *val) \
typedef struct { \
bool initialized; \
a_type val; \
} a_name##_tsd_wrapper_t; \
} a_name##tsd_wrapper_t; \
/* Initialization/cleanup. */ \
a_attr void \
a_name##_tsd_cleanup_wrapper(void *arg) \
a_name##tsd_cleanup_wrapper(void *arg) \
{ \
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \
\
if (a_cleanup != malloc_tsd_no_cleanup && \
wrapper->initialized) { \
@ -314,7 +318,7 @@ a_name##_tsd_cleanup_wrapper(void *arg) \
a_cleanup(&wrapper->val); \
if (wrapper->initialized) { \
/* Trigger another cleanup round. */ \
if (pthread_setspecific(a_name##_tsd, \
if (pthread_setspecific(a_name##tsd_tsd, \
(void *)wrapper)) { \
malloc_write("<jemalloc>: Error" \
" setting TSD for "#a_name"\n"); \
@ -327,66 +331,65 @@ a_name##_tsd_cleanup_wrapper(void *arg) \
malloc_tsd_dalloc(wrapper); \
} \
a_attr bool \
a_name##_tsd_boot(void) \
a_name##tsd_boot(void) \
{ \
\
if (pthread_key_create(&a_name##_tsd, \
a_name##_tsd_cleanup_wrapper) != 0) \
if (pthread_key_create(&a_name##tsd_tsd, \
a_name##tsd_cleanup_wrapper) != 0) \
return (true); \
a_name##_booted = true; \
a_name##tsd_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_name##_tsd_wrapper_t * \
a_name##_tsd_get_wrapper(void) \
a_attr a_name##tsd_wrapper_t * \
a_name##tsd_get_wrapper(void) \
{ \
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \
pthread_getspecific(a_name##_tsd); \
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
pthread_getspecific(a_name##tsd_tsd); \
\
if (wrapper == NULL) { \
if (unlikely(wrapper == NULL)) { \
tsd_init_block_t block; \
wrapper = tsd_init_check_recursion( \
&a_name##_tsd_init_head, &block); \
&a_name##tsd_init_head, &block); \
if (wrapper) \
return (wrapper); \
wrapper = (a_name##_tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \
wrapper = (a_name##tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
block.data = wrapper; \
if (wrapper == NULL) { \
malloc_write("<jemalloc>: Error allocating" \
" TSD for "#a_name"\n"); \
abort(); \
} else { \
static a_type tsd_static_data = a_initializer; \
wrapper->initialized = false; \
wrapper->val = tsd_static_data; \
wrapper->val = a_initializer; \
} \
if (pthread_setspecific(a_name##_tsd, \
if (pthread_setspecific(a_name##tsd_tsd, \
(void *)wrapper)) { \
malloc_write("<jemalloc>: Error setting" \
" TSD for "#a_name"\n"); \
abort(); \
} \
tsd_init_finish(&a_name##_tsd_init_head, &block); \
tsd_init_finish(&a_name##tsd_init_head, &block); \
} \
return (wrapper); \
} \
a_attr a_type * \
a_name##_tsd_get(void) \
a_name##tsd_get(void) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_get_wrapper(); \
return (&wrapper->val); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
a_name##tsd_set(a_type *val) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_get_wrapper(); \
wrapper->val = *(val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
wrapper->initialized = true; \
@ -410,25 +413,123 @@ struct tsd_init_head_s {
};
#endif
#define MALLOC_TSD \
/* O(name, type) */ \
O(tcache, tcache_t *) \
O(thread_allocated, uint64_t) \
O(thread_deallocated, uint64_t) \
O(prof_tdata, prof_tdata_t *) \
O(arena, arena_t *) \
O(tcache_enabled, tcache_enabled_t) \
O(quarantine, quarantine_t *) \
#define TSD_INITIALIZER { \
tsd_state_uninitialized, \
NULL, \
0, \
0, \
NULL, \
NULL, \
tcache_enabled_default, \
NULL \
}
struct tsd_s {
tsd_state_t state;
#define O(n, t) \
t n;
MALLOC_TSD
#undef O
};
static const tsd_t tsd_initializer = TSD_INITIALIZER;
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *malloc_tsd_malloc(size_t size);
void malloc_tsd_dalloc(void *wrapper);
void malloc_tsd_no_cleanup(void *);
void malloc_tsd_no_cleanup(void *arg);
void malloc_tsd_cleanup_register(bool (*f)(void));
void malloc_tsd_boot(void);
bool malloc_tsd_boot(void);
#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
!defined(_WIN32))
void *tsd_init_check_recursion(tsd_init_head_t *head,
tsd_init_block_t *block);
void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
#endif
void tsd_cleanup(void *arg);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
tsd_t *tsd_tryget(void);
#define O(n, t) \
t *tsd_##n##p_get(tsd_t *tsd); \
t tsd_##n##_get(tsd_t *tsd); \
void tsd_##n##_set(tsd_t *tsd, t n);
MALLOC_TSD
#undef O
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
malloc_tsd_externs(, tsd_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
JEMALLOC_INLINE tsd_t *
tsd_tryget(void)
{
tsd_t *tsd;
tsd = tsd_get();
if (unlikely(tsd == NULL))
return (NULL);
if (likely(tsd->state == tsd_state_nominal))
return (tsd);
else if (tsd->state == tsd_state_uninitialized) {
tsd->state = tsd_state_nominal;
tsd_set(tsd);
return (tsd);
} else if (tsd->state == tsd_state_purgatory) {
tsd->state = tsd_state_reincarnated;
tsd_set(tsd);
return (NULL);
} else {
assert(tsd->state == tsd_state_reincarnated);
return (NULL);
}
}
#define O(n, t) \
JEMALLOC_INLINE t * \
tsd_##n##p_get(tsd_t *tsd) \
{ \
\
return (&tsd->n); \
} \
\
JEMALLOC_INLINE t \
tsd_##n##_get(tsd_t *tsd) \
{ \
\
return (*tsd_##n##p_get(tsd)); \
} \
\
JEMALLOC_INLINE void \
tsd_##n##_set(tsd_t *tsd, t n) \
{ \
\
tsd->n = n; \
}
MALLOC_TSD
#undef O
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/

23
src/arena.c
View File

@ -2058,7 +2058,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
}
void *
arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc)
{
@ -2078,9 +2078,12 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t usize = sa2u(size + extra, alignment);
if (usize == 0)
return (NULL);
ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
} else
ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc);
ret = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc,
arena);
} else {
ret = arena_malloc(tsd, arena, size + extra, zero,
try_tcache_alloc);
}
if (ret == NULL) {
if (extra == 0)
@ -2090,10 +2093,12 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t usize = sa2u(size, alignment);
if (usize == 0)
return (NULL);
ret = ipalloct(usize, alignment, zero, try_tcache_alloc,
arena);
} else
ret = arena_malloc(arena, size, zero, try_tcache_alloc);
ret = ipalloct(tsd, usize, alignment, zero,
try_tcache_alloc, arena);
} else {
ret = arena_malloc(tsd, arena, size, zero,
try_tcache_alloc);
}
if (ret == NULL)
return (NULL);
@ -2108,7 +2113,7 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
copysize = (size < oldsize) ? size : oldsize;
JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize);
memcpy(ret, ptr, copysize);
iqalloc(ptr, try_tcache_dalloc);
iqalloc(tsd, ptr, try_tcache_dalloc);
return (ret);
}

38
src/ckh.c
View File

@ -40,8 +40,8 @@
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static bool ckh_grow(ckh_t *ckh);
static void ckh_shrink(ckh_t *ckh);
static bool ckh_grow(tsd_t *tsd, ckh_t *ckh);
static void ckh_shrink(tsd_t *tsd, ckh_t *ckh);
/******************************************************************************/
@ -243,7 +243,7 @@ ckh_rebuild(ckh_t *ckh, ckhc_t *aTab)
}
static bool
ckh_grow(ckh_t *ckh)
ckh_grow(tsd_t *tsd, ckh_t *ckh)
{
bool ret;
ckhc_t *tab, *ttab;
@ -270,7 +270,7 @@ ckh_grow(ckh_t *ckh)
ret = true;
goto label_return;
}
tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true);
if (tab == NULL) {
ret = true;
goto label_return;
@ -282,12 +282,12 @@ ckh_grow(ckh_t *ckh)
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
if (ckh_rebuild(ckh, tab) == false) {
idalloc(tab);
idalloc(tsd, tab);
break;
}
/* Rebuilding failed, so back out partially rebuilt table. */
idalloc(ckh->tab);
idalloc(tsd, ckh->tab);
ckh->tab = tab;
ckh->lg_curbuckets = lg_prevbuckets;
}
@ -298,7 +298,7 @@ label_return:
}
static void
ckh_shrink(ckh_t *ckh)
ckh_shrink(tsd_t *tsd, ckh_t *ckh)
{
ckhc_t *tab, *ttab;
size_t lg_curcells, usize;
@ -313,7 +313,7 @@ ckh_shrink(ckh_t *ckh)
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
if (usize == 0)
return;
tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true);
if (tab == NULL) {
/*
* An OOM error isn't worth propagating, since it doesn't
@ -328,7 +328,7 @@ ckh_shrink(ckh_t *ckh)
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
if (ckh_rebuild(ckh, tab) == false) {
idalloc(tab);
idalloc(tsd, tab);
#ifdef CKH_COUNT
ckh->nshrinks++;
#endif
@ -336,7 +336,7 @@ ckh_shrink(ckh_t *ckh)
}
/* Rebuilding failed, so back out partially rebuilt table. */
idalloc(ckh->tab);
idalloc(tsd, ckh->tab);
ckh->tab = tab;
ckh->lg_curbuckets = lg_prevbuckets;
#ifdef CKH_COUNT
@ -345,7 +345,8 @@ ckh_shrink(ckh_t *ckh)
}
bool
ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh_keycomp_t *keycomp)
{
bool ret;
size_t mincells, usize;
@ -388,7 +389,7 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
ret = true;
goto label_return;
}
ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true);
ckh->tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true);
if (ckh->tab == NULL) {
ret = true;
goto label_return;
@ -400,7 +401,7 @@ label_return:
}
void
ckh_delete(ckh_t *ckh)
ckh_delete(tsd_t *tsd, ckh_t *ckh)
{
assert(ckh != NULL);
@ -417,7 +418,7 @@ ckh_delete(ckh_t *ckh)
(unsigned long long)ckh->nrelocs);
#endif
idalloc(ckh->tab);
idalloc(tsd, ckh->tab);
if (config_debug)
memset(ckh, 0x5a, sizeof(ckh_t));
}
@ -452,7 +453,7 @@ ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data)
}
bool
ckh_insert(ckh_t *ckh, const void *key, const void *data)
ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data)
{
bool ret;
@ -464,7 +465,7 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data)
#endif
while (ckh_try_insert(ckh, &key, &data)) {
if (ckh_grow(ckh)) {
if (ckh_grow(tsd, ckh)) {
ret = true;
goto label_return;
}
@ -476,7 +477,8 @@ label_return:
}
bool
ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data)
ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
void **data)
{
size_t cell;
@ -497,7 +499,7 @@ ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data)
+ LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets
> ckh->lg_minbuckets) {
/* Ignore error due to OOM. */
ckh_shrink(ckh);
ckh_shrink(tsd, ckh);
}
return (false);

93
src/ctl.c
View File

@ -565,18 +565,23 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
static bool
ctl_grow(void)
{
tsd_t *tsd;
ctl_arena_stats_t *astats;
arena_t **tarenas;
tsd = tsd_tryget();
if (tsd == NULL)
return (true);
/* Allocate extended arena stats and arenas arrays. */
astats = (ctl_arena_stats_t *)imalloc((ctl_stats.narenas + 2) *
astats = (ctl_arena_stats_t *)imalloc(tsd, (ctl_stats.narenas + 2) *
sizeof(ctl_arena_stats_t));
if (astats == NULL)
return (true);
tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) *
tarenas = (arena_t **)imalloc(tsd, (ctl_stats.narenas + 1) *
sizeof(arena_t *));
if (tarenas == NULL) {
idalloc(astats);
idalloc(tsd, astats);
return (true);
}
@ -585,8 +590,8 @@ ctl_grow(void)
sizeof(ctl_arena_stats_t));
memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) {
idalloc(tarenas);
idalloc(astats);
idalloc(tsd, tarenas);
idalloc(tsd, astats);
return (true);
}
/* Swap merged stats to their new location. */
@ -623,7 +628,7 @@ ctl_grow(void)
* base_alloc()).
*/
if (ctl_stats.narenas != narenas_auto)
idalloc(arenas_old);
idalloc(tsd, arenas_old);
}
ctl_stats.arenas = astats;
ctl_stats.narenas++;
@ -1105,6 +1110,31 @@ label_return: \
return (ret); \
}
#define CTL_TSD_RO_NL_CGEN(c, n, m, t) \
static int \
n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
void *newp, size_t newlen) \
{ \
int ret; \
t oldval; \
tsd_t *tsd; \
\
if ((c) == false) \
return (ENOENT); \
READONLY(); \
tsd = tsd_tryget(); \
if (tsd == NULL) { \
ret = EAGAIN; \
goto label_return; \
} \
oldval = (m(tsd)); \
READ(oldval, t); \
\
ret = 0; \
label_return: \
return (ret); \
}
#define CTL_RO_BOOL_CONFIG_GEN(n) \
static int \
n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
@ -1194,10 +1224,15 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
void *newp, size_t newlen)
{
int ret;
tsd_t *tsd;
unsigned newind, oldind;
tsd = tsd_tryget();
if (tsd == NULL)
return (EAGAIN);
malloc_mutex_lock(&ctl_mtx);
newind = oldind = choose_arena(NULL)->ind;
newind = oldind = choose_arena(tsd, NULL)->ind;
WRITE(newind, unsigned);
READ(oldind, unsigned);
if (newind != oldind) {
@ -1224,14 +1259,14 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
/* Set new arena association. */
if (config_tcache) {
tcache_t *tcache;
if ((uintptr_t)(tcache = *tcache_tsd_get()) >
(uintptr_t)TCACHE_STATE_MAX) {
tcache_t *tcache = tsd_tcache_get(tsd);
if (tcache != NULL) {
tcache_arena_dissociate(tcache);
tcache_arena_associate(tcache, arena);
}
}
arenas_tsd_set(&arena);
tsd_arena_set(tsd, arena);
}
ret = 0;
@ -1240,14 +1275,14 @@ label_return:
return (ret);
}
CTL_RO_NL_CGEN(config_stats, thread_allocated,
thread_allocated_tsd_get()->allocated, uint64_t)
CTL_RO_NL_CGEN(config_stats, thread_allocatedp,
&thread_allocated_tsd_get()->allocated, uint64_t *)
CTL_RO_NL_CGEN(config_stats, thread_deallocated,
thread_allocated_tsd_get()->deallocated, uint64_t)
CTL_RO_NL_CGEN(config_stats, thread_deallocatedp,
&thread_allocated_tsd_get()->deallocated, uint64_t *)
CTL_TSD_RO_NL_CGEN(config_stats, thread_allocated, tsd_thread_allocated_get,
uint64_t)
CTL_TSD_RO_NL_CGEN(config_stats, thread_allocatedp, tsd_thread_allocatedp_get,
uint64_t *)
CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocated, tsd_thread_deallocated_get,
uint64_t)
CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp,
tsd_thread_deallocatedp_get, uint64_t *)
static int
thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
@ -1305,11 +1340,20 @@ thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp,
oldname = prof_thread_name_get();
if (newp != NULL) {
tsd_t *tsd;
if (newlen != sizeof(const char *)) {
ret = EINVAL;
goto label_return;
}
if (prof_thread_name_set(*(const char **)newp)) {
tsd = tsd_tryget();
if (tsd == NULL) {
ret = EAGAIN;
goto label_return;
}
if (prof_thread_name_set(tsd, *(const char **)newp)) {
ret = EAGAIN;
goto label_return;
}
@ -1675,6 +1719,7 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
{
int ret;
size_t lg_sample = lg_prof_sample;
tsd_t *tsd;
if (config_prof == false)
return (ENOENT);
@ -1684,7 +1729,13 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
if (lg_sample >= (sizeof(uint64_t) << 3))
lg_sample = (sizeof(uint64_t) << 3) - 1;
prof_reset(lg_sample);
tsd = tsd_tryget();
if (tsd == NULL) {
ret = EAGAIN;
goto label_return;
}
prof_reset(tsd, lg_sample);
ret = 0;
label_return:

21
src/huge.c
View File

@ -13,14 +13,15 @@ static malloc_mutex_t huge_mtx;
static extent_tree_t huge;
void *
huge_malloc(arena_t *arena, size_t size, bool zero)
huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero)
{
return (huge_palloc(arena, size, chunksize, zero));
return (huge_palloc(tsd, arena, size, chunksize, zero));
}
void *
huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
bool zero)
{
void *ret;
size_t csize;
@ -45,7 +46,7 @@ huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
* it is possible to make correct junk/zero fill decisions below.
*/
is_zeroed = zero;
arena = choose_arena(arena);
arena = choose_arena(tsd, arena);
ret = arena_chunk_alloc_huge(arena, csize, alignment, &is_zeroed);
if (ret == NULL) {
base_node_dalloc(node);
@ -90,7 +91,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
}
void *
huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc)
{
void *ret;
@ -106,18 +107,18 @@ huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
* space and copying.
*/
if (alignment > chunksize)
ret = huge_palloc(arena, size + extra, alignment, zero);
ret = huge_palloc(tsd, arena, size + extra, alignment, zero);
else
ret = huge_malloc(arena, size + extra, zero);
ret = huge_malloc(tsd, arena, size + extra, zero);
if (ret == NULL) {
if (extra == 0)
return (NULL);
/* Try again, this time without extra. */
if (alignment > chunksize)
ret = huge_palloc(arena, size, alignment, zero);
ret = huge_palloc(tsd, arena, size, alignment, zero);
else
ret = huge_malloc(arena, size, zero);
ret = huge_malloc(tsd, arena, size, zero);
if (ret == NULL)
return (NULL);
@ -129,7 +130,7 @@ huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(ret, ptr, copysize);
iqalloc(ptr, try_tcache_dalloc);
iqalloc(tsd, ptr, try_tcache_dalloc);
return (ret);
}

348
src/jemalloc.c
View File

@ -5,8 +5,6 @@
/* Data. */
malloc_tsd_data(, arenas, arena_t *, NULL)
malloc_tsd_data(, thread_allocated, thread_allocated_t,
THREAD_ALLOCATED_INITIALIZER)
/* Runtime configuration options. */
const char *je_malloc_conf;
@ -142,7 +140,7 @@ arenas_extend(unsigned ind)
/* Slow path, called only by choose_arena(). */
arena_t *
choose_arena_hard(void)
choose_arena_hard(tsd_t *tsd)
{
arena_t *ret;
@ -196,11 +194,32 @@ choose_arena_hard(void)
malloc_mutex_unlock(&arenas_lock);
}
arenas_tsd_set(&ret);
tsd_arena_set(tsd, ret);
return (ret);
}
void
thread_allocated_cleanup(tsd_t *tsd)
{
/* Do nothing. */
}
void
thread_deallocated_cleanup(tsd_t *tsd)
{
/* Do nothing. */
}
void
arena_cleanup(tsd_t *tsd)
{
/* Do nothing. */
}
static void
stats_print_atexit(void)
{
@ -691,7 +710,11 @@ malloc_init_hard(void)
#endif
malloc_initializer = INITIALIZER;
malloc_tsd_boot();
if (malloc_tsd_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_prof)
prof_boot0();
@ -726,7 +749,7 @@ malloc_init_hard(void)
arena_boot();
if (config_tcache && tcache_boot0()) {
if (config_tcache && tcache_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
@ -759,27 +782,6 @@ malloc_init_hard(void)
return (true);
}
/* Initialize allocation counters before any allocations can occur. */
if (config_stats && thread_allocated_tsd_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (arenas_tsd_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_tcache && tcache_boot1()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_fill && quarantine_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_prof && prof_boot2()) {
malloc_mutex_unlock(&init_lock);
return (true);
@ -863,36 +865,36 @@ malloc_init_hard(void)
*/
static void *
imalloc_prof_sample(size_t usize, prof_tctx_t *tctx)
imalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
{
void *p;
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
p = imalloc(LARGE_MINCLASS);
p = imalloc(tsd, LARGE_MINCLASS);
if (p == NULL)
return (NULL);
arena_prof_promoted(p, usize);
} else
p = imalloc(usize);
p = imalloc(tsd, usize);
return (p);
}
JEMALLOC_ALWAYS_INLINE_C void *
imalloc_prof(size_t usize)
imalloc_prof(tsd_t *tsd, size_t usize)
{
void *p;
prof_tctx_t *tctx;
tctx = prof_alloc_prep(usize, true);
tctx = prof_alloc_prep(tsd, usize, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
p = imalloc_prof_sample(usize, tctx);
p = imalloc_prof_sample(tsd, usize, tctx);
else
p = imalloc(usize);
p = imalloc(tsd, usize);
if (p == NULL) {
prof_alloc_rollback(tctx, true);
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
prof_malloc(p, usize, tctx);
@ -901,32 +903,33 @@ imalloc_prof(size_t usize)
}
JEMALLOC_ALWAYS_INLINE_C void *
imalloc_body(size_t size, size_t *usize)
imalloc_body(size_t size, tsd_t **tsd, size_t *usize)
{
if (unlikely(malloc_init()))
if (unlikely(malloc_init()) || unlikely((*tsd = tsd_tryget()) == NULL))
return (NULL);
if (config_prof && opt_prof) {
*usize = s2u(size);
return (imalloc_prof(*usize));
return (imalloc_prof(*tsd, *usize));
}
if (config_stats || (config_valgrind && unlikely(in_valgrind)))
*usize = s2u(size);
return (imalloc(size));
return (imalloc(*tsd, size));
}
void *
je_malloc(size_t size)
{
void *ret;
tsd_t *tsd;
size_t usize JEMALLOC_CC_SILENCE_INIT(0);
if (size == 0)
size = 1;
ret = imalloc_body(size, &usize);
ret = imalloc_body(size, &tsd, &usize);
if (unlikely(ret == NULL)) {
if (config_xmalloc && unlikely(opt_xmalloc)) {
malloc_write("<jemalloc>: Error in malloc(): "
@ -937,7 +940,7 @@ je_malloc(size_t size)
}
if (config_stats && likely(ret != NULL)) {
assert(usize == isalloc(ret, config_prof));
thread_allocated_tsd_get()->allocated += usize;
*tsd_thread_allocatedp_get(tsd) += usize;
}
UTRACE(0, size, ret);
JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
@ -945,7 +948,8 @@ je_malloc(size_t size)
}
static void *
imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx)
imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
prof_tctx_t *tctx)
{
void *p;
@ -953,29 +957,29 @@ imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS);
p = imalloc(LARGE_MINCLASS);
p = imalloc(tsd, LARGE_MINCLASS);
if (p == NULL)
return (NULL);
arena_prof_promoted(p, usize);
} else
p = ipalloc(usize, alignment, false);
p = ipalloc(tsd, usize, alignment, false);
return (p);
}
JEMALLOC_ALWAYS_INLINE_C void *
imemalign_prof(size_t alignment, size_t usize)
imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
{
void *p;
prof_tctx_t *tctx;
tctx = prof_alloc_prep(usize, true);
tctx = prof_alloc_prep(tsd, usize, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
p = imemalign_prof_sample(alignment, usize, tctx);
p = imemalign_prof_sample(tsd, alignment, usize, tctx);
else
p = ipalloc(usize, alignment, false);
p = ipalloc(tsd, usize, alignment, false);
if (p == NULL) {
prof_alloc_rollback(tctx, true);
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
prof_malloc(p, usize, tctx);
@ -988,12 +992,13 @@ static int
imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
{
int ret;
tsd_t *tsd;
size_t usize;
void *result;
assert(min_alignment != 0);
if (unlikely(malloc_init())) {
if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) {
result = NULL;
goto label_oom;
} else {
@ -1020,9 +1025,9 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
}
if (config_prof && opt_prof)
result = imemalign_prof(alignment, usize);
result = imemalign_prof(tsd, alignment, usize);
else
result = ipalloc(usize, alignment, false);
result = ipalloc(tsd, usize, alignment, false);
if (unlikely(result == NULL))
goto label_oom;
}
@ -1032,7 +1037,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
label_return:
if (config_stats && likely(result != NULL)) {
assert(usize == isalloc(result, config_prof));
thread_allocated_tsd_get()->allocated += usize;
*tsd_thread_allocatedp_get(tsd) += usize;
}
UTRACE(0, size, result);
return (ret);
@ -1072,36 +1077,36 @@ je_aligned_alloc(size_t alignment, size_t size)
}
static void *
icalloc_prof_sample(size_t usize, prof_tctx_t *tctx)
icalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
{
void *p;
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
p = icalloc(LARGE_MINCLASS);
p = icalloc(tsd, LARGE_MINCLASS);
if (p == NULL)
return (NULL);
arena_prof_promoted(p, usize);
} else
p = icalloc(usize);
p = icalloc(tsd, usize);
return (p);
}
JEMALLOC_ALWAYS_INLINE_C void *
icalloc_prof(size_t usize)
icalloc_prof(tsd_t *tsd, size_t usize)
{
void *p;
prof_tctx_t *tctx;
tctx = prof_alloc_prep(usize, true);
tctx = prof_alloc_prep(tsd, usize, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
p = icalloc_prof_sample(usize, tctx);
p = icalloc_prof_sample(tsd, usize, tctx);
else
p = icalloc(usize);
p = icalloc(tsd, usize);
if (p == NULL) {
prof_alloc_rollback(tctx, true);
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
prof_malloc(p, usize, tctx);
@ -1113,10 +1118,11 @@ void *
je_calloc(size_t num, size_t size)
{
void *ret;
tsd_t *tsd;
size_t num_size;
size_t usize JEMALLOC_CC_SILENCE_INIT(0);
if (unlikely(malloc_init())) {
if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) {
num_size = 0;
ret = NULL;
goto label_return;
@ -1144,11 +1150,11 @@ je_calloc(size_t num, size_t size)
if (config_prof && opt_prof) {
usize = s2u(num_size);
ret = icalloc_prof(usize);
ret = icalloc_prof(tsd, usize);
} else {
if (config_stats || (config_valgrind && unlikely(in_valgrind)))
usize = s2u(num_size);
ret = icalloc(num_size);
ret = icalloc(tsd, num_size);
}
label_return:
@ -1162,7 +1168,7 @@ label_return:
}
if (config_stats && likely(ret != NULL)) {
assert(usize == isalloc(ret, config_prof));
thread_allocated_tsd_get()->allocated += usize;
*tsd_thread_allocatedp_get(tsd) += usize;
}
UTRACE(0, num_size, ret);
JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
@ -1170,44 +1176,44 @@ label_return:
}
static void *
irealloc_prof_sample(void *oldptr, size_t usize, prof_tctx_t *tctx)
irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t usize, prof_tctx_t *tctx)
{
void *p;
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
p = iralloc(oldptr, LARGE_MINCLASS, 0, false);
p = iralloc(tsd, oldptr, LARGE_MINCLASS, 0, false);
if (p == NULL)
return (NULL);
arena_prof_promoted(p, usize);
} else
p = iralloc(oldptr, usize, 0, false);
p = iralloc(tsd, oldptr, usize, 0, false);
return (p);
}
JEMALLOC_ALWAYS_INLINE_C void *
irealloc_prof(void *oldptr, size_t old_usize, size_t usize)
irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize)
{
void *p;
prof_tctx_t *old_tctx, *tctx;
old_tctx = prof_tctx_get(oldptr);
tctx = prof_alloc_prep(usize, true);
tctx = prof_alloc_prep(tsd, usize, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
p = irealloc_prof_sample(oldptr, usize, tctx);
p = irealloc_prof_sample(tsd, oldptr, usize, tctx);
else
p = iralloc(oldptr, usize, 0, false);
p = iralloc(tsd, oldptr, usize, 0, false);
if (p == NULL)
return (NULL);
prof_realloc(p, usize, tctx, true, old_usize, old_tctx);
prof_realloc(tsd, p, usize, tctx, true, old_usize, old_tctx);
return (p);
}
JEMALLOC_INLINE_C void
ifree(void *ptr, bool try_tcache)
ifree(tsd_t *tsd, void *ptr, bool try_tcache)
{
size_t usize;
UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
@ -1217,19 +1223,19 @@ ifree(void *ptr, bool try_tcache)
if (config_prof && opt_prof) {
usize = isalloc(ptr, config_prof);
prof_free(ptr, usize);
prof_free(tsd, ptr, usize);
} else if (config_stats || config_valgrind)
usize = isalloc(ptr, config_prof);
if (config_stats)
thread_allocated_tsd_get()->deallocated += usize;
if (config_stats && likely(tsd != NULL))
*tsd_thread_deallocatedp_get(tsd) += usize;
if (config_valgrind && unlikely(in_valgrind))
rzsize = p2rz(ptr);
iqalloc(ptr, try_tcache);
iqalloc(tsd, ptr, try_tcache);
JEMALLOC_VALGRIND_FREE(ptr, rzsize);
}
JEMALLOC_INLINE_C void
isfree(void *ptr, size_t usize, bool try_tcache)
isfree(tsd_t *tsd, void *ptr, size_t usize, bool try_tcache)
{
UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
@ -1237,12 +1243,12 @@ isfree(void *ptr, size_t usize, bool try_tcache)
assert(malloc_initialized || IS_INITIALIZER);
if (config_prof && opt_prof)
prof_free(ptr, usize);
if (config_stats)
thread_allocated_tsd_get()->deallocated += usize;
prof_free(tsd, ptr, usize);
if (config_stats && likely(tsd != NULL))
*tsd_thread_deallocatedp_get(tsd) += usize;
if (config_valgrind && unlikely(in_valgrind))
rzsize = p2rz(ptr);
isqalloc(ptr, usize, try_tcache);
isqalloc(tsd, ptr, usize, try_tcache);
JEMALLOC_VALGRIND_FREE(ptr, rzsize);
}
@ -1250,6 +1256,7 @@ void *
je_realloc(void *ptr, size_t size)
{
void *ret;
tsd_t *tsd;
size_t usize JEMALLOC_CC_SILENCE_INIT(0);
size_t old_usize = 0;
UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
@ -1258,7 +1265,8 @@ je_realloc(void *ptr, size_t size)
if (ptr != NULL) {
/* realloc(ptr, 0) is equivalent to free(ptr). */
UTRACE(ptr, 0, 0);
ifree(ptr, true);
tsd = tsd_tryget();
ifree(tsd, ptr, true);
return (NULL);
}
size = 1;
@ -1268,24 +1276,29 @@ je_realloc(void *ptr, size_t size)
assert(malloc_initialized || IS_INITIALIZER);
malloc_thread_init();
if ((tsd = tsd_tryget()) != NULL) {
if ((config_prof && opt_prof) || config_stats ||
(config_valgrind && unlikely(in_valgrind)))
old_usize = isalloc(ptr, config_prof);
if (config_valgrind && unlikely(in_valgrind))
old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize);
if (config_valgrind && unlikely(in_valgrind)) {
old_rzsize = config_prof ? p2rz(ptr) :
u2rz(old_usize);
}
if (config_prof && opt_prof) {
usize = s2u(size);
ret = irealloc_prof(ptr, old_usize, usize);
ret = irealloc_prof(tsd, ptr, old_usize, usize);
} else {
if (config_stats || (config_valgrind &&
unlikely(in_valgrind)))
usize = s2u(size);
ret = iralloc(ptr, size, 0, false);
ret = iralloc(tsd, ptr, size, 0, false);
}
} else
ret = NULL;
} else {
/* realloc(NULL, size) is equivalent to malloc(size). */
ret = imalloc_body(size, &usize);
ret = imalloc_body(size, &tsd, &usize);
}
if (unlikely(ret == NULL)) {
@ -1297,11 +1310,11 @@ je_realloc(void *ptr, size_t size)
set_errno(ENOMEM);
}
if (config_stats && likely(ret != NULL)) {
thread_allocated_t *ta;
assert(usize == isalloc(ret, config_prof));
ta = thread_allocated_tsd_get();
ta->allocated += usize;
ta->deallocated += old_usize;
if (tsd != NULL) {
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
}
}
UTRACE(ptr, size, ret);
JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize,
@ -1315,7 +1328,7 @@ je_free(void *ptr)
UTRACE(ptr, 0, 0);
if (likely(ptr != NULL))
ifree(ptr, true);
ifree(tsd_tryget(), ptr, true);
}
/*
@ -1425,50 +1438,52 @@ imallocx_flags_decode(size_t size, int flags, size_t *usize, size_t *alignment,
}
JEMALLOC_ALWAYS_INLINE_C void *
imallocx_flags(size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena)
imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
bool try_tcache, arena_t *arena)
{
if (alignment != 0)
return (ipalloct(usize, alignment, zero, try_tcache, arena));
if (alignment != 0) {
return (ipalloct(tsd, usize, alignment, zero, try_tcache,
arena));
}
if (zero)
return (icalloct(usize, try_tcache, arena));
return (imalloct(usize, try_tcache, arena));
return (icalloct(tsd, usize, try_tcache, arena));
return (imalloct(tsd, usize, try_tcache, arena));
}
JEMALLOC_ALWAYS_INLINE_C void *
imallocx_maybe_flags(size_t size, int flags, size_t usize, size_t alignment,
bool zero, bool try_tcache, arena_t *arena)
imallocx_maybe_flags(tsd_t *tsd, size_t size, int flags, size_t usize,
size_t alignment, bool zero, bool try_tcache, arena_t *arena)
{
if (likely(flags == 0))
return (imalloc(size));
return (imallocx_flags(usize, alignment, zero, try_tcache, arena));
return (imalloc(tsd, size));
return (imallocx_flags(tsd, usize, alignment, zero, try_tcache, arena));
}
static void *
imallocx_prof_sample(size_t size, int flags, size_t usize, size_t alignment,
bool zero, bool try_tcache, arena_t *arena)
imallocx_prof_sample(tsd_t *tsd, size_t size, int flags, size_t usize,
size_t alignment, bool zero, bool try_tcache, arena_t *arena)
{
void *p;
if (usize <= SMALL_MAXCLASS) {
assert(((alignment == 0) ? s2u(LARGE_MINCLASS) :
sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS);
p = imalloct(LARGE_MINCLASS, try_tcache, arena);
p = imalloct(tsd, LARGE_MINCLASS, try_tcache, arena);
if (p == NULL)
return (NULL);
arena_prof_promoted(p, usize);
} else {
p = imallocx_maybe_flags(size, flags, usize, alignment, zero,
try_tcache, arena);
p = imallocx_maybe_flags(tsd, size, flags, usize, alignment,
zero, try_tcache, arena);
}
return (p);
}
JEMALLOC_ALWAYS_INLINE_C void *
imallocx_prof(size_t size, int flags, size_t *usize)
imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize)
{
void *p;
size_t alignment;
@ -1479,17 +1494,17 @@ imallocx_prof(size_t size, int flags, size_t *usize)
imallocx_flags_decode(size, flags, usize, &alignment, &zero,
&try_tcache, &arena);
tctx = prof_alloc_prep(*usize, true);
tctx = prof_alloc_prep(tsd, *usize, true);
if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
p = imallocx_maybe_flags(size, flags, *usize, alignment, zero,
try_tcache, arena);
p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment,
zero, try_tcache, arena);
} else if ((uintptr_t)tctx > (uintptr_t)1U) {
p = imallocx_prof_sample(size, flags, *usize, alignment, zero,
try_tcache, arena);
p = imallocx_prof_sample(tsd, size, flags, *usize, alignment,
zero, try_tcache, arena);
} else
p = NULL;
if (unlikely(p == NULL)) {
prof_alloc_rollback(tctx, true);
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
prof_malloc(p, *usize, tctx);
@ -1498,7 +1513,7 @@ imallocx_prof(size_t size, int flags, size_t *usize)
}
JEMALLOC_ALWAYS_INLINE_C void *
imallocx_no_prof(size_t size, int flags, size_t *usize)
imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize)
{
size_t alignment;
bool zero;
@ -1508,35 +1523,39 @@ imallocx_no_prof(size_t size, int flags, size_t *usize)
if (likely(flags == 0)) {
if (config_stats || (config_valgrind && unlikely(in_valgrind)))
*usize = s2u(size);
return (imalloc(size));
return (imalloc(tsd, size));
}
imallocx_flags_decode_hard(size, flags, usize, &alignment, &zero,
&try_tcache, &arena);
return (imallocx_flags(*usize, alignment, zero, try_tcache, arena));
return (imallocx_flags(tsd, *usize, alignment, zero, try_tcache,
arena));
}
void *
je_mallocx(size_t size, int flags)
{
tsd_t *tsd;
void *p;
size_t usize;
assert(size != 0);
if (unlikely(malloc_init()))
if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL))
goto label_oom;
if (config_prof && opt_prof)
p = imallocx_prof(size, flags, &usize);
p = imallocx_prof(tsd, size, flags, &usize);
else
p = imallocx_no_prof(size, flags, &usize);
p = imallocx_no_prof(tsd, size, flags, &usize);
if (unlikely(p == NULL))
goto label_oom;
if (config_stats) {
tsd_t *tsd = tsd_tryget();
assert(usize == isalloc(p, config_prof));
thread_allocated_tsd_get()->allocated += usize;
if (tsd != NULL)
*tsd_thread_allocatedp_get(tsd) += usize;
}
UTRACE(0, size, p);
JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags));
@ -1551,47 +1570,47 @@ label_oom:
}
static void *
irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize,
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena,
prof_tctx_t *tctx)
irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t size, size_t alignment,
size_t usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena, prof_tctx_t *tctx)
{
void *p;
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
p = iralloct(oldptr, LARGE_MINCLASS, alignment, zero,
p = iralloct(tsd, oldptr, LARGE_MINCLASS, alignment, zero,
try_tcache_alloc, try_tcache_dalloc, arena);
if (p == NULL)
return (NULL);
arena_prof_promoted(p, usize);
} else {
p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc,
try_tcache_dalloc, arena);
p = iralloct(tsd, oldptr, size, alignment, zero,
try_tcache_alloc, try_tcache_dalloc, arena);
}
return (p);
}
JEMALLOC_ALWAYS_INLINE_C void *
irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena)
irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size,
size_t alignment, size_t *usize, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc, arena_t *arena)
{
void *p;
prof_tctx_t *old_tctx, *tctx;
old_tctx = prof_tctx_get(oldptr);
tctx = prof_alloc_prep(*usize, false);
tctx = prof_alloc_prep(tsd, *usize, false);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero,
try_tcache_alloc, try_tcache_dalloc, arena, tctx);
p = irallocx_prof_sample(tsd, oldptr, size, alignment, *usize,
zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx);
} else {
p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc,
try_tcache_dalloc, arena);
p = iralloct(tsd, oldptr, size, alignment, zero,
try_tcache_alloc, try_tcache_dalloc, arena);
}
if (unlikely(p == NULL)) {
prof_alloc_rollback(tctx, false);
prof_alloc_rollback(tsd, tctx, false);
return (NULL);
}
@ -1606,7 +1625,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
*/
*usize = isalloc(p, config_prof);
}
prof_realloc(p, *usize, tctx, false, old_usize, old_tctx);
prof_realloc(tsd, p, *usize, tctx, false, old_usize, old_tctx);
return (p);
}
@ -1615,6 +1634,7 @@ void *
je_rallocx(void *ptr, size_t size, int flags)
{
void *p;
tsd_t *tsd;
size_t usize;
UNUSED size_t old_usize JEMALLOC_CC_SILENCE_INIT(0);
UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
@ -1628,6 +1648,9 @@ je_rallocx(void *ptr, size_t size, int flags)
assert(malloc_initialized || IS_INITIALIZER);
malloc_thread_init();
if (unlikely((tsd = tsd_tryget()) == NULL))
goto label_oom;
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
arena_chunk_t *chunk;
@ -1651,12 +1674,12 @@ je_rallocx(void *ptr, size_t size, int flags)
if (config_prof && opt_prof) {
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
assert(usize != 0);
p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero,
try_tcache_alloc, try_tcache_dalloc, arena);
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
zero, try_tcache_alloc, try_tcache_dalloc, arena);
if (unlikely(p == NULL))
goto label_oom;
} else {
p = iralloct(ptr, size, alignment, zero, try_tcache_alloc,
p = iralloct(tsd, ptr, size, alignment, zero, try_tcache_alloc,
try_tcache_dalloc, arena);
if (unlikely(p == NULL))
goto label_oom;
@ -1665,10 +1688,8 @@ je_rallocx(void *ptr, size_t size, int flags)
}
if (config_stats) {
thread_allocated_t *ta;
ta = thread_allocated_tsd_get();
ta->allocated += usize;
ta->deallocated += old_usize;
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
}
UTRACE(ptr, size, p);
JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize,
@ -1724,8 +1745,8 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
}
JEMALLOC_ALWAYS_INLINE_C size_t
ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
size_t alignment, bool zero, arena_t *arena)
ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
size_t extra, size_t alignment, bool zero, arena_t *arena)
{
size_t max_usize, usize;
prof_tctx_t *old_tctx, *tctx;
@ -1739,7 +1760,7 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
*/
max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra,
alignment);
tctx = prof_alloc_prep(max_usize, false);
tctx = prof_alloc_prep(tsd, max_usize, false);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
alignment, zero, max_usize, arena, tctx);
@ -1748,10 +1769,10 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
zero, arena);
}
if (unlikely(usize == old_usize)) {
prof_alloc_rollback(tctx, false);
prof_alloc_rollback(tsd, tctx, false);
return (usize);
}
prof_realloc(ptr, usize, tctx, false, old_usize, old_tctx);
prof_realloc(tsd, ptr, usize, tctx, false, old_usize, old_tctx);
return (usize);
}
@ -1759,6 +1780,7 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
size_t
je_xallocx(void *ptr, size_t size, size_t extra, int flags)
{
tsd_t *tsd;
size_t usize, old_usize;
UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
size_t alignment = MALLOCX_ALIGN_GET(flags);
@ -1778,12 +1800,16 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
arena = NULL;
old_usize = isalloc(ptr, config_prof);
if (unlikely((tsd = tsd_tryget()) == NULL)) {
usize = old_usize;
goto label_not_resized;
}
if (config_valgrind && unlikely(in_valgrind))
old_rzsize = u2rz(old_usize);
if (config_prof && opt_prof) {
usize = ixallocx_prof(ptr, old_usize, size, extra, alignment,
zero, arena);
usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
alignment, zero, arena);
} else {
usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
zero, arena);
@ -1792,10 +1818,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
goto label_not_resized;
if (config_stats) {
thread_allocated_t *ta;
ta = thread_allocated_tsd_get();
ta->allocated += usize;
ta->deallocated += old_usize;
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
}
JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize,
old_rzsize, false, zero);
@ -1839,7 +1863,7 @@ je_dallocx(void *ptr, int flags)
try_tcache = true;
UTRACE(ptr, 0, 0);
ifree(ptr, try_tcache);
ifree(tsd_tryget(), ptr, try_tcache);
}
JEMALLOC_ALWAYS_INLINE_C size_t
@ -1875,7 +1899,7 @@ je_sdallocx(void *ptr, size_t size, int flags)
try_tcache = true;
UTRACE(ptr, 0, 0);
isfree(ptr, usize, try_tcache);
isfree(tsd_tryget(), ptr, usize, try_tcache);
}
size_t
@ -2072,9 +2096,9 @@ a0alloc(size_t size, bool zero)
size = 1;
if (size <= arena_maxclass)
return (arena_malloc(arenas[0], size, zero, false));
return (arena_malloc(NULL, arenas[0], size, zero, false));
else
return (huge_malloc(NULL, size, zero));
return (huge_malloc(NULL, arenas[0], size, zero));
}
void *
@ -2101,7 +2125,7 @@ a0free(void *ptr)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
arena_dalloc(chunk, ptr, false);
arena_dalloc(NULL, chunk, ptr, false);
else
huge_dalloc(ptr);
}

244
src/prof.c
View File

@ -14,8 +14,6 @@
/******************************************************************************/
/* Data. */
malloc_tsd_data(, prof_tdata, prof_tdata_t *, NULL)
bool opt_prof = false;
bool opt_prof_active = true;
size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
@ -102,9 +100,9 @@ static bool prof_booted = false;
*/
static bool prof_tctx_should_destroy(prof_tctx_t *tctx);
static void prof_tctx_destroy(prof_tctx_t *tctx);
static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
static bool prof_tdata_should_destroy(prof_tdata_t *tdata);
static void prof_tdata_destroy(prof_tdata_t *tdata);
static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata);
/******************************************************************************/
/* Red-black trees. */
@ -151,7 +149,7 @@ rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
/******************************************************************************/
void
prof_alloc_rollback(prof_tctx_t *tctx, bool updated)
prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
{
prof_tdata_t *tdata;
@ -164,8 +162,8 @@ prof_alloc_rollback(prof_tctx_t *tctx, bool updated)
* potential for sample bias is minimal except in contrived
* programs.
*/
tdata = prof_tdata_get(true);
if ((uintptr_t)tdata > (uintptr_t)PROF_TDATA_STATE_MAX)
tdata = prof_tdata_get(tsd, true);
if (tdata != NULL)
prof_sample_threshold_update(tctx->tdata);
}
@ -173,7 +171,7 @@ prof_alloc_rollback(prof_tctx_t *tctx, bool updated)
malloc_mutex_lock(tctx->tdata->lock);
tctx->prepared = false;
if (prof_tctx_should_destroy(tctx))
prof_tctx_destroy(tctx);
prof_tctx_destroy(tsd, tctx);
else
malloc_mutex_unlock(tctx->tdata->lock);
}
@ -195,7 +193,7 @@ prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) {
}
void
prof_free_sampled_object(size_t usize, prof_tctx_t *tctx)
prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
{
malloc_mutex_lock(tctx->tdata->lock);
@ -205,7 +203,7 @@ prof_free_sampled_object(size_t usize, prof_tctx_t *tctx)
tctx->cnts.curbytes -= usize;
if (prof_tctx_should_destroy(tctx))
prof_tctx_destroy(tctx);
prof_tctx_destroy(tsd, tctx);
else
malloc_mutex_unlock(tctx->tdata->lock);
}
@ -494,13 +492,13 @@ prof_tdata_mutex_choose(uint64_t thr_uid)
}
static prof_gctx_t *
prof_gctx_create(prof_bt_t *bt)
prof_gctx_create(tsd_t *tsd, prof_bt_t *bt)
{
/*
* Create a single allocation that has space for vec of length bt->len.
*/
prof_gctx_t *gctx = (prof_gctx_t *)imalloc(offsetof(prof_gctx_t, vec) +
(bt->len * sizeof(void *)));
prof_gctx_t *gctx = (prof_gctx_t *)imalloc(tsd, offsetof(prof_gctx_t,
vec) + (bt->len * sizeof(void *)));
if (gctx == NULL)
return (NULL);
gctx->lock = prof_gctx_mutex_choose();
@ -518,7 +516,7 @@ prof_gctx_create(prof_bt_t *bt)
}
static void
prof_gctx_maybe_destroy(prof_gctx_t *gctx, prof_tdata_t *tdata)
prof_gctx_maybe_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata)
{
cassert(config_prof);
@ -534,12 +532,12 @@ prof_gctx_maybe_destroy(prof_gctx_t *gctx, prof_tdata_t *tdata)
malloc_mutex_lock(gctx->lock);
if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
/* Remove gctx from bt2gctx. */
if (ckh_remove(&bt2gctx, &gctx->bt, NULL, NULL))
if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL))
not_reached();
prof_leave(tdata);
/* Destroy gctx. */
malloc_mutex_unlock(gctx->lock);
idalloc(gctx);
idalloc(tsd, gctx);
} else {
/*
* Compensate for increment in prof_tctx_destroy() or
@ -580,7 +578,7 @@ prof_gctx_should_destroy(prof_gctx_t *gctx)
/* tctx->tdata->lock is held upon entry, and released before return. */
static void
prof_tctx_destroy(prof_tctx_t *tctx)
prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
{
prof_tdata_t *tdata = tctx->tdata;
prof_gctx_t *gctx = tctx->gctx;
@ -592,7 +590,7 @@ prof_tctx_destroy(prof_tctx_t *tctx)
assert(tctx->cnts.accumobjs == 0);
assert(tctx->cnts.accumbytes == 0);
ckh_remove(&tdata->bt2tctx, &gctx->bt, NULL, NULL);
ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
destroy_tdata = prof_tdata_should_destroy(tdata);
malloc_mutex_unlock(tdata->lock);
@ -618,17 +616,17 @@ prof_tctx_destroy(prof_tctx_t *tctx)
destroy_gctx = false;
malloc_mutex_unlock(gctx->lock);
if (destroy_gctx)
prof_gctx_maybe_destroy(gctx, tdata);
prof_gctx_maybe_destroy(tsd, gctx, tdata);
if (destroy_tdata)
prof_tdata_destroy(tdata);
prof_tdata_destroy(tsd, tdata);
idalloc(tctx);
idalloc(tsd, tctx);
}
static bool
prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey,
prof_gctx_t **p_gctx, bool *p_new_gctx)
prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx)
{
union {
prof_gctx_t *p;
@ -643,16 +641,16 @@ prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey,
prof_enter(tdata);
if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
/* bt has never been seen before. Insert it. */
gctx.p = prof_gctx_create(bt);
gctx.p = prof_gctx_create(tsd, bt);
if (gctx.v == NULL) {
prof_leave(tdata);
return (true);
}
btkey.p = &gctx.p->bt;
if (ckh_insert(&bt2gctx, btkey.v, gctx.v)) {
if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
/* OOM. */
prof_leave(tdata);
idalloc(gctx.v);
idalloc(tsd, gctx.v);
return (true);
}
new_gctx = true;
@ -675,7 +673,7 @@ prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey,
}
prof_tctx_t *
prof_lookup(prof_bt_t *bt)
prof_lookup(tsd_t *tsd, prof_bt_t *bt)
{
union {
prof_tctx_t *p;
@ -686,8 +684,8 @@ prof_lookup(prof_bt_t *bt)
cassert(config_prof);
tdata = prof_tdata_get(false);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL)
return (NULL);
malloc_mutex_lock(tdata->lock);
@ -704,15 +702,15 @@ prof_lookup(prof_bt_t *bt)
* This thread's cache lacks bt. Look for it in the global
* cache.
*/
if (prof_lookup_global(bt, tdata, &btkey, &gctx,
if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
&new_gctx))
return (NULL);
/* Link a prof_tctx_t into gctx for this thread. */
ret.v = imalloc(sizeof(prof_tctx_t));
ret.v = imalloc(tsd, sizeof(prof_tctx_t));
if (ret.p == NULL) {
if (new_gctx)
prof_gctx_maybe_destroy(gctx, tdata);
prof_gctx_maybe_destroy(tsd, gctx, tdata);
return (NULL);
}
ret.p->tdata = tdata;
@ -721,12 +719,12 @@ prof_lookup(prof_bt_t *bt)
ret.p->prepared = true;
ret.p->state = prof_tctx_state_nominal;
malloc_mutex_lock(tdata->lock);
error = ckh_insert(&tdata->bt2tctx, btkey, ret.v);
error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
malloc_mutex_unlock(tdata->lock);
if (error) {
if (new_gctx)
prof_gctx_maybe_destroy(gctx, tdata);
idalloc(ret.v);
prof_gctx_maybe_destroy(tsd, gctx, tdata);
idalloc(tsd, ret.v);
return (NULL);
}
malloc_mutex_lock(gctx->lock);
@ -798,10 +796,13 @@ size_t
prof_bt_count(void)
{
size_t bt_count;
tsd_t *tsd;
prof_tdata_t *tdata;
tdata = prof_tdata_get(false);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
if ((tsd = tsd_tryget()) == NULL)
return (0);
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL)
return (0);
prof_enter(tdata);
@ -989,6 +990,7 @@ static prof_tctx_t *
prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
{
prof_tctx_t *ret;
tsd_t *tsd = (tsd_t *)arg;
switch (tctx->state) {
case prof_tctx_state_nominal:
@ -1000,7 +1002,7 @@ prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
case prof_tctx_state_purgatory:
ret = tctx_tree_next(tctxs, tctx);
tctx_tree_remove(tctxs, tctx);
idalloc(tctx);
idalloc(tsd, tctx);
goto label_return;
default:
not_reached();
@ -1049,7 +1051,8 @@ prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
static prof_gctx_t *
prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
{
prof_tdata_t *tdata = (prof_tdata_t *)arg;
tsd_t *tsd = (tsd_t *)arg;
prof_tdata_t *tdata = prof_tdata_get(tsd, false);
prof_tctx_t *next;
bool destroy_gctx;
@ -1057,13 +1060,13 @@ prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
next = NULL;
do {
next = tctx_tree_iter(&gctx->tctxs, next, prof_tctx_finish_iter,
NULL);
tsd);
} while (next != NULL);
gctx->nlimbo--;
destroy_gctx = prof_gctx_should_destroy(gctx);
malloc_mutex_unlock(gctx->lock);
if (destroy_gctx)
prof_gctx_maybe_destroy(gctx, tdata);
prof_gctx_maybe_destroy(tsd, gctx, tdata);
return (NULL);
}
@ -1277,7 +1280,7 @@ label_return:
}
static bool
prof_dump(bool propagate_err, const char *filename, bool leakcheck)
prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
{
prof_tdata_t *tdata;
prof_cnt_t cnt_all;
@ -1291,8 +1294,8 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
cassert(config_prof);
tdata = prof_tdata_get(false);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL)
return (true);
malloc_mutex_lock(&prof_dump_mtx);
@ -1341,7 +1344,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
if (prof_dump_close(propagate_err))
goto label_open_close_error;
gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tdata);
gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tsd);
malloc_mutex_unlock(&prof_dump_mtx);
if (leakcheck)
@ -1351,7 +1354,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
label_write_error:
prof_dump_close(propagate_err);
label_open_close_error:
gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tdata);
gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tsd);
malloc_mutex_unlock(&prof_dump_mtx);
return (true);
}
@ -1381,24 +1384,28 @@ prof_dump_filename(char *filename, char v, uint64_t vseq)
static void
prof_fdump(void)
{
tsd_t *tsd;
char filename[DUMP_FILENAME_BUFSIZE];
cassert(config_prof);
if (prof_booted == false)
return;
if ((tsd = tsd_tryget()) == NULL)
return;
if (opt_prof_final && opt_prof_prefix[0] != '\0') {
malloc_mutex_lock(&prof_dump_seq_mtx);
prof_dump_filename(filename, 'f', VSEQ_INVALID);
malloc_mutex_unlock(&prof_dump_seq_mtx);
prof_dump(false, filename, opt_prof_leak);
prof_dump(tsd, false, filename, opt_prof_leak);
}
}
void
prof_idump(void)
{
tsd_t *tsd;
prof_tdata_t *tdata;
char filename[PATH_MAX + 1];
@ -1406,8 +1413,10 @@ prof_idump(void)
if (prof_booted == false)
return;
tdata = prof_tdata_get(false);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
if ((tsd = tsd_tryget()) == NULL)
return;
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL)
return;
if (tdata->enq) {
tdata->enq_idump = true;
@ -1419,19 +1428,22 @@ prof_idump(void)
prof_dump_filename(filename, 'i', prof_dump_iseq);
prof_dump_iseq++;
malloc_mutex_unlock(&prof_dump_seq_mtx);
prof_dump(false, filename, false);
prof_dump(tsd, false, filename, false);
}
}
bool
prof_mdump(const char *filename)
{
tsd_t *tsd;
char filename_buf[DUMP_FILENAME_BUFSIZE];
cassert(config_prof);
if (opt_prof == false || prof_booted == false)
return (true);
if ((tsd = tsd_tryget()) == NULL)
return (true);
if (filename == NULL) {
/* No filename specified, so automatically generate one. */
@ -1443,12 +1455,13 @@ prof_mdump(const char *filename)
malloc_mutex_unlock(&prof_dump_seq_mtx);
filename = filename_buf;
}
return (prof_dump(true, filename, false));
return (prof_dump(tsd, true, filename, false));
}
void
prof_gdump(void)
{
tsd_t *tsd;
prof_tdata_t *tdata;
char filename[DUMP_FILENAME_BUFSIZE];
@ -1456,8 +1469,10 @@ prof_gdump(void)
if (prof_booted == false)
return;
tdata = prof_tdata_get(false);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
if ((tsd = tsd_tryget()) == NULL)
return;
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL)
return;
if (tdata->enq) {
tdata->enq_gdump = true;
@ -1469,7 +1484,7 @@ prof_gdump(void)
prof_dump_filename(filename, 'u', prof_dump_useq);
prof_dump_useq++;
malloc_mutex_unlock(&prof_dump_seq_mtx);
prof_dump(false, filename, false);
prof_dump(tsd, false, filename, false);
}
}
@ -1510,14 +1525,14 @@ prof_thr_uid_alloc(void)
}
static prof_tdata_t *
prof_tdata_init_impl(uint64_t thr_uid)
prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid)
{
prof_tdata_t *tdata;
cassert(config_prof);
/* Initialize an empty cache for this thread. */
tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t));
tdata = (prof_tdata_t *)imalloc(tsd, sizeof(prof_tdata_t));
if (tdata == NULL)
return (NULL);
@ -1526,9 +1541,9 @@ prof_tdata_init_impl(uint64_t thr_uid)
tdata->thread_name = NULL;
tdata->state = prof_tdata_state_attached;
if (ckh_new(&tdata->bt2tctx, PROF_CKH_MINITEMS,
if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS,
prof_bt_hash, prof_bt_keycomp)) {
idalloc(tdata);
idalloc(tsd, tdata);
return (NULL);
}
@ -1542,8 +1557,6 @@ prof_tdata_init_impl(uint64_t thr_uid)
tdata->dumping = false;
tdata->active = true;
prof_tdata_tsd_set(&tdata);
malloc_mutex_lock(&tdatas_mtx);
tdata_tree_insert(&tdatas, tdata);
malloc_mutex_unlock(&tdatas_mtx);
@ -1552,17 +1565,17 @@ prof_tdata_init_impl(uint64_t thr_uid)
}
prof_tdata_t *
prof_tdata_init(void)
prof_tdata_init(tsd_t *tsd)
{
return (prof_tdata_init_impl(prof_thr_uid_alloc()));
return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc()));
}
prof_tdata_t *
prof_tdata_reinit(prof_tdata_t *tdata)
prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata)
{
return (prof_tdata_init_impl(tdata->thr_uid));
return (prof_tdata_init_impl(tsd, tdata->thr_uid));
}
/* tdata->lock must be held. */
@ -1578,7 +1591,7 @@ prof_tdata_should_destroy(prof_tdata_t *tdata)
}
static void
prof_tdata_destroy(prof_tdata_t *tdata)
prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata)
{
assert(prof_tdata_should_destroy(tdata));
@ -1588,13 +1601,14 @@ prof_tdata_destroy(prof_tdata_t *tdata)
malloc_mutex_unlock(&tdatas_mtx);
if (tdata->thread_name != NULL)
idalloc(tdata->thread_name);
ckh_delete(&tdata->bt2tctx);
idalloc(tdata);
idalloc(tsd, tdata->thread_name);
ckh_delete(tsd, &tdata->bt2tctx);
idalloc(tsd, tdata);
}
static void
prof_tdata_state_transition(prof_tdata_t *tdata, prof_tdata_state_t state)
prof_tdata_state_transition(tsd_t *tsd, prof_tdata_t *tdata,
prof_tdata_state_t state)
{
bool destroy_tdata;
@ -1606,33 +1620,34 @@ prof_tdata_state_transition(prof_tdata_t *tdata, prof_tdata_state_t state)
destroy_tdata = false;
malloc_mutex_unlock(tdata->lock);
if (destroy_tdata)
prof_tdata_destroy(tdata);
prof_tdata_destroy(tsd, tdata);
}
static void
prof_tdata_detach(prof_tdata_t *tdata)
prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
{
prof_tdata_state_transition(tdata, prof_tdata_state_detached);
prof_tdata_state_transition(tsd, tdata, prof_tdata_state_detached);
}
static void
prof_tdata_expire(prof_tdata_t *tdata)
prof_tdata_expire(tsd_t *tsd, prof_tdata_t *tdata)
{
prof_tdata_state_transition(tdata, prof_tdata_state_expired);
prof_tdata_state_transition(tsd, tdata, prof_tdata_state_expired);
}
static prof_tdata_t *
prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
{
tsd_t *tsd = (tsd_t *)arg;
prof_tdata_expire(tdata);
prof_tdata_expire(tsd, tdata);
return (NULL);
}
void
prof_reset(size_t lg_sample)
prof_reset(tsd_t *tsd, size_t lg_sample)
{
assert(lg_sample < (sizeof(uint64_t) << 3));
@ -1641,69 +1656,58 @@ prof_reset(size_t lg_sample)
malloc_mutex_lock(&tdatas_mtx);
lg_prof_sample = lg_sample;
tdata_tree_iter(&tdatas, NULL, prof_tdata_reset_iter, NULL);
tdata_tree_iter(&tdatas, NULL, prof_tdata_reset_iter, tsd);
malloc_mutex_unlock(&tdatas_mtx);
malloc_mutex_unlock(&prof_dump_mtx);
}
void
prof_tdata_cleanup(void *arg)
prof_tdata_cleanup(tsd_t *tsd)
{
prof_tdata_t *tdata = *(prof_tdata_t **)arg;
prof_tdata_t *tdata;
cassert(config_prof);
if (!config_prof)
return;
if (tdata == PROF_TDATA_STATE_REINCARNATED) {
/*
* Another destructor deallocated memory after this destructor
* was called. Reset tdata to PROF_TDATA_STATE_PURGATORY in
* order to receive another callback.
*/
tdata = PROF_TDATA_STATE_PURGATORY;
prof_tdata_tsd_set(&tdata);
} else if (tdata == PROF_TDATA_STATE_PURGATORY) {
/*
* The previous time this destructor was called, we set the key
* to PROF_TDATA_STATE_PURGATORY so that other destructors
* wouldn't cause re-creation of the tdata. This time, do
* nothing, so that the destructor will not be called again.
*/
} else if (tdata != NULL) {
prof_tdata_detach(tdata);
tdata = PROF_TDATA_STATE_PURGATORY;
prof_tdata_tsd_set(&tdata);
}
tdata = tsd_prof_tdata_get(tsd);
if (tdata != NULL)
prof_tdata_detach(tsd, tdata);
}
const char *
prof_thread_name_get(void)
{
prof_tdata_t *tdata = prof_tdata_get(true);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
tsd_t *tsd;
prof_tdata_t *tdata;
if ((tsd = tsd_tryget()) == NULL)
return (NULL);
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL)
return (NULL);
return (tdata->thread_name);
}
bool
prof_thread_name_set(const char *thread_name)
prof_thread_name_set(tsd_t *tsd, const char *thread_name)
{
prof_tdata_t *tdata;
size_t size;
char *s;
tdata = prof_tdata_get(true);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL)
return (true);
size = strlen(thread_name) + 1;
s = imalloc(size);
s = imalloc(tsd, size);
if (s == NULL)
return (true);
memcpy(s, thread_name, size);
if (tdata->thread_name != NULL)
idalloc(tdata->thread_name);
idalloc(tsd, tdata->thread_name);
tdata->thread_name = s;
return (false);
}
@ -1711,8 +1715,13 @@ prof_thread_name_set(const char *thread_name)
bool
prof_thread_active_get(void)
{
prof_tdata_t *tdata = prof_tdata_get(true);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
tsd_t *tsd;
prof_tdata_t *tdata;
if ((tsd = tsd_tryget()) == NULL)
return (false);
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL)
return (false);
return (tdata->active);
}
@ -1720,10 +1729,13 @@ prof_thread_active_get(void)
bool
prof_thread_active_set(bool active)
{
tsd_t *tsd;
prof_tdata_t *tdata;
tdata = prof_tdata_get(true);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
if ((tsd = tsd_tryget()) == NULL)
return (true);
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL)
return (true);
tdata->active = active;
return (false);
@ -1772,20 +1784,18 @@ prof_boot2(void)
cassert(config_prof);
if (opt_prof) {
tsd_t *tsd;
unsigned i;
lg_prof_sample = opt_lg_prof_sample;
if (ckh_new(&bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
if ((tsd = tsd_tryget()) == NULL)
return (true);
if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
prof_bt_keycomp))
return (true);
if (malloc_mutex_init(&bt2gctx_mtx))
return (true);
if (prof_tdata_tsd_boot()) {
malloc_write(
"<jemalloc>: Error in pthread_key_create()\n");
abort();
}
tdata_tree_new(&tdatas);
if (malloc_mutex_init(&tdatas_mtx))

99
src/quarantine.c
View File

@ -9,26 +9,22 @@
#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2)
#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY
/******************************************************************************/
/* Data. */
malloc_tsd_data(, quarantine, quarantine_t *, NULL)
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static quarantine_t *quarantine_grow(quarantine_t *quarantine);
static void quarantine_drain_one(quarantine_t *quarantine);
static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
static quarantine_t *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine);
static void quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine);
static void quarantine_drain(tsd_t *tsd, quarantine_t *quarantine,
size_t upper_bound);
/******************************************************************************/
quarantine_t *
quarantine_init(size_t lg_maxobjs)
quarantine_init(tsd_t *tsd, size_t lg_maxobjs)
{
quarantine_t *quarantine;
quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
quarantine = (quarantine_t *)imalloc(tsd, offsetof(quarantine_t, objs) +
((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)));
if (quarantine == NULL)
return (NULL);
@ -37,19 +33,17 @@ quarantine_init(size_t lg_maxobjs)
quarantine->first = 0;
quarantine->lg_maxobjs = lg_maxobjs;
quarantine_tsd_set(&quarantine);
return (quarantine);
}
static quarantine_t *
quarantine_grow(quarantine_t *quarantine)
quarantine_grow(tsd_t *tsd, quarantine_t *quarantine)
{
quarantine_t *ret;
ret = quarantine_init(quarantine->lg_maxobjs + 1);
ret = quarantine_init(tsd, quarantine->lg_maxobjs + 1);
if (ret == NULL) {
quarantine_drain_one(quarantine);
quarantine_drain_one(tsd, quarantine);
return (quarantine);
}
@ -71,17 +65,17 @@ quarantine_grow(quarantine_t *quarantine)
memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
sizeof(quarantine_obj_t));
}
idalloc(quarantine);
idalloc(tsd, quarantine);
return (ret);
}
static void
quarantine_drain_one(quarantine_t *quarantine)
quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine)
{
quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
assert(obj->usize == isalloc(obj->ptr, config_prof));
idalloc(obj->ptr);
idalloc(tsd, obj->ptr);
quarantine->curbytes -= obj->usize;
quarantine->curobjs--;
quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
@ -89,15 +83,15 @@ quarantine_drain_one(quarantine_t *quarantine)
}
static void
quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, size_t upper_bound)
{
while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
quarantine_drain_one(quarantine);
quarantine_drain_one(tsd, quarantine);
}
void
quarantine(void *ptr)
quarantine(tsd_t *tsd, void *ptr)
{
quarantine_t *quarantine;
size_t usize = isalloc(ptr, config_prof);
@ -105,17 +99,8 @@ quarantine(void *ptr)
cassert(config_fill);
assert(opt_quarantine);
quarantine = *quarantine_tsd_get();
if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
if (quarantine == QUARANTINE_STATE_PURGATORY) {
/*
* Make a note that quarantine() was called after
* quarantine_cleanup() was called.
*/
quarantine = QUARANTINE_STATE_REINCARNATED;
quarantine_tsd_set(&quarantine);
}
idalloc(ptr);
if ((quarantine = tsd_quarantine_get(tsd)) == NULL) {
idalloc(tsd, ptr);
return;
}
/*
@ -125,11 +110,11 @@ quarantine(void *ptr)
if (quarantine->curbytes + usize > opt_quarantine) {
size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
- usize : 0;
quarantine_drain(quarantine, upper_bound);
quarantine_drain(tsd, quarantine, upper_bound);
}
/* Grow the quarantine ring buffer if it's full. */
if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
quarantine = quarantine_grow(quarantine);
quarantine = quarantine_grow(tsd, quarantine);
/* quarantine_grow() must free a slot if it fails to grow. */
assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
/* Append ptr if its size doesn't exceed the quarantine size. */
@ -154,46 +139,22 @@ quarantine(void *ptr)
}
} else {
assert(quarantine->curbytes == 0);
idalloc(ptr);
idalloc(tsd, ptr);
}
}
void
quarantine_cleanup(void *arg)
quarantine_cleanup(tsd_t *tsd)
{
quarantine_t *quarantine = *(quarantine_t **)arg;
quarantine_t *quarantine;
if (quarantine == QUARANTINE_STATE_REINCARNATED) {
/*
* Another destructor deallocated memory after this destructor
* was called. Reset quarantine to QUARANTINE_STATE_PURGATORY
* in order to receive another callback.
*/
quarantine = QUARANTINE_STATE_PURGATORY;
quarantine_tsd_set(&quarantine);
} else if (quarantine == QUARANTINE_STATE_PURGATORY) {
/*
* The previous time this destructor was called, we set the key
* to QUARANTINE_STATE_PURGATORY so that other destructors
* wouldn't cause re-creation of the quarantine. This time, do
* nothing, so that the destructor will not be called again.
*/
} else if (quarantine != NULL) {
quarantine_drain(quarantine, 0);
idalloc(quarantine);
quarantine = QUARANTINE_STATE_PURGATORY;
quarantine_tsd_set(&quarantine);
if (!config_fill)
return;
quarantine = tsd_quarantine_get(tsd);
if (quarantine != NULL) {
quarantine_drain(tsd, quarantine, 0);
idalloc(tsd, quarantine);
tsd_quarantine_set(tsd, NULL);
}
}
bool
quarantine_boot(void)
{
cassert(config_fill);
if (quarantine_tsd_boot())
return (true);
return (false);
}

6
src/rtree.c
View File

@ -9,8 +9,10 @@ rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc)
assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3));
bits_per_level = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1;
bits_in_leaf = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1;
bits_per_level = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void
*)))) - 1;
bits_in_leaf = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE /
sizeof(uint8_t)))) - 1;
if (bits > bits_in_leaf) {
height = 1 + (bits - bits_in_leaf) / bits_per_level;
if ((height-1) * bits_per_level + bits_in_leaf != bits)

97
src/tcache.c
View File

@ -4,9 +4,6 @@
/******************************************************************************/
/* Data. */
malloc_tsd_data(, tcache, tcache_t *, NULL)
malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default)
bool opt_tcache = true;
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
@ -262,43 +259,14 @@ tcache_arena_dissociate(tcache_t *tcache)
}
tcache_t *
tcache_get_hard(tcache_t *tcache, bool create)
tcache_get_hard(tsd_t *tsd)
{
if (tcache == NULL) {
if (create == false) {
/*
* Creating a tcache here would cause
* allocation as a side effect of free().
* Ordinarily that would be okay since
* tcache_create() failure is a soft failure
* that doesn't propagate. However, if TLS
* data are freed via free() as in glibc,
* subtle corruption could result from setting
* a TLS variable after its backing memory is
* freed.
*/
return (NULL);
}
if (tcache_enabled_get() == false) {
tcache_enabled_set(false); /* Memoize. */
return (NULL);
}
return (tcache_create(choose_arena(NULL)));
}
if (tcache == TCACHE_STATE_PURGATORY) {
/*
* Make a note that an allocator function was called
* after tcache_thread_cleanup() was called.
*/
tcache = TCACHE_STATE_REINCARNATED;
tcache_tsd_set(&tcache);
return (NULL);
}
if (tcache == TCACHE_STATE_REINCARNATED)
return (NULL);
not_reached();
return (NULL);
return (tcache_create(choose_arena(tsd, NULL)));
}
tcache_t *
@ -328,7 +296,7 @@ tcache_create(arena_t *arena)
else if (size <= tcache_maxclass)
tcache = (tcache_t *)arena_malloc_large(arena, size, true);
else
tcache = (tcache_t *)icalloct(size, false, arena);
tcache = (tcache_t *)icalloct(NULL, size, false, arena);
if (tcache == NULL)
return (NULL);
@ -343,13 +311,11 @@ tcache_create(arena_t *arena)
stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
}
tcache_tsd_set(&tcache);
return (tcache);
}
void
tcache_destroy(tcache_t *tcache)
static void
tcache_destroy(tsd_t *tsd, tcache_t *tcache)
{
unsigned i;
size_t tcache_size;
@ -403,39 +369,30 @@ tcache_destroy(tcache_t *tcache)
arena_dalloc_large(arena, chunk, tcache);
} else
idalloct(tcache, false);
idalloct(tsd, tcache, false);
}
void
tcache_thread_cleanup(void *arg)
tcache_cleanup(tsd_t *tsd)
{
tcache_t *tcache = *(tcache_t **)arg;
tcache_t *tcache;
if (tcache == TCACHE_STATE_DISABLED) {
/* Do nothing. */
} else if (tcache == TCACHE_STATE_REINCARNATED) {
/*
* Another destructor called an allocator function after this
* destructor was called. Reset tcache to
* TCACHE_STATE_PURGATORY in order to receive another callback.
*/
tcache = TCACHE_STATE_PURGATORY;
tcache_tsd_set(&tcache);
} else if (tcache == TCACHE_STATE_PURGATORY) {
/*
* The previous time this destructor was called, we set the key
* to TCACHE_STATE_PURGATORY so that other destructors wouldn't
* cause re-creation of the tcache. This time, do nothing, so
* that the destructor will not be called again.
*/
} else if (tcache != NULL) {
assert(tcache != TCACHE_STATE_PURGATORY);
tcache_destroy(tcache);
tcache = TCACHE_STATE_PURGATORY;
tcache_tsd_set(&tcache);
if (!config_tcache)
return;
if ((tcache = tsd_tcache_get(tsd)) != NULL) {
tcache_destroy(tsd, tcache);
tsd_tcache_set(tsd, NULL);
}
}
void
tcache_enabled_cleanup(tsd_t *tsd)
{
/* Do nothing. */
}
/* Caller must own arena->lock. */
void
tcache_stats_merge(tcache_t *tcache, arena_t *arena)
@ -464,7 +421,7 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
}
bool
tcache_boot0(void)
tcache_boot(void)
{
unsigned i;
@ -504,13 +461,3 @@ tcache_boot0(void)
return (false);
}
bool
tcache_boot1(void)
{
if (tcache_tsd_boot() || tcache_enabled_tsd_boot())
return (true);
return (false);
}

51
src/tsd.c
View File

@ -7,6 +7,8 @@
static unsigned ncleanups;
static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
malloc_tsd_data(, , tsd_t, TSD_INITIALIZER)
/******************************************************************************/
void *
@ -14,14 +16,15 @@ malloc_tsd_malloc(size_t size)
{
/* Avoid choose_arena() in order to dodge bootstrapping issues. */
return (arena_malloc(arenas[0], size, false, false));
return (arena_malloc(NULL, arenas[0], CACHELINE_CEILING(size), false,
false));
}
void
malloc_tsd_dalloc(void *wrapper)
{
idalloct(wrapper, false);
idalloct(NULL, wrapper, false);
}
void
@ -67,10 +70,54 @@ malloc_tsd_cleanup_register(bool (*f)(void))
}
void
tsd_cleanup(void *arg)
{
tsd_t *tsd = (tsd_t *)arg;
if (tsd == NULL) {
/* OOM during re-initialization. */
return;
}
switch (tsd->state) {
case tsd_state_nominal:
#define O(n, t) \
n##_cleanup(tsd);
MALLOC_TSD
#undef O
tsd->state = tsd_state_purgatory;
tsd_set(tsd);
break;
case tsd_state_purgatory:
/*
* The previous time this destructor was called, we set the
* state to tsd_state_purgatory so that other destructors
* wouldn't cause re-creation of the tsd. This time, do
* nothing, and do not request another callback.
*/
break;
case tsd_state_reincarnated:
/*
* Another destructor deallocated memory after this destructor
* was called. Reset state to tsd_state_purgatory and request
* another callback.
*/
tsd->state = tsd_state_purgatory;
tsd_set(tsd);
break;
default:
not_reached();
}
}
bool
malloc_tsd_boot(void)
{
ncleanups = 0;
if (tsd_boot())
return (true);
return (false);
}
#ifdef _WIN32

46
test/unit/ckh.c
View File

@ -2,20 +2,25 @@
TEST_BEGIN(test_new_delete)
{
tsd_t *tsd;
ckh_t ckh;
assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp),
"Unexpected ckh_new() error");
ckh_delete(&ckh);
tsd = tsd_tryget();
assert_ptr_not_null(tsd, "Unexpected tsd failure");
assert_false(ckh_new(&ckh, 3, ckh_pointer_hash, ckh_pointer_keycomp),
assert_false(ckh_new(tsd, &ckh, 2, ckh_string_hash, ckh_string_keycomp),
"Unexpected ckh_new() error");
ckh_delete(&ckh);
ckh_delete(tsd, &ckh);
assert_false(ckh_new(tsd, &ckh, 3, ckh_pointer_hash,
ckh_pointer_keycomp), "Unexpected ckh_new() error");
ckh_delete(tsd, &ckh);
}
TEST_END
TEST_BEGIN(test_count_insert_search_remove)
{
tsd_t *tsd;
ckh_t ckh;
const char *strs[] = {
"a string",
@ -26,7 +31,10 @@ TEST_BEGIN(test_count_insert_search_remove)
const char *missing = "A string not in the hash table.";
size_t i;
assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp),
tsd = tsd_tryget();
assert_ptr_not_null(tsd, "Unexpected tsd failure");
assert_false(ckh_new(tsd, &ckh, 2, ckh_string_hash, ckh_string_keycomp),
"Unexpected ckh_new() error");
assert_zu_eq(ckh_count(&ckh), 0,
"ckh_count() should return %zu, but it returned %zu", ZU(0),
@ -34,7 +42,7 @@ TEST_BEGIN(test_count_insert_search_remove)
/* Insert. */
for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) {
ckh_insert(&ckh, strs[i], strs[i]);
ckh_insert(tsd, &ckh, strs[i], strs[i]);
assert_zu_eq(ckh_count(&ckh), i+1,
"ckh_count() should return %zu, but it returned %zu", i+1,
ckh_count(&ckh));
@ -79,7 +87,7 @@ TEST_BEGIN(test_count_insert_search_remove)
vp = (i & 2) ? &v.p : NULL;
k.p = NULL;
v.p = NULL;
assert_false(ckh_remove(&ckh, strs[i], kp, vp),
assert_false(ckh_remove(tsd, &ckh, strs[i], kp, vp),
"Unexpected ckh_remove() error");
ks = (i & 1) ? strs[i] : (const char *)NULL;
@ -95,20 +103,24 @@ TEST_BEGIN(test_count_insert_search_remove)
ckh_count(&ckh));
}
ckh_delete(&ckh);
ckh_delete(tsd, &ckh);
}
TEST_END
TEST_BEGIN(test_insert_iter_remove)
{
#define NITEMS ZU(1000)
tsd_t *tsd;
ckh_t ckh;
void **p[NITEMS];
void *q, *r;
size_t i;
assert_false(ckh_new(&ckh, 2, ckh_pointer_hash, ckh_pointer_keycomp),
"Unexpected ckh_new() error");
tsd = tsd_tryget();
assert_ptr_not_null(tsd, "Unexpected tsd failure");
assert_false(ckh_new(tsd, &ckh, 2, ckh_pointer_hash,
ckh_pointer_keycomp), "Unexpected ckh_new() error");
for (i = 0; i < NITEMS; i++) {
p[i] = mallocx(i+1, 0);
@ -119,7 +131,7 @@ TEST_BEGIN(test_insert_iter_remove)
size_t j;
for (j = i; j < NITEMS; j++) {
assert_false(ckh_insert(&ckh, p[j], p[j]),
assert_false(ckh_insert(tsd, &ckh, p[j], p[j]),
"Unexpected ckh_insert() failure");
assert_false(ckh_search(&ckh, p[j], &q, &r),
"Unexpected ckh_search() failure");
@ -134,13 +146,13 @@ TEST_BEGIN(test_insert_iter_remove)
for (j = i + 1; j < NITEMS; j++) {
assert_false(ckh_search(&ckh, p[j], NULL, NULL),
"Unexpected ckh_search() failure");
assert_false(ckh_remove(&ckh, p[j], &q, &r),
assert_false(ckh_remove(tsd, &ckh, p[j], &q, &r),
"Unexpected ckh_remove() failure");
assert_ptr_eq(p[j], q, "Key pointer mismatch");
assert_ptr_eq(p[j], r, "Value pointer mismatch");
assert_true(ckh_search(&ckh, p[j], NULL, NULL),
"Unexpected ckh_search() success");
assert_true(ckh_remove(&ckh, p[j], &q, &r),
assert_true(ckh_remove(tsd, &ckh, p[j], &q, &r),
"Unexpected ckh_remove() success");
}
@ -176,13 +188,13 @@ TEST_BEGIN(test_insert_iter_remove)
for (i = 0; i < NITEMS; i++) {
assert_false(ckh_search(&ckh, p[i], NULL, NULL),
"Unexpected ckh_search() failure");
assert_false(ckh_remove(&ckh, p[i], &q, &r),
assert_false(ckh_remove(tsd, &ckh, p[i], &q, &r),
"Unexpected ckh_remove() failure");
assert_ptr_eq(p[i], q, "Key pointer mismatch");
assert_ptr_eq(p[i], r, "Value pointer mismatch");
assert_true(ckh_search(&ckh, p[i], NULL, NULL),
"Unexpected ckh_search() success");
assert_true(ckh_remove(&ckh, p[i], &q, &r),
assert_true(ckh_remove(tsd, &ckh, p[i], &q, &r),
"Unexpected ckh_remove() success");
dallocx(p[i], 0);
}
@ -190,7 +202,7 @@ TEST_BEGIN(test_insert_iter_remove)
assert_zu_eq(ckh_count(&ckh), 0,
"ckh_count() should return %zu, but it returned %zu", ZU(0),
ckh_count(&ckh));
ckh_delete(&ckh);
ckh_delete(tsd, &ckh);
#undef NITEMS
}
TEST_END

8
test/unit/rtree.c
View File

@ -5,7 +5,7 @@ TEST_BEGIN(test_rtree_get_empty)
unsigned i;
for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
rtree_t *rtree = rtree_new(i, imalloc, idalloc);
rtree_t *rtree = rtree_new(i, malloc, free);
assert_u_eq(rtree_get(rtree, 0), 0,
"rtree_get() should return NULL for empty tree");
rtree_delete(rtree);
@ -18,7 +18,7 @@ TEST_BEGIN(test_rtree_extrema)
unsigned i;
for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
rtree_t *rtree = rtree_new(i, imalloc, idalloc);
rtree_t *rtree = rtree_new(i, malloc, free);
rtree_set(rtree, 0, 1);
assert_u_eq(rtree_get(rtree, 0), 1,
@ -40,7 +40,7 @@ TEST_BEGIN(test_rtree_bits)
for (i = 1; i < (sizeof(uintptr_t) << 3); i++) {
uintptr_t keys[] = {0, 1,
(((uintptr_t)1) << (sizeof(uintptr_t)*8-i)) - 1};
rtree_t *rtree = rtree_new(i, imalloc, idalloc);
rtree_t *rtree = rtree_new(i, malloc, free);
for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) {
rtree_set(rtree, keys[j], 1);
@ -73,7 +73,7 @@ TEST_BEGIN(test_rtree_random)
sfmt = init_gen_rand(SEED);
for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
rtree_t *rtree = rtree_new(i, imalloc, idalloc);
rtree_t *rtree = rtree_new(i, malloc, free);
uintptr_t keys[NSET];
unsigned j;

8
test/unit/tsd.c
View File

@ -16,11 +16,11 @@ data_cleanup(void *arg)
data_cleanup_executed = true;
}
malloc_tsd_protos(, data, data_t)
malloc_tsd_externs(data, data_t)
malloc_tsd_protos(, data_, data_t)
malloc_tsd_externs(data_, data_t)
#define DATA_INIT 0x12345678
malloc_tsd_data(, data, data_t, DATA_INIT)
malloc_tsd_funcs(, data, data_t, DATA_INIT, data_cleanup)
malloc_tsd_data(, data_, data_t, DATA_INIT)
malloc_tsd_funcs(, data_, data_t, DATA_INIT, data_cleanup)
static void *
thd_start(void *arg)