project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Convert all tsd variables to reside in a single tsd structure.

Browse Source
This commit is contained in:
Jason Evans
2014-09-22 21:09:23 -07:00
parent 42f5955938
commit 5460aa6f66
22 changed files with 1027 additions and 935 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
include/jemalloc/internal/arena.h
View File

@@ -419,9 +419,9 @@ extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
#endif
bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero);
void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc);
void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t extra, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc);
dss_prec_t arena_dss_prec_get(arena_t *arena);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive,
@@ -485,10 +485,12 @@ unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
const void *ptr);
prof_tctx_t *arena_prof_tctx_get(const void *ptr);
void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache);
void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
bool try_tcache);
size_t arena_salloc(const void *ptr, bool demote);
void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache);
void arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size,
void arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr,
bool try_tcache);
void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
bool try_tcache);
#endif
@@ -1053,7 +1055,8 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
}
JEMALLOC_ALWAYS_INLINE void *
arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
bool try_tcache)
{
tcache_t *tcache;
@@ -1061,12 +1064,12 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
assert(size <= arena_maxclass);
if (likely(size <= SMALL_MAXCLASS)) {
if (likely(try_tcache) && likely((tcache = tcache_get(true)) !=
NULL))
if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
true)) != NULL))
return (tcache_alloc_small(tcache, size, zero));
else {
return (arena_malloc_small(choose_arena(arena), size,
zero));
return (arena_malloc_small(choose_arena(tsd, arena),
size, zero));
}
} else {
/*
@@ -1074,11 +1077,11 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
* infinite recursion during tcache initialization.
*/
if (try_tcache && size <= tcache_maxclass && likely((tcache =
tcache_get(true)) != NULL))
tcache_get(tsd, true)) != NULL))
return (tcache_alloc_large(tcache, size, zero));
else {
return (arena_malloc_large(choose_arena(arena), size,
zero));
return (arena_malloc_large(choose_arena(tsd, arena),
size, zero));
}
}
}
@@ -1128,7 +1131,7 @@ arena_salloc(const void *ptr, bool demote)
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache)
{
size_t pageind, mapbits;
tcache_t *tcache;
@@ -1141,8 +1144,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) {
/* Small allocation. */
if (likely(try_tcache) && likely((tcache = tcache_get(false)) !=
NULL)) {
if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
false)) != NULL)) {
size_t binind = arena_ptr_small_binind_get(ptr,
mapbits);
tcache_dalloc_small(tcache, ptr, binind);
@@ -1154,7 +1157,7 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
if (try_tcache && size <= tcache_maxclass && likely((tcache =
tcache_get(false)) != NULL)) {
tcache_get(tsd, false)) != NULL)) {
tcache_dalloc_large(tcache, ptr, size);
} else
arena_dalloc_large(chunk->arena, chunk, ptr);
@@ -1162,7 +1165,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache)
}
JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size,
bool try_tcache)
{
tcache_t *tcache;
@@ -1171,8 +1175,8 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
if (likely(size <= SMALL_MAXCLASS)) {
/* Small allocation. */
if (likely(try_tcache) && likely((tcache = tcache_get(false)) !=
NULL)) {
if (likely(try_tcache) && likely((tcache = tcache_get(tsd,
false)) != NULL)) {
size_t binind = small_size2bin(size);
tcache_dalloc_small(tcache, ptr, binind);
} else {
@@ -1184,7 +1188,7 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache)
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
if (try_tcache && size <= tcache_maxclass && (tcache =
tcache_get(false)) != NULL) {
tcache_get(tsd, false)) != NULL) {
tcache_dalloc_large(tcache, ptr, size);
} else
arena_dalloc_large(chunk->arena, chunk, ptr);

8
include/jemalloc/internal/ckh.h
View File

@@ -66,13 +66,13 @@ struct ckh_s {
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
bool ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh_keycomp_t *keycomp);
void ckh_delete(ckh_t *ckh);
void ckh_delete(tsd_t *tsd, ckh_t *ckh);
size_t ckh_count(ckh_t *ckh);
bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
bool ckh_insert(ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key,
bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
void **data);
bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data);
void ckh_string_hash(const void *key, size_t r_hash[2]);

10
include/jemalloc/internal/huge.h
View File

@@ -9,12 +9,14 @@
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *huge_malloc(arena_t *arena, size_t size, bool zero);
void *huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero);
void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
bool zero);
bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra);
void *huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc);
void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t extra, size_t alignment, bool zero,
bool try_tcache_dalloc);
#ifdef JEMALLOC_JET
typedef void (huge_dalloc_junk_t)(void *, size_t);
extern huge_dalloc_junk_t *huge_dalloc_junk;

168
include/jemalloc/internal/jemalloc_internal.h.in
View File

@@ -350,7 +350,6 @@ static const bool config_ivsalloc =
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/tsd.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
@@ -364,15 +363,7 @@ static const bool config_ivsalloc =
#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
typedef struct {
uint64_t allocated;
uint64_t deallocated;
} thread_allocated_t;
/*
* The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro
* argument.
*/
#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_ARG_CONCAT({0, 0})
#include "jemalloc/internal/tsd.h"
#undef JEMALLOC_H_STRUCTS
/******************************************************************************/
@@ -407,8 +398,10 @@ extern unsigned narenas_total;
extern unsigned narenas_auto; /* Read-only after initialization. */
arena_t *arenas_extend(unsigned ind);
void arenas_cleanup(void *arg);
arena_t *choose_arena_hard(void);
arena_t *choose_arena_hard(tsd_t *tsd);
void thread_allocated_cleanup(tsd_t *tsd);
void thread_deallocated_cleanup(tsd_t *tsd);
void arena_cleanup(tsd_t *tsd);
void jemalloc_prefork(void);
void jemalloc_postfork_parent(void);
void jemalloc_postfork_child(void);
@@ -422,7 +415,6 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/tsd.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
@@ -435,6 +427,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/hash.h"
#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
#include "jemalloc/internal/tsd.h"
#undef JEMALLOC_H_EXTERNS
/******************************************************************************/
@@ -465,23 +458,13 @@ void jemalloc_postfork_child(void);
#undef JEMALLOC_ARENA_INLINE_A
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
size_t s2u(size_t size);
size_t sa2u(size_t size, size_t alignment);
unsigned narenas_total_get(void);
arena_t *choose_arena(arena_t *arena);
arena_t *choose_arena(tsd_t *tsd, arena_t *arena);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
/*
* Map of pthread_self() --> arenas[???], used for selecting an arena to use
* for allocations.
*/
malloc_tsd_externs(arenas, arena_t *)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL,
arenas_cleanup)
/*
* Compute usable size that would result from allocating an object with the
* specified size.
@@ -589,15 +572,15 @@ narenas_total_get(void)
/* Choose an arena based on a per-thread value. */
JEMALLOC_INLINE arena_t *
choose_arena(arena_t *arena)
choose_arena(tsd_t *tsd, arena_t *arena)
{
arena_t *ret;
if (arena != NULL)
return (arena);
if ((ret = *arenas_tsd_get()) == NULL) {
ret = choose_arena_hard();
if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) {
ret = choose_arena_hard(tsd);
assert(ret != NULL);
}
@@ -622,72 +605,72 @@ choose_arena(arena_t *arena)
#include "jemalloc/internal/quarantine.h"
#ifndef JEMALLOC_ENABLE_INLINE
void *imalloct(size_t size, bool try_tcache, arena_t *arena);
void *imalloc(size_t size);
void *icalloct(size_t size, bool try_tcache, arena_t *arena);
void *icalloc(size_t size);
void *ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena);
void *ipalloc(size_t usize, size_t alignment, bool zero);
void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
void *imalloc(tsd_t *tsd, size_t size);
void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
void *icalloc(tsd_t *tsd, size_t size);
void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
bool try_tcache, arena_t *arena);
void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
size_t isalloc(const void *ptr, bool demote);
size_t ivsalloc(const void *ptr, bool demote);
size_t u2rz(size_t usize);
size_t p2rz(const void *ptr);
void idalloct(void *ptr, bool try_tcache);
void isdalloct(void *ptr, size_t size, bool try_tcache);
void idalloc(void *ptr);
void iqalloc(void *ptr, bool try_tcache);
void isqalloc(void *ptr, size_t size, bool try_tcache);
void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena);
void *iralloct(void *ptr, size_t size, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
void *iralloc(void *ptr, size_t size, size_t alignment, bool zero);
void idalloct(tsd_t *tsd, void *ptr, bool try_tcache);
void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
void idalloc(tsd_t *tsd, void *ptr);
void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache);
void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc, arena_t *arena);
void *iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
void *iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
bool zero);
bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero);
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE void *
imalloct(size_t size, bool try_tcache, arena_t *arena)
imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
{
assert(size != 0);
if (size <= arena_maxclass)
return (arena_malloc(arena, size, false, try_tcache));
return (arena_malloc(tsd, arena, size, false, try_tcache));
else
return (huge_malloc(arena, size, false));
return (huge_malloc(tsd, arena, size, false));
}
JEMALLOC_ALWAYS_INLINE void *
imalloc(size_t size)
imalloc(tsd_t *tsd, size_t size)
{
return (imalloct(size, true, NULL));
return (imalloct(tsd, size, true, NULL));
}
JEMALLOC_ALWAYS_INLINE void *
icalloct(size_t size, bool try_tcache, arena_t *arena)
icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
{
if (size <= arena_maxclass)
return (arena_malloc(arena, size, true, try_tcache));
return (arena_malloc(tsd, arena, size, true, try_tcache));
else
return (huge_malloc(arena, size, true));
return (huge_malloc(tsd, arena, size, true));
}
JEMALLOC_ALWAYS_INLINE void *
icalloc(size_t size)
icalloc(tsd_t *tsd, size_t size)
{
return (icalloct(size, true, NULL));
return (icalloct(tsd, size, true, NULL));
}
JEMALLOC_ALWAYS_INLINE void *
ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena)
{
void *ret;
@@ -696,15 +679,15 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
assert(usize == sa2u(usize, alignment));
if (usize <= arena_maxclass && alignment <= PAGE)
ret = arena_malloc(arena, usize, zero, try_tcache);
ret = arena_malloc(tsd, arena, usize, zero, try_tcache);
else {
if (usize <= arena_maxclass) {
ret = arena_palloc(choose_arena(arena), usize,
ret = arena_palloc(choose_arena(tsd, arena), usize,
alignment, zero);
} else if (alignment <= chunksize)
ret = huge_malloc(arena, usize, zero);
ret = huge_malloc(tsd, arena, usize, zero);
else
ret = huge_palloc(arena, usize, alignment, zero);
ret = huge_palloc(tsd, arena, usize, alignment, zero);
}
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
@@ -712,10 +695,10 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
}
JEMALLOC_ALWAYS_INLINE void *
ipalloc(size_t usize, size_t alignment, bool zero)
ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero)
{
return (ipalloct(usize, alignment, zero, true, NULL));
return (ipalloct(tsd, usize, alignment, zero, true, NULL));
}
/*
@@ -776,7 +759,7 @@ p2rz(const void *ptr)
}
JEMALLOC_ALWAYS_INLINE void
idalloct(void *ptr, bool try_tcache)
idalloct(tsd_t *tsd, void *ptr, bool try_tcache)
{
arena_chunk_t *chunk;
@@ -784,13 +767,13 @@ idalloct(void *ptr, bool try_tcache)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
arena_dalloc(chunk, ptr, try_tcache);
arena_dalloc(tsd, chunk, ptr, try_tcache);
else
huge_dalloc(ptr);
}
JEMALLOC_ALWAYS_INLINE void
isdalloct(void *ptr, size_t size, bool try_tcache)
isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
{
arena_chunk_t *chunk;
@@ -798,42 +781,42 @@ isdalloct(void *ptr, size_t size, bool try_tcache)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
arena_sdalloc(chunk, ptr, size, try_tcache);
arena_sdalloc(tsd, chunk, ptr, size, try_tcache);
else
huge_dalloc(ptr);
}
JEMALLOC_ALWAYS_INLINE void
idalloc(void *ptr)
idalloc(tsd_t *tsd, void *ptr)
{
idalloct(ptr, true);
idalloct(tsd, ptr, true);
}
JEMALLOC_ALWAYS_INLINE void
iqalloc(void *ptr, bool try_tcache)
iqalloc(tsd_t *tsd, void *ptr, bool try_tcache)
{
if (config_fill && unlikely(opt_quarantine))
quarantine(ptr);
quarantine(tsd, ptr);
else
idalloct(ptr, try_tcache);
idalloct(tsd, ptr, try_tcache);
}
JEMALLOC_ALWAYS_INLINE void
isqalloc(void *ptr, size_t size, bool try_tcache)
isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
{
if (config_fill && unlikely(opt_quarantine))
quarantine(ptr);
quarantine(tsd, ptr);
else
isdalloct(ptr, size, try_tcache);
isdalloct(tsd, ptr, size, try_tcache);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena)
iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc, arena_t *arena)
{
void *p;
size_t usize, copysize;
@@ -841,7 +824,7 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
usize = sa2u(size + extra, alignment);
if (usize == 0)
return (NULL);
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, arena);
if (p == NULL) {
if (extra == 0)
return (NULL);
@@ -849,7 +832,8 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
usize = sa2u(size, alignment);
if (usize == 0)
return (NULL);
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc,
arena);
if (p == NULL)
return (NULL);
}
@@ -859,12 +843,12 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(p, ptr, copysize);
iqalloc(ptr, try_tcache_dalloc);
iqalloc(tsd, ptr, try_tcache_dalloc);
return (p);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct(void *ptr, size_t size, size_t alignment, bool zero,
iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
{
size_t oldsize;
@@ -880,24 +864,24 @@ iralloct(void *ptr, size_t size, size_t alignment, bool zero,
* Existing object alignment is inadequate; allocate new space
* and copy.
*/
return (iralloct_realign(ptr, oldsize, size, 0, alignment, zero,
try_tcache_alloc, try_tcache_dalloc, arena));
return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment,
zero, try_tcache_alloc, try_tcache_dalloc, arena));
}
if (size <= arena_maxclass) {
return (arena_ralloc(arena, ptr, oldsize, size, 0, alignment,
zero, try_tcache_alloc, try_tcache_dalloc));
return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0,
alignment, zero, try_tcache_alloc, try_tcache_dalloc));
} else {
return (huge_ralloc(arena, ptr, oldsize, size, 0, alignment,
zero, try_tcache_dalloc));
return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0,
alignment, zero, try_tcache_dalloc));
}
}
JEMALLOC_ALWAYS_INLINE void *
iralloc(void *ptr, size_t size, size_t alignment, bool zero)
iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero)
{
return (iralloct(ptr, size, alignment, zero, true, true, NULL));
return (iralloct(tsd, ptr, size, alignment, zero, true, true, NULL));
}
JEMALLOC_ALWAYS_INLINE bool
@@ -920,10 +904,6 @@ ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
else
return (huge_ralloc_no_move(ptr, oldsize, size, extra));
}
malloc_tsd_externs(thread_allocated, thread_allocated_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t,
THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
#endif
#include "jemalloc/internal/prof.h"

96
include/jemalloc/internal/private_symbols.txt
View File

@@ -8,6 +8,7 @@ arena_bitselm_get
arena_boot
arena_chunk_alloc_huge
arena_chunk_dalloc_huge
arena_cleanup
arena_dalloc
arena_dalloc_bin
arena_dalloc_bin_locked
@@ -65,19 +66,9 @@ arena_sdalloc
arena_stats_merge
arena_tcache_fill_small
arenas
arenas_booted
arenas_cleanup
arenas_extend
arenas_initialized
arenas_lock
arenas_tls
arenas_tsd
arenas_tsd_boot
arenas_tsd_cleanup_wrapper
arenas_tsd_get
arenas_tsd_get_wrapper
arenas_tsd_init_head
arenas_tsd_set
atomic_add_u
atomic_add_uint32
atomic_add_uint64
@@ -317,37 +308,17 @@ prof_sample_accum_update
prof_sample_threshold_update
prof_tctx_get
prof_tctx_set
prof_tdata_booted
prof_tdata_cleanup
prof_tdata_get
prof_tdata_init
prof_tdata_initialized
prof_tdata_tls
prof_tdata_tsd
prof_tdata_tsd_boot
prof_tdata_tsd_cleanup_wrapper
prof_tdata_tsd_get
prof_tdata_tsd_get_wrapper
prof_tdata_tsd_init_head
prof_tdata_tsd_set
prof_thread_active_get
prof_thread_active_set
prof_thread_name_get
prof_thread_name_set
quarantine
quarantine_alloc_hook
quarantine_boot
quarantine_booted
quarantine_cleanup
quarantine_init
quarantine_tls
quarantine_tsd
quarantine_tsd_boot
quarantine_tsd_cleanup_wrapper
quarantine_tsd_get
quarantine_tsd_get_wrapper
quarantine_tsd_init_head
quarantine_tsd_set
register_zone
rtree_delete
rtree_get
@@ -386,55 +357,52 @@ tcache_arena_dissociate
tcache_bin_flush_large
tcache_bin_flush_small
tcache_bin_info
tcache_boot0
tcache_boot1
tcache_booted
tcache_boot
tcache_cleanup
tcache_create
tcache_dalloc_large
tcache_dalloc_small
tcache_destroy
tcache_enabled_booted
tcache_enabled_cleanup
tcache_enabled_get
tcache_enabled_initialized
tcache_enabled_set
tcache_enabled_tls
tcache_enabled_tsd
tcache_enabled_tsd_boot
tcache_enabled_tsd_cleanup_wrapper
tcache_enabled_tsd_get
tcache_enabled_tsd_get_wrapper
tcache_enabled_tsd_init_head
tcache_enabled_tsd_set
tcache_event
tcache_event_hard
tcache_flush
tcache_get
tcache_get_hard
tcache_initialized
tcache_maxclass
tcache_salloc
tcache_stats_merge
tcache_thread_cleanup
tcache_tls
tcache_tsd
tcache_tsd_boot
tcache_tsd_cleanup_wrapper
tcache_tsd_get
tcache_tsd_get_wrapper
tcache_tsd_init_head
tcache_tsd_set
thread_allocated_booted
thread_allocated_initialized
thread_allocated_tls
thread_allocated_tsd
thread_allocated_tsd_boot
thread_allocated_tsd_cleanup_wrapper
thread_allocated_tsd_get
thread_allocated_tsd_get_wrapper
thread_allocated_tsd_init_head
thread_allocated_tsd_set
thread_allocated_cleanup
thread_deallocated_cleanup
tsd_booted
tsd_arena_get
tsd_arena_set
tsd_boot
tsd_cleanup
tsd_cleanup_wrapper
tsd_get
tsd_get_wrapper
tsd_initialized
tsd_init_check_recursion
tsd_init_finish
tsd_init_head
tsd_quarantine_get
tsd_quarantine_set
tsd_set
tsd_tcache_enabled_get
tsd_tcache_enabled_set
tsd_tcache_get
tsd_tcache_set
tsd_tls
tsd_tsd
tsd_prof_tdata_get
tsd_prof_tdata_set
tsd_thread_allocated_get
tsd_thread_allocated_set
tsd_thread_deallocated_get
tsd_thread_deallocated_set
tsd_tryget
u2rz
valgrind_freelike_block
valgrind_make_mem_defined

78
include/jemalloc/internal/prof.h
View File

@@ -248,13 +248,13 @@ extern uint64_t prof_interval;
*/
extern size_t lg_prof_sample;
void prof_alloc_rollback(prof_tctx_t *tctx, bool updated);
void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
void prof_malloc_sample_object(const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_free_sampled_object(size_t usize, prof_tctx_t *tctx);
void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
void bt_init(prof_bt_t *bt, void **vec);
void prof_backtrace(prof_bt_t *bt);
prof_tctx_t *prof_lookup(prof_bt_t *bt);
prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt);
#ifdef JEMALLOC_JET
size_t prof_bt_count(void);
typedef int (prof_dump_open_t)(bool, const char *);
@@ -263,12 +263,12 @@ extern prof_dump_open_t *prof_dump_open;
void prof_idump(void);
bool prof_mdump(const char *filename);
void prof_gdump(void);
prof_tdata_t *prof_tdata_init(void);
prof_tdata_t *prof_tdata_reinit(prof_tdata_t *tdata);
void prof_reset(size_t lg_sample);
void prof_tdata_cleanup(void *arg);
prof_tdata_t *prof_tdata_init(tsd_t *tsd);
prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
void prof_reset(tsd_t *tsd, size_t lg_sample);
void prof_tdata_cleanup(tsd_t *tsd);
const char *prof_thread_name_get(void);
bool prof_thread_name_set(const char *thread_name);
bool prof_thread_name_set(tsd_t *tsd, const char *thread_name);
bool prof_thread_active_get(void);
bool prof_thread_active_set(bool active);
void prof_boot0(void);
@@ -284,43 +284,38 @@ void prof_sample_threshold_update(prof_tdata_t *tdata);
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
prof_tdata_t *prof_tdata_get(bool create);
bool prof_sample_accum_update(size_t usize, bool commit,
prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create);
bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
prof_tdata_t **tdata_out);
prof_tctx_t *prof_alloc_prep(size_t usize, bool update);
prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool update);
prof_tctx_t *prof_tctx_get(const void *ptr);
void prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
void prof_malloc_sample_object(const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx);
void prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx,
bool updated, size_t old_usize, prof_tctx_t *old_tctx);
void prof_free(const void *ptr, size_t usize);
void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize,
prof_tctx_t *tctx, bool updated, size_t old_usize, prof_tctx_t *old_tctx);
void prof_free(tsd_t *tsd, const void *ptr, size_t usize);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
/* Thread-specific backtrace cache, used to reduce bt2gctx contention. */
malloc_tsd_externs(prof_tdata, prof_tdata_t *)
malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
prof_tdata_cleanup)
JEMALLOC_INLINE prof_tdata_t *
prof_tdata_get(bool create)
prof_tdata_get(tsd_t *tsd, bool create)
{
prof_tdata_t *tdata;
cassert(config_prof);
tdata = *prof_tdata_tsd_get();
tdata = tsd_prof_tdata_get(tsd);
if (create) {
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {
if (tdata == NULL)
tdata = prof_tdata_init();
} else if (tdata->state == prof_tdata_state_expired)
tdata = prof_tdata_reinit(tdata);
assert((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX ||
if (unlikely(tdata == NULL)) {
tdata = prof_tdata_init(tsd);
tsd_prof_tdata_set(tsd, tdata);
} else if (unlikely(tdata->state == prof_tdata_state_expired)) {
tdata = prof_tdata_reinit(tsd, tdata);
tsd_prof_tdata_set(tsd, tdata);
}
assert(tdata == NULL ||
tdata->state == prof_tdata_state_attached);
}
@@ -363,13 +358,14 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
}
JEMALLOC_INLINE bool
prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out)
prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
prof_tdata_t **tdata_out)
{
prof_tdata_t *tdata;
cassert(config_prof);
tdata = prof_tdata_get(true);
tdata = prof_tdata_get(tsd, true);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
tdata = NULL;
@@ -392,7 +388,7 @@ prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out)
}
JEMALLOC_INLINE prof_tctx_t *
prof_alloc_prep(size_t usize, bool update)
prof_alloc_prep(tsd_t *tsd, size_t usize, bool update)
{
prof_tctx_t *ret;
prof_tdata_t *tdata;
@@ -400,13 +396,13 @@ prof_alloc_prep(size_t usize, bool update)
assert(usize == s2u(usize));
if (!opt_prof_active || likely(prof_sample_accum_update(usize, update,
&tdata)))
if (!opt_prof_active || likely(prof_sample_accum_update(tsd, usize,
update, &tdata)))
ret = (prof_tctx_t *)(uintptr_t)1U;
else {
bt_init(&bt, tdata->vec);
prof_backtrace(&bt);
ret = prof_lookup(&bt);
ret = prof_lookup(tsd, &bt);
}
return (ret);
@@ -427,8 +423,8 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx)
}
JEMALLOC_INLINE void
prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
size_t old_usize, prof_tctx_t *old_tctx)
prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
bool updated, size_t old_usize, prof_tctx_t *old_tctx)
{
cassert(config_prof);
@@ -436,7 +432,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
if (!updated && ptr != NULL) {
assert(usize == isalloc(ptr, true));
if (prof_sample_accum_update(usize, true, NULL)) {
if (prof_sample_accum_update(tsd, usize, true, NULL)) {
/*
* Don't sample. The usize passed to PROF_ALLOC_PREP()
* was larger than what actually got allocated, so a
@@ -449,7 +445,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
}
if (unlikely((uintptr_t)old_tctx > (uintptr_t)1U))
prof_free_sampled_object(old_usize, old_tctx);
prof_free_sampled_object(tsd, old_usize, old_tctx);
if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
prof_malloc_sample_object(ptr, usize, tctx);
else
@@ -457,7 +453,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated,
}
JEMALLOC_INLINE void
prof_free(const void *ptr, size_t usize)
prof_free(tsd_t *tsd, const void *ptr, size_t usize)
{
prof_tctx_t *tctx = prof_tctx_get(ptr);
@@ -465,7 +461,7 @@ prof_free(const void *ptr, size_t usize)
assert(usize == isalloc(ptr, true));
if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
prof_free_sampled_object(usize, tctx);
prof_free_sampled_object(tsd, usize, tctx);
}
#endif

21
include/jemalloc/internal/quarantine.h
View File

@@ -29,36 +29,29 @@ struct quarantine_s {
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
quarantine_t *quarantine_init(size_t lg_maxobjs);
void quarantine(void *ptr);
void quarantine_cleanup(void *arg);
bool quarantine_boot(void);
quarantine_t *quarantine_init(tsd_t *tsd, size_t lg_maxobjs);
void quarantine(tsd_t *tsd, void *ptr);
void quarantine_cleanup(tsd_t *tsd);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *)
void quarantine_alloc_hook(void);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_))
malloc_tsd_externs(quarantine, quarantine_t *)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL,
quarantine_cleanup)
JEMALLOC_ALWAYS_INLINE void
quarantine_alloc_hook(void)
{
quarantine_t *quarantine;
tsd_t *tsd;
assert(config_fill && opt_quarantine);
quarantine = *quarantine_tsd_get();
if (quarantine == NULL)
quarantine_init(LG_MAXOBJS_INIT);
tsd = tsd_tryget();
if (tsd != NULL && tsd_quarantine_get(tsd) == NULL)
tsd_quarantine_set(tsd, quarantine_init(tsd, LG_MAXOBJS_INIT));
}
#endif

88
include/jemalloc/internal/tcache.h
View File

@@ -110,26 +110,22 @@ void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
tcache_t *tcache);
void tcache_arena_associate(tcache_t *tcache, arena_t *arena);
void tcache_arena_dissociate(tcache_t *tcache);
tcache_t *tcache_get_hard(tcache_t *tcache, bool create);
tcache_t *tcache_get_hard(tsd_t *tsd);
tcache_t *tcache_create(arena_t *arena);
void tcache_destroy(tcache_t *tcache);
void tcache_thread_cleanup(void *arg);
void tcache_cleanup(tsd_t *tsd);
void tcache_enabled_cleanup(tsd_t *tsd);
void tcache_stats_merge(tcache_t *tcache, arena_t *arena);
bool tcache_boot0(void);
bool tcache_boot1(void);
bool tcache_boot(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *)
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t)
void tcache_event(tcache_t *tcache);
void tcache_flush(void);
bool tcache_enabled_get(void);
tcache_t *tcache_get(bool create);
tcache_t *tcache_get(tsd_t *tsd, bool create);
void tcache_enabled_set(bool enabled);
void *tcache_alloc_easy(tcache_bin_t *tbin);
void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
@@ -139,41 +135,33 @@ void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
/* Map of thread-specific caches. */
malloc_tsd_externs(tcache, tcache_t *)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL,
tcache_thread_cleanup)
/* Per thread flag that allows thread caches to be disabled. */
malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t,
tcache_enabled_default, malloc_tsd_no_cleanup)
JEMALLOC_INLINE void
tcache_flush(void)
{
tcache_t *tcache;
tsd_t *tsd;
cassert(config_tcache);
tcache = *tcache_tsd_get();
if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)
return;
tcache_destroy(tcache);
tcache = NULL;
tcache_tsd_set(&tcache);
tsd = tsd_tryget();
if (tsd != NULL)
tcache_cleanup(tsd);
}
JEMALLOC_INLINE bool
tcache_enabled_get(void)
{
tsd_t *tsd;
tcache_enabled_t tcache_enabled;
cassert(config_tcache);
tcache_enabled = *tcache_enabled_tsd_get();
tsd = tsd_tryget();
if (tsd == NULL)
return (false);
tcache_enabled = tsd_tcache_enabled_get(tsd);
if (tcache_enabled == tcache_enabled_default) {
tcache_enabled = (tcache_enabled_t)opt_tcache;
tcache_enabled_tsd_set(&tcache_enabled);
tsd_tcache_enabled_set(tsd, tcache_enabled);
}
return ((bool)tcache_enabled);
@@ -182,33 +170,24 @@ tcache_enabled_get(void)
JEMALLOC_INLINE void
tcache_enabled_set(bool enabled)
{
tsd_t *tsd;
tcache_enabled_t tcache_enabled;
tcache_t *tcache;
cassert(config_tcache);
tsd = tsd_tryget();
if (tsd == NULL)
return;
tcache_enabled = (tcache_enabled_t)enabled;
tcache_enabled_tsd_set(&tcache_enabled);
tcache = *tcache_tsd_get();
if (enabled) {
if (tcache == TCACHE_STATE_DISABLED) {
tcache = NULL;
tcache_tsd_set(&tcache);
}
} else /* disabled */ {
if (tcache > TCACHE_STATE_MAX) {
tcache_destroy(tcache);
tcache = NULL;
}
if (tcache == NULL) {
tcache = TCACHE_STATE_DISABLED;
tcache_tsd_set(&tcache);
}
}
tsd_tcache_enabled_set(tsd, tcache_enabled);
if (!enabled)
tcache_cleanup(tsd);
}
JEMALLOC_ALWAYS_INLINE tcache_t *
tcache_get(bool create)
tcache_get(tsd_t *tsd, bool create)
{
tcache_t *tcache;
@@ -216,12 +195,19 @@ tcache_get(bool create)
return (NULL);
if (config_lazy_lock && isthreaded == false)
return (NULL);
/*
* If create is true, the caller has already assured that tsd is
* non-NULL.
*/
if (!create && unlikely(tsd == NULL))
return (NULL);
tcache = *tcache_tsd_get();
if (unlikely((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)) {
if (tcache == TCACHE_STATE_DISABLED)
return (NULL);
tcache = tcache_get_hard(tcache, create);
tcache = tsd_tcache_get(tsd);
if (!create)
return (tcache);
if (unlikely(tcache == NULL)) {
tcache = tcache_get_hard(tsd);
tsd_tcache_set(tsd, tcache);
}
return (tcache);

341
include/jemalloc/internal/tsd.h
View File

@@ -12,6 +12,15 @@ typedef struct tsd_init_block_s tsd_init_block_t;
typedef struct tsd_init_head_s tsd_init_head_t;
#endif
typedef struct tsd_s tsd_t;
typedef enum {
tsd_state_uninitialized,
tsd_state_nominal,
tsd_state_purgatory,
tsd_state_reincarnated
} tsd_state_t;
/*
* TLS/TSD-agnostic macro-based implementation of thread-specific data. There
* are four macros that support (at least) three use cases: file-private,
@@ -24,11 +33,11 @@ typedef struct tsd_init_head_s tsd_init_head_t;
* int y;
* } example_t;
* #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
* malloc_tsd_protos(, example, example_t *)
* malloc_tsd_externs(example, example_t *)
* malloc_tsd_protos(, example_, example_t *)
* malloc_tsd_externs(example_, example_t *)
* In example.c:
* malloc_tsd_data(, example, example_t *, EX_INITIALIZER)
* malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER,
* malloc_tsd_data(, example_, example_t *, EX_INITIALIZER)
* malloc_tsd_funcs(, example_, example_t *, EX_INITIALIZER,
* example_tsd_cleanup)
*
* The result is a set of generated functions, e.g.:
@@ -43,15 +52,13 @@ typedef struct tsd_init_head_s tsd_init_head_t;
* cast to (void *). This means that the cleanup function needs to cast *and*
* dereference the function argument, e.g.:
*
* void
* bool
* example_tsd_cleanup(void *arg)
* {
* example_t *example = *(example_t **)arg;
*
* [...]
* if ([want the cleanup function to be called again]) {
* example_tsd_set(&example);
* }
* return ([want the cleanup function to be called again]);
* }
*
* If example_tsd_set() is called within example_tsd_cleanup(), it will be
@@ -63,60 +70,60 @@ typedef struct tsd_init_head_s tsd_init_head_t;
/* malloc_tsd_protos(). */
#define malloc_tsd_protos(a_attr, a_name, a_type) \
a_attr bool \
a_name##_tsd_boot(void); \
a_name##tsd_boot(void); \
a_attr a_type * \
a_name##_tsd_get(void); \
a_name##tsd_get(void); \
a_attr void \
a_name##_tsd_set(a_type *val);
a_name##tsd_set(a_type *val);
/* malloc_tsd_externs(). */
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
#define malloc_tsd_externs(a_name, a_type) \
extern __thread a_type a_name##_tls; \
extern __thread bool a_name##_initialized; \
extern bool a_name##_booted;
extern __thread a_type a_name##tsd_tls; \
extern __thread bool a_name##tsd_initialized; \
extern bool a_name##tsd_booted;
#elif (defined(JEMALLOC_TLS))
#define malloc_tsd_externs(a_name, a_type) \
extern __thread a_type a_name##_tls; \
extern pthread_key_t a_name##_tsd; \
extern bool a_name##_booted;
extern __thread a_type a_name##tsd_tls; \
extern pthread_key_t a_name##tsd_tsd; \
extern bool a_name##tsd_booted;
#elif (defined(_WIN32))
#define malloc_tsd_externs(a_name, a_type) \
extern DWORD a_name##_tsd; \
extern bool a_name##_booted;
extern DWORD a_name##tsd_tsd; \
extern bool a_name##tsd_booted;
#else
#define malloc_tsd_externs(a_name, a_type) \
extern pthread_key_t a_name##_tsd; \
extern tsd_init_head_t a_name##_tsd_init_head; \
extern bool a_name##_booted;
extern pthread_key_t a_name##tsd_tsd; \
extern tsd_init_head_t a_name##tsd_init_head; \
extern bool a_name##tsd_booted;
#endif
/* malloc_tsd_data(). */
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr __thread a_type JEMALLOC_TLS_MODEL \
a_name##_tls = a_initializer; \
a_name##tsd_tls = a_initializer; \
a_attr __thread bool JEMALLOC_TLS_MODEL \
a_name##_initialized = false; \
a_attr bool a_name##_booted = false;
a_name##tsd_initialized = false; \
a_attr bool a_name##tsd_booted = false;
#elif (defined(JEMALLOC_TLS))
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr __thread a_type JEMALLOC_TLS_MODEL \
a_name##_tls = a_initializer; \
a_attr pthread_key_t a_name##_tsd; \
a_attr bool a_name##_booted = false;
a_name##tsd_tls = a_initializer; \
a_attr pthread_key_t a_name##tsd_tsd; \
a_attr bool a_name##tsd_booted = false;
#elif (defined(_WIN32))
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr DWORD a_name##_tsd; \
a_attr bool a_name##_booted = false;
a_attr DWORD a_name##tsd_tsd; \
a_attr bool a_name##tsd_booted = false;
#else
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr pthread_key_t a_name##_tsd; \
a_attr tsd_init_head_t a_name##_tsd_init_head = { \
a_attr pthread_key_t a_name##tsd_tsd; \
a_attr tsd_init_head_t a_name##tsd_init_head = { \
ql_head_initializer(blocks), \
MALLOC_MUTEX_INITIALIZER \
}; \
a_attr bool a_name##_booted = false;
a_attr bool a_name##tsd_booted = false;
#endif
/* malloc_tsd_funcs(). */
@@ -125,75 +132,76 @@ a_attr bool a_name##_booted = false;
a_cleanup) \
/* Initialization/cleanup. */ \
a_attr bool \
a_name##_tsd_cleanup_wrapper(void) \
a_name##tsd_cleanup_wrapper(void) \
{ \
\
if (a_name##_initialized) { \
a_name##_initialized = false; \
a_cleanup(&a_name##_tls); \
if (a_name##tsd_initialized) { \
a_name##tsd_initialized = false; \
a_cleanup(&a_name##tsd_tls); \
} \
return (a_name##_initialized); \
return (a_name##tsd_initialized); \
} \
a_attr bool \
a_name##_tsd_boot(void) \
a_name##tsd_boot(void) \
{ \
\
if (a_cleanup != malloc_tsd_no_cleanup) { \
malloc_tsd_cleanup_register( \
&a_name##_tsd_cleanup_wrapper); \
&a_name##tsd_cleanup_wrapper); \
} \
a_name##_booted = true; \
a_name##tsd_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_type * \
a_name##_tsd_get(void) \
a_name##tsd_get(void) \
{ \
\
assert(a_name##_booted); \
return (&a_name##_tls); \
assert(a_name##tsd_booted); \
return (&a_name##tsd_tls); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
a_name##tsd_set(a_type *val) \
{ \
\
assert(a_name##_booted); \
a_name##_tls = (*val); \
assert(a_name##tsd_booted); \
a_name##tsd_tls = (*val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
a_name##_initialized = true; \
a_name##tsd_initialized = true; \
}
#elif (defined(JEMALLOC_TLS))
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
a_cleanup) \
/* Initialization/cleanup. */ \
a_attr bool \
a_name##_tsd_boot(void) \
a_name##tsd_boot(void) \
{ \
\
if (a_cleanup != malloc_tsd_no_cleanup) { \
if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0) \
if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \
0) \
return (true); \
} \
a_name##_booted = true; \
a_name##tsd_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_type * \
a_name##_tsd_get(void) \
a_name##tsd_get(void) \
{ \
\
assert(a_name##_booted); \
return (&a_name##_tls); \
assert(a_name##tsd_booted); \
return (&a_name##tsd_tls); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
a_name##tsd_set(a_type *val) \
{ \
\
assert(a_name##_booted); \
a_name##_tls = (*val); \
assert(a_name##tsd_booted); \
a_name##tsd_tls = (*val); \
if (a_cleanup != malloc_tsd_no_cleanup) { \
if (pthread_setspecific(a_name##_tsd, \
(void *)(&a_name##_tls))) { \
if (pthread_setspecific(a_name##tsd_tsd, \
(void *)(&a_name##tsd_tls))) { \
malloc_write("<jemalloc>: Error" \
" setting TSD for "#a_name"\n"); \
if (opt_abort) \
@@ -208,23 +216,20 @@ a_name##_tsd_set(a_type *val) \
typedef struct { \
bool initialized; \
a_type val; \
} a_name##_tsd_wrapper_t; \
} a_name##tsd_wrapper_t; \
/* Initialization/cleanup. */ \
a_attr bool \
a_name##_tsd_cleanup_wrapper(void) \
a_name##tsd_cleanup_wrapper(void) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd); \
wrapper = (a_name##tsd_wrapper_t *)TlsGetValue(a_name##tsd_tsd);\
if (wrapper == NULL) \
return (false); \
if (a_cleanup != malloc_tsd_no_cleanup && \
wrapper->initialized) { \
a_type val = wrapper->val; \
a_type tsd_static_data = a_initializer; \
wrapper->initialized = false; \
wrapper->val = tsd_static_data; \
a_cleanup(&val); \
a_cleanup(&wrapper->val); \
if (wrapper->initialized) { \
/* Trigger another cleanup round. */ \
return (true); \
@@ -234,39 +239,38 @@ a_name##_tsd_cleanup_wrapper(void) \
return (false); \
} \
a_attr bool \
a_name##_tsd_boot(void) \
a_name##tsd_boot(void) \
{ \
\
a_name##_tsd = TlsAlloc(); \
if (a_name##_tsd == TLS_OUT_OF_INDEXES) \
a_name##tsd_tsd = TlsAlloc(); \
if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \
return (true); \
if (a_cleanup != malloc_tsd_no_cleanup) { \
malloc_tsd_cleanup_register( \
&a_name##_tsd_cleanup_wrapper); \
&a_name##tsd_cleanup_wrapper); \
} \
a_name##_booted = true; \
a_name##tsd_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_name##_tsd_wrapper_t * \
a_name##_tsd_get_wrapper(void) \
a_attr a_name##tsd_wrapper_t * \
a_name##tsd_get_wrapper(void) \
{ \
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \
TlsGetValue(a_name##_tsd); \
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
TlsGetValue(a_name##tsd_tsd); \
\
if (wrapper == NULL) { \
wrapper = (a_name##_tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \
if (unlikely(wrapper == NULL)) { \
wrapper = (a_name##tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
if (wrapper == NULL) { \
malloc_write("<jemalloc>: Error allocating" \
" TSD for "#a_name"\n"); \
abort(); \
} else { \
static a_type tsd_static_data = a_initializer; \
wrapper->initialized = false; \
wrapper->val = tsd_static_data; \
wrapper->val = a_initializer; \
} \
if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) { \
if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \
malloc_write("<jemalloc>: Error setting" \
" TSD for "#a_name"\n"); \
abort(); \
@@ -275,21 +279,21 @@ a_name##_tsd_get_wrapper(void) \
return (wrapper); \
} \
a_attr a_type * \
a_name##_tsd_get(void) \
a_name##tsd_get(void) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_get_wrapper(); \
return (&wrapper->val); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
a_name##tsd_set(a_type *val) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_get_wrapper(); \
wrapper->val = *(val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
wrapper->initialized = true; \
@@ -301,12 +305,12 @@ a_name##_tsd_set(a_type *val) \
typedef struct { \
bool initialized; \
a_type val; \
} a_name##_tsd_wrapper_t; \
} a_name##tsd_wrapper_t; \
/* Initialization/cleanup. */ \
a_attr void \
a_name##_tsd_cleanup_wrapper(void *arg) \
a_name##tsd_cleanup_wrapper(void *arg) \
{ \
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \
\
if (a_cleanup != malloc_tsd_no_cleanup && \
wrapper->initialized) { \
@@ -314,7 +318,7 @@ a_name##_tsd_cleanup_wrapper(void *arg) \
a_cleanup(&wrapper->val); \
if (wrapper->initialized) { \
/* Trigger another cleanup round. */ \
if (pthread_setspecific(a_name##_tsd, \
if (pthread_setspecific(a_name##tsd_tsd, \
(void *)wrapper)) { \
malloc_write("<jemalloc>: Error" \
" setting TSD for "#a_name"\n"); \
@@ -327,66 +331,65 @@ a_name##_tsd_cleanup_wrapper(void *arg) \
malloc_tsd_dalloc(wrapper); \
} \
a_attr bool \
a_name##_tsd_boot(void) \
a_name##tsd_boot(void) \
{ \
\
if (pthread_key_create(&a_name##_tsd, \
a_name##_tsd_cleanup_wrapper) != 0) \
if (pthread_key_create(&a_name##tsd_tsd, \
a_name##tsd_cleanup_wrapper) != 0) \
return (true); \
a_name##_booted = true; \
a_name##tsd_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_name##_tsd_wrapper_t * \
a_name##_tsd_get_wrapper(void) \
a_attr a_name##tsd_wrapper_t * \
a_name##tsd_get_wrapper(void) \
{ \
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \
pthread_getspecific(a_name##_tsd); \
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
pthread_getspecific(a_name##tsd_tsd); \
\
if (wrapper == NULL) { \
if (unlikely(wrapper == NULL)) { \
tsd_init_block_t block; \
wrapper = tsd_init_check_recursion( \
&a_name##_tsd_init_head, &block); \
&a_name##tsd_init_head, &block); \
if (wrapper) \
return (wrapper); \
wrapper = (a_name##_tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \
wrapper = (a_name##tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
block.data = wrapper; \
if (wrapper == NULL) { \
malloc_write("<jemalloc>: Error allocating" \
" TSD for "#a_name"\n"); \
abort(); \
} else { \
static a_type tsd_static_data = a_initializer; \
wrapper->initialized = false; \
wrapper->val = tsd_static_data; \
wrapper->val = a_initializer; \
} \
if (pthread_setspecific(a_name##_tsd, \
if (pthread_setspecific(a_name##tsd_tsd, \
(void *)wrapper)) { \
malloc_write("<jemalloc>: Error setting" \
" TSD for "#a_name"\n"); \
abort(); \
} \
tsd_init_finish(&a_name##_tsd_init_head, &block); \
tsd_init_finish(&a_name##tsd_init_head, &block); \
} \
return (wrapper); \
} \
a_attr a_type * \
a_name##_tsd_get(void) \
a_name##tsd_get(void) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_get_wrapper(); \
return (&wrapper->val); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
a_name##tsd_set(a_type *val) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_get_wrapper(); \
wrapper->val = *(val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
wrapper->initialized = true; \
@@ -410,25 +413,123 @@ struct tsd_init_head_s {
};
#endif
#define MALLOC_TSD \
/* O(name, type) */ \
O(tcache, tcache_t *) \
O(thread_allocated, uint64_t) \
O(thread_deallocated, uint64_t) \
O(prof_tdata, prof_tdata_t *) \
O(arena, arena_t *) \
O(tcache_enabled, tcache_enabled_t) \
O(quarantine, quarantine_t *) \
#define TSD_INITIALIZER { \
tsd_state_uninitialized, \
NULL, \
0, \
0, \
NULL, \
NULL, \
tcache_enabled_default, \
NULL \
}
struct tsd_s {
tsd_state_t state;
#define O(n, t) \
t n;
MALLOC_TSD
#undef O
};
static const tsd_t tsd_initializer = TSD_INITIALIZER;
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *malloc_tsd_malloc(size_t size);
void malloc_tsd_dalloc(void *wrapper);
void malloc_tsd_no_cleanup(void *);
void malloc_tsd_no_cleanup(void *arg);
void malloc_tsd_cleanup_register(bool (*f)(void));
void malloc_tsd_boot(void);
bool malloc_tsd_boot(void);
#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
!defined(_WIN32))
void *tsd_init_check_recursion(tsd_init_head_t *head,
tsd_init_block_t *block);
void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
#endif
void tsd_cleanup(void *arg);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
tsd_t *tsd_tryget(void);
#define O(n, t) \
t *tsd_##n##p_get(tsd_t *tsd); \
t tsd_##n##_get(tsd_t *tsd); \
void tsd_##n##_set(tsd_t *tsd, t n);
MALLOC_TSD
#undef O
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
malloc_tsd_externs(, tsd_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
JEMALLOC_INLINE tsd_t *
tsd_tryget(void)
{
tsd_t *tsd;
tsd = tsd_get();
if (unlikely(tsd == NULL))
return (NULL);
if (likely(tsd->state == tsd_state_nominal))
return (tsd);
else if (tsd->state == tsd_state_uninitialized) {
tsd->state = tsd_state_nominal;
tsd_set(tsd);
return (tsd);
} else if (tsd->state == tsd_state_purgatory) {
tsd->state = tsd_state_reincarnated;
tsd_set(tsd);
return (NULL);
} else {
assert(tsd->state == tsd_state_reincarnated);
return (NULL);
}
}
#define O(n, t) \
JEMALLOC_INLINE t * \
tsd_##n##p_get(tsd_t *tsd) \
{ \
\
return (&tsd->n); \
} \
\
JEMALLOC_INLINE t \
tsd_##n##_get(tsd_t *tsd) \
{ \
\
return (*tsd_##n##p_get(tsd)); \
} \
\
JEMALLOC_INLINE void \
tsd_##n##_set(tsd_t *tsd, t n) \
{ \
\
tsd->n = n; \
}
MALLOC_TSD
#undef O
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/