project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Convert all tsd variables to reside in a single tsd structure.

Browse Source
This commit is contained in:
Jason Evans
2014-09-22 21:09:23 -07:00
parent 42f5955938
commit 5460aa6f66
22 changed files with 1027 additions and 935 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
include/jemalloc/internal/jemalloc_internal.h.in
View File

@@ -350,7 +350,6 @@ static const bool config_ivsalloc =
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/tsd.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
@@ -364,15 +363,7 @@ static const bool config_ivsalloc =
#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
typedef struct {
uint64_t allocated;
uint64_t deallocated;
} thread_allocated_t;
/*
* The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro
* argument.
*/
#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_ARG_CONCAT({0, 0})
#include "jemalloc/internal/tsd.h"
#undef JEMALLOC_H_STRUCTS
/******************************************************************************/
@@ -407,8 +398,10 @@ extern unsigned narenas_total;
extern unsigned narenas_auto; /* Read-only after initialization. */
arena_t *arenas_extend(unsigned ind);
void arenas_cleanup(void *arg);
arena_t *choose_arena_hard(void);
arena_t *choose_arena_hard(tsd_t *tsd);
void thread_allocated_cleanup(tsd_t *tsd);
void thread_deallocated_cleanup(tsd_t *tsd);
void arena_cleanup(tsd_t *tsd);
void jemalloc_prefork(void);
void jemalloc_postfork_parent(void);
void jemalloc_postfork_child(void);
@@ -422,7 +415,6 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/stats.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/tsd.h"
#include "jemalloc/internal/mb.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent.h"
@@ -435,6 +427,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/hash.h"
#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
#include "jemalloc/internal/tsd.h"
#undef JEMALLOC_H_EXTERNS
/******************************************************************************/
@@ -465,23 +458,13 @@ void jemalloc_postfork_child(void);
#undef JEMALLOC_ARENA_INLINE_A
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
size_t s2u(size_t size);
size_t sa2u(size_t size, size_t alignment);
unsigned narenas_total_get(void);
arena_t *choose_arena(arena_t *arena);
arena_t *choose_arena(tsd_t *tsd, arena_t *arena);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
/*
* Map of pthread_self() --> arenas[???], used for selecting an arena to use
* for allocations.
*/
malloc_tsd_externs(arenas, arena_t *)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL,
arenas_cleanup)
/*
* Compute usable size that would result from allocating an object with the
* specified size.
@@ -589,15 +572,15 @@ narenas_total_get(void)
/* Choose an arena based on a per-thread value. */
JEMALLOC_INLINE arena_t *
choose_arena(arena_t *arena)
choose_arena(tsd_t *tsd, arena_t *arena)
{
arena_t *ret;
if (arena != NULL)
return (arena);
if ((ret = *arenas_tsd_get()) == NULL) {
ret = choose_arena_hard();
if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) {
ret = choose_arena_hard(tsd);
assert(ret != NULL);
}
@@ -622,72 +605,72 @@ choose_arena(arena_t *arena)
#include "jemalloc/internal/quarantine.h"
#ifndef JEMALLOC_ENABLE_INLINE
void *imalloct(size_t size, bool try_tcache, arena_t *arena);
void *imalloc(size_t size);
void *icalloct(size_t size, bool try_tcache, arena_t *arena);
void *icalloc(size_t size);
void *ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena);
void *ipalloc(size_t usize, size_t alignment, bool zero);
void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
void *imalloc(tsd_t *tsd, size_t size);
void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
void *icalloc(tsd_t *tsd, size_t size);
void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
bool try_tcache, arena_t *arena);
void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
size_t isalloc(const void *ptr, bool demote);
size_t ivsalloc(const void *ptr, bool demote);
size_t u2rz(size_t usize);
size_t p2rz(const void *ptr);
void idalloct(void *ptr, bool try_tcache);
void isdalloct(void *ptr, size_t size, bool try_tcache);
void idalloc(void *ptr);
void iqalloc(void *ptr, bool try_tcache);
void isqalloc(void *ptr, size_t size, bool try_tcache);
void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena);
void *iralloct(void *ptr, size_t size, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
void *iralloc(void *ptr, size_t size, size_t alignment, bool zero);
void idalloct(tsd_t *tsd, void *ptr, bool try_tcache);
void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
void idalloc(tsd_t *tsd, void *ptr);
void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache);
void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc, arena_t *arena);
void *iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
void *iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
bool zero);
bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero);
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE void *
imalloct(size_t size, bool try_tcache, arena_t *arena)
imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
{
assert(size != 0);
if (size <= arena_maxclass)
return (arena_malloc(arena, size, false, try_tcache));
return (arena_malloc(tsd, arena, size, false, try_tcache));
else
return (huge_malloc(arena, size, false));
return (huge_malloc(tsd, arena, size, false));
}
JEMALLOC_ALWAYS_INLINE void *
imalloc(size_t size)
imalloc(tsd_t *tsd, size_t size)
{
return (imalloct(size, true, NULL));
return (imalloct(tsd, size, true, NULL));
}
JEMALLOC_ALWAYS_INLINE void *
icalloct(size_t size, bool try_tcache, arena_t *arena)
icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
{
if (size <= arena_maxclass)
return (arena_malloc(arena, size, true, try_tcache));
return (arena_malloc(tsd, arena, size, true, try_tcache));
else
return (huge_malloc(arena, size, true));
return (huge_malloc(tsd, arena, size, true));
}
JEMALLOC_ALWAYS_INLINE void *
icalloc(size_t size)
icalloc(tsd_t *tsd, size_t size)
{
return (icalloct(size, true, NULL));
return (icalloct(tsd, size, true, NULL));
}
JEMALLOC_ALWAYS_INLINE void *
ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena)
{
void *ret;
@@ -696,15 +679,15 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
assert(usize == sa2u(usize, alignment));
if (usize <= arena_maxclass && alignment <= PAGE)
ret = arena_malloc(arena, usize, zero, try_tcache);
ret = arena_malloc(tsd, arena, usize, zero, try_tcache);
else {
if (usize <= arena_maxclass) {
ret = arena_palloc(choose_arena(arena), usize,
ret = arena_palloc(choose_arena(tsd, arena), usize,
alignment, zero);
} else if (alignment <= chunksize)
ret = huge_malloc(arena, usize, zero);
ret = huge_malloc(tsd, arena, usize, zero);
else
ret = huge_palloc(arena, usize, alignment, zero);
ret = huge_palloc(tsd, arena, usize, alignment, zero);
}
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
@@ -712,10 +695,10 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
}
JEMALLOC_ALWAYS_INLINE void *
ipalloc(size_t usize, size_t alignment, bool zero)
ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero)
{
return (ipalloct(usize, alignment, zero, true, NULL));
return (ipalloct(tsd, usize, alignment, zero, true, NULL));
}
/*
@@ -776,7 +759,7 @@ p2rz(const void *ptr)
}
JEMALLOC_ALWAYS_INLINE void
idalloct(void *ptr, bool try_tcache)
idalloct(tsd_t *tsd, void *ptr, bool try_tcache)
{
arena_chunk_t *chunk;
@@ -784,13 +767,13 @@ idalloct(void *ptr, bool try_tcache)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
arena_dalloc(chunk, ptr, try_tcache);
arena_dalloc(tsd, chunk, ptr, try_tcache);
else
huge_dalloc(ptr);
}
JEMALLOC_ALWAYS_INLINE void
isdalloct(void *ptr, size_t size, bool try_tcache)
isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
{
arena_chunk_t *chunk;
@@ -798,42 +781,42 @@ isdalloct(void *ptr, size_t size, bool try_tcache)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
arena_sdalloc(chunk, ptr, size, try_tcache);
arena_sdalloc(tsd, chunk, ptr, size, try_tcache);
else
huge_dalloc(ptr);
}
JEMALLOC_ALWAYS_INLINE void
idalloc(void *ptr)
idalloc(tsd_t *tsd, void *ptr)
{
idalloct(ptr, true);
idalloct(tsd, ptr, true);
}
JEMALLOC_ALWAYS_INLINE void
iqalloc(void *ptr, bool try_tcache)
iqalloc(tsd_t *tsd, void *ptr, bool try_tcache)
{
if (config_fill && unlikely(opt_quarantine))
quarantine(ptr);
quarantine(tsd, ptr);
else
idalloct(ptr, try_tcache);
idalloct(tsd, ptr, try_tcache);
}
JEMALLOC_ALWAYS_INLINE void
isqalloc(void *ptr, size_t size, bool try_tcache)
isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
{
if (config_fill && unlikely(opt_quarantine))
quarantine(ptr);
quarantine(tsd, ptr);
else
isdalloct(ptr, size, try_tcache);
isdalloct(tsd, ptr, size, try_tcache);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena)
iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc, arena_t *arena)
{
void *p;
size_t usize, copysize;
@@ -841,7 +824,7 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
usize = sa2u(size + extra, alignment);
if (usize == 0)
return (NULL);
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, arena);
if (p == NULL) {
if (extra == 0)
return (NULL);
@@ -849,7 +832,8 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
usize = sa2u(size, alignment);
if (usize == 0)
return (NULL);
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc,
arena);
if (p == NULL)
return (NULL);
}
@@ -859,12 +843,12 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(p, ptr, copysize);
iqalloc(ptr, try_tcache_dalloc);
iqalloc(tsd, ptr, try_tcache_dalloc);
return (p);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct(void *ptr, size_t size, size_t alignment, bool zero,
iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
{
size_t oldsize;
@@ -880,24 +864,24 @@ iralloct(void *ptr, size_t size, size_t alignment, bool zero,
* Existing object alignment is inadequate; allocate new space
* and copy.
*/
return (iralloct_realign(ptr, oldsize, size, 0, alignment, zero,
try_tcache_alloc, try_tcache_dalloc, arena));
return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment,
zero, try_tcache_alloc, try_tcache_dalloc, arena));
}
if (size <= arena_maxclass) {
return (arena_ralloc(arena, ptr, oldsize, size, 0, alignment,
zero, try_tcache_alloc, try_tcache_dalloc));
return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0,
alignment, zero, try_tcache_alloc, try_tcache_dalloc));
} else {
return (huge_ralloc(arena, ptr, oldsize, size, 0, alignment,
zero, try_tcache_dalloc));
return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0,
alignment, zero, try_tcache_dalloc));
}
}
JEMALLOC_ALWAYS_INLINE void *
iralloc(void *ptr, size_t size, size_t alignment, bool zero)
iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero)
{
return (iralloct(ptr, size, alignment, zero, true, true, NULL));
return (iralloct(tsd, ptr, size, alignment, zero, true, true, NULL));
}
JEMALLOC_ALWAYS_INLINE bool
@@ -920,10 +904,6 @@ ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
else
return (huge_ralloc_no_move(ptr, oldsize, size, extra));
}
malloc_tsd_externs(thread_allocated, thread_allocated_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t,
THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
#endif
#include "jemalloc/internal/prof.h"