8bb3198f72
Abstract arenas access to use arena_get() (or a0get() where appropriate) rather than directly reading e.g. arenas[ind]. Prior to the addition of the arenas.extend mallctl, the worst possible outcome of directly accessing arenas was a stale read, but arenas.extend may allocate and assign a new array to arenas. Add a tsd-based arenas_cache, which amortizes arenas reads. This introduces some subtle bootstrapping issues, with tsd_boot() now being split into tsd_boot[01]() to support tsd wrapper allocation bootstrapping, as well as an arenas_cache_bypass tsd variable which dynamically terminates allocation of arenas_cache itself. Promote a0malloc(), a0calloc(), and a0free() to be generally useful for internal allocation, and use them in several places (more may be appropriate). Abstract arena->nthreads management and fix a missing decrement during thread destruction (recent tsd refactoring left arenas_cleanup() unused). Change arena_choose() to propagate OOM, and handle OOM in all callers. This is important for providing consistent allocation behavior when the MALLOCX_ARENA() flag is being used. Prior to this fix, it was possible for an OOM to result in allocation silently allocating from a different arena than the one specified.
1083 lines
27 KiB
C
1083 lines
27 KiB
C
#ifndef JEMALLOC_INTERNAL_H
|
|
#define JEMALLOC_INTERNAL_H
|
|
|
|
#include "jemalloc_internal_defs.h"
|
|
#include "jemalloc/internal/jemalloc_internal_decls.h"
|
|
|
|
#ifdef JEMALLOC_UTRACE
|
|
#include <sys/ktrace.h>
|
|
#endif
|
|
|
|
#define JEMALLOC_NO_DEMANGLE
|
|
#ifdef JEMALLOC_JET
|
|
# define JEMALLOC_N(n) jet_##n
|
|
# include "jemalloc/internal/public_namespace.h"
|
|
# define JEMALLOC_NO_RENAME
|
|
# include "../jemalloc@install_suffix@.h"
|
|
# undef JEMALLOC_NO_RENAME
|
|
#else
|
|
# define JEMALLOC_N(n) @private_namespace@##n
|
|
# include "../jemalloc@install_suffix@.h"
|
|
#endif
|
|
#include "jemalloc/internal/private_namespace.h"
|
|
|
|
static const bool config_debug =
|
|
#ifdef JEMALLOC_DEBUG
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool have_dss =
|
|
#ifdef JEMALLOC_DSS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_fill =
|
|
#ifdef JEMALLOC_FILL
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_lazy_lock =
|
|
#ifdef JEMALLOC_LAZY_LOCK
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof =
|
|
#ifdef JEMALLOC_PROF
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof_libgcc =
|
|
#ifdef JEMALLOC_PROF_LIBGCC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof_libunwind =
|
|
#ifdef JEMALLOC_PROF_LIBUNWIND
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_munmap =
|
|
#ifdef JEMALLOC_MUNMAP
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_stats =
|
|
#ifdef JEMALLOC_STATS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_tcache =
|
|
#ifdef JEMALLOC_TCACHE
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_tls =
|
|
#ifdef JEMALLOC_TLS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_utrace =
|
|
#ifdef JEMALLOC_UTRACE
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_valgrind =
|
|
#ifdef JEMALLOC_VALGRIND
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_xmalloc =
|
|
#ifdef JEMALLOC_XMALLOC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_ivsalloc =
|
|
#ifdef JEMALLOC_IVSALLOC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
|
|
#ifdef JEMALLOC_ATOMIC9
|
|
#include <machine/atomic.h>
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
|
|
#include <libkern/OSAtomic.h>
|
|
#endif
|
|
|
|
#ifdef JEMALLOC_ZONE
|
|
#include <mach/mach_error.h>
|
|
#include <mach/mach_init.h>
|
|
#include <mach/vm_map.h>
|
|
#include <malloc/malloc.h>
|
|
#endif
|
|
|
|
#define RB_COMPACT
|
|
#include "jemalloc/internal/rb.h"
|
|
#include "jemalloc/internal/qr.h"
|
|
#include "jemalloc/internal/ql.h"
|
|
|
|
/*
|
|
* jemalloc can conceptually be broken into components (arena, tcache, etc.),
|
|
* but there are circular dependencies that cannot be broken without
|
|
* substantial performance degradation. In order to reduce the effect on
|
|
* visual code flow, read the header files in multiple passes, with one of the
|
|
* following cpp variables defined during each pass:
|
|
*
|
|
* JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data
|
|
* types.
|
|
* JEMALLOC_H_STRUCTS : Data structures.
|
|
* JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
|
|
* JEMALLOC_H_INLINES : Inline functions.
|
|
*/
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_TYPES
|
|
|
|
#include "jemalloc/internal/jemalloc_internal_macros.h"
|
|
|
|
/* Size class index type. */
|
|
typedef unsigned index_t;
|
|
|
|
#define MALLOCX_ARENA_MASK ((int)~0xff)
|
|
#define MALLOCX_LG_ALIGN_MASK ((int)0x3f)
|
|
/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */
|
|
#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \
|
|
(ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
|
|
#define MALLOCX_ALIGN_GET(flags) \
|
|
(MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1))
|
|
#define MALLOCX_ZERO_GET(flags) \
|
|
((bool)(flags & MALLOCX_ZERO))
|
|
#define MALLOCX_ARENA_GET(flags) \
|
|
(((unsigned)(flags >> 8)) - 1)
|
|
|
|
/* Smallest size class to support. */
|
|
#define LG_TINY_MIN 3
|
|
#define TINY_MIN (1U << LG_TINY_MIN)
|
|
|
|
/*
|
|
* Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size
|
|
* classes).
|
|
*/
|
|
#ifndef LG_QUANTUM
|
|
# if (defined(__i386__) || defined(_M_IX86))
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __ia64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __alpha__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __sparc64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64))
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __arm__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __aarch64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __hppa__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __mips__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __or1k__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __powerpc__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __s390__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __SH4__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __tile__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __le32__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifndef LG_QUANTUM
|
|
# error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS"
|
|
# endif
|
|
#endif
|
|
|
|
#define QUANTUM ((size_t)(1U << LG_QUANTUM))
|
|
#define QUANTUM_MASK (QUANTUM - 1)
|
|
|
|
/* Return the smallest quantum multiple that is >= a. */
|
|
#define QUANTUM_CEILING(a) \
|
|
(((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
|
|
|
|
#define LONG ((size_t)(1U << LG_SIZEOF_LONG))
|
|
#define LONG_MASK (LONG - 1)
|
|
|
|
/* Return the smallest long multiple that is >= a. */
|
|
#define LONG_CEILING(a) \
|
|
(((a) + LONG_MASK) & ~LONG_MASK)
|
|
|
|
#define SIZEOF_PTR (1U << LG_SIZEOF_PTR)
|
|
#define PTR_MASK (SIZEOF_PTR - 1)
|
|
|
|
/* Return the smallest (void *) multiple that is >= a. */
|
|
#define PTR_CEILING(a) \
|
|
(((a) + PTR_MASK) & ~PTR_MASK)
|
|
|
|
/*
|
|
* Maximum size of L1 cache line. This is used to avoid cache line aliasing.
|
|
* In addition, this controls the spacing of cacheline-spaced size classes.
|
|
*
|
|
* CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can
|
|
* only handle raw constants.
|
|
*/
|
|
#define LG_CACHELINE 6
|
|
#define CACHELINE 64
|
|
#define CACHELINE_MASK (CACHELINE - 1)
|
|
|
|
/* Return the smallest cacheline multiple that is >= s. */
|
|
#define CACHELINE_CEILING(s) \
|
|
(((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
|
|
|
|
/* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */
|
|
#ifdef PAGE_MASK
|
|
# undef PAGE_MASK
|
|
#endif
|
|
#define LG_PAGE STATIC_PAGE_SHIFT
|
|
#define PAGE ((size_t)(1U << STATIC_PAGE_SHIFT))
|
|
#define PAGE_MASK ((size_t)(PAGE - 1))
|
|
|
|
/* Return the smallest pagesize multiple that is >= s. */
|
|
#define PAGE_CEILING(s) \
|
|
(((s) + PAGE_MASK) & ~PAGE_MASK)
|
|
|
|
/* Return the nearest aligned address at or below a. */
|
|
#define ALIGNMENT_ADDR2BASE(a, alignment) \
|
|
((void *)((uintptr_t)(a) & (-(alignment))))
|
|
|
|
/* Return the offset between a and the nearest aligned address at or below a. */
|
|
#define ALIGNMENT_ADDR2OFFSET(a, alignment) \
|
|
((size_t)((uintptr_t)(a) & (alignment - 1)))
|
|
|
|
/* Return the smallest alignment multiple that is >= s. */
|
|
#define ALIGNMENT_CEILING(s, alignment) \
|
|
(((s) + (alignment - 1)) & (-(alignment)))
|
|
|
|
/* Declare a variable length array */
|
|
#if __STDC_VERSION__ < 199901L
|
|
# ifdef _MSC_VER
|
|
# include <malloc.h>
|
|
# define alloca _alloca
|
|
# else
|
|
# ifdef JEMALLOC_HAS_ALLOCA_H
|
|
# include <alloca.h>
|
|
# else
|
|
# include <stdlib.h>
|
|
# endif
|
|
# endif
|
|
# define VARIABLE_ARRAY(type, name, count) \
|
|
type *name = alloca(sizeof(type) * (count))
|
|
#else
|
|
# define VARIABLE_ARRAY(type, name, count) type name[(count)]
|
|
#endif
|
|
|
|
#include "jemalloc/internal/valgrind.h"
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_TYPES
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_STRUCTS
|
|
|
|
#include "jemalloc/internal/valgrind.h"
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#include "jemalloc/internal/tsd.h"
|
|
|
|
#undef JEMALLOC_H_STRUCTS
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_EXTERNS
|
|
|
|
extern bool opt_abort;
|
|
extern bool opt_junk;
|
|
extern size_t opt_quarantine;
|
|
extern bool opt_redzone;
|
|
extern bool opt_utrace;
|
|
extern bool opt_xmalloc;
|
|
extern bool opt_zero;
|
|
extern size_t opt_narenas;
|
|
|
|
extern bool in_valgrind;
|
|
|
|
/* Number of CPUs. */
|
|
extern unsigned ncpus;
|
|
|
|
/*
|
|
* index2size_tab encodes the same information as could be computed (at
|
|
* unacceptable cost in some code paths) by index2size_compute().
|
|
*/
|
|
extern size_t const index2size_tab[NSIZES];
|
|
/*
|
|
* size2index_tab is a compact lookup table that rounds request sizes up to
|
|
* size classes. In order to reduce cache footprint, the table is compressed,
|
|
* and all accesses are via size2index().
|
|
*/
|
|
extern uint8_t const size2index_tab[];
|
|
|
|
arena_t *a0get(void);
|
|
void *a0malloc(size_t size);
|
|
void *a0calloc(size_t num, size_t size);
|
|
void a0free(void *ptr);
|
|
arena_t *arenas_extend(unsigned ind);
|
|
arena_t *arena_init(unsigned ind);
|
|
unsigned narenas_total_get(void);
|
|
arena_t *arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing);
|
|
arena_t *arena_choose_hard(tsd_t *tsd);
|
|
void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind);
|
|
unsigned arena_nbound(unsigned ind);
|
|
void thread_allocated_cleanup(tsd_t *tsd);
|
|
void thread_deallocated_cleanup(tsd_t *tsd);
|
|
void arena_cleanup(tsd_t *tsd);
|
|
void arenas_cache_cleanup(tsd_t *tsd);
|
|
void narenas_cache_cleanup(tsd_t *tsd);
|
|
void arenas_cache_bypass_cleanup(tsd_t *tsd);
|
|
void jemalloc_prefork(void);
|
|
void jemalloc_postfork_parent(void);
|
|
void jemalloc_postfork_child(void);
|
|
|
|
#include "jemalloc/internal/valgrind.h"
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
|
|
#undef JEMALLOC_H_EXTERNS
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_INLINES
|
|
|
|
#include "jemalloc/internal/valgrind.h"
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
index_t size2index_compute(size_t size);
|
|
index_t size2index_lookup(size_t size);
|
|
index_t size2index(size_t size);
|
|
size_t index2size_compute(index_t index);
|
|
size_t index2size_lookup(index_t index);
|
|
size_t index2size(index_t index);
|
|
size_t s2u_compute(size_t size);
|
|
size_t s2u_lookup(size_t size);
|
|
size_t s2u(size_t size);
|
|
size_t sa2u(size_t size, size_t alignment);
|
|
arena_t *arena_choose(tsd_t *tsd, arena_t *arena);
|
|
arena_t *arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing,
|
|
bool refresh_if_missing);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
|
JEMALLOC_INLINE index_t
|
|
size2index_compute(size_t size)
|
|
{
|
|
|
|
#if (NTBINS != 0)
|
|
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
|
|
size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
|
|
size_t lg_ceil = lg_floor(pow2_ceil(size));
|
|
return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin);
|
|
} else
|
|
#endif
|
|
{
|
|
size_t x = lg_floor((size<<1)-1);
|
|
size_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 :
|
|
x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM);
|
|
size_t grp = shift << LG_SIZE_CLASS_GROUP;
|
|
|
|
size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1)
|
|
? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1;
|
|
|
|
size_t delta_inverse_mask = ZI(-1) << lg_delta;
|
|
size_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) &
|
|
((ZU(1) << LG_SIZE_CLASS_GROUP) - 1);
|
|
|
|
size_t index = NTBINS + grp + mod;
|
|
return (index);
|
|
}
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE index_t
|
|
size2index_lookup(size_t size)
|
|
{
|
|
|
|
assert(size <= LOOKUP_MAXCLASS);
|
|
{
|
|
size_t ret = ((size_t)(size2index_tab[(size-1) >>
|
|
LG_TINY_MIN]));
|
|
assert(ret == size2index_compute(size));
|
|
return (ret);
|
|
}
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE index_t
|
|
size2index(size_t size)
|
|
{
|
|
|
|
assert(size > 0);
|
|
if (likely(size <= LOOKUP_MAXCLASS))
|
|
return (size2index_lookup(size));
|
|
else
|
|
return (size2index_compute(size));
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
index2size_compute(index_t index)
|
|
{
|
|
|
|
#if (NTBINS > 0)
|
|
if (index < NTBINS)
|
|
return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index));
|
|
else
|
|
#endif
|
|
{
|
|
size_t reduced_index = index - NTBINS;
|
|
size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP;
|
|
size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) -
|
|
1);
|
|
|
|
size_t grp_size_mask = ~((!!grp)-1);
|
|
size_t grp_size = ((ZU(1) << (LG_QUANTUM +
|
|
(LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask;
|
|
|
|
size_t shift = (grp == 0) ? 1 : grp;
|
|
size_t lg_delta = shift + (LG_QUANTUM-1);
|
|
size_t mod_size = (mod+1) << lg_delta;
|
|
|
|
size_t usize = grp_size + mod_size;
|
|
return (usize);
|
|
}
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
index2size_lookup(index_t index)
|
|
{
|
|
size_t ret = (size_t)index2size_tab[index];
|
|
assert(ret == index2size_compute(index));
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
index2size(index_t index)
|
|
{
|
|
|
|
assert(index < NSIZES);
|
|
return (index2size_lookup(index));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
s2u_compute(size_t size)
|
|
{
|
|
|
|
#if (NTBINS > 0)
|
|
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
|
|
size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
|
|
size_t lg_ceil = lg_floor(pow2_ceil(size));
|
|
return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) :
|
|
(ZU(1) << lg_ceil));
|
|
} else
|
|
#endif
|
|
{
|
|
size_t x = lg_floor((size<<1)-1);
|
|
size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1)
|
|
? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1;
|
|
size_t delta = ZU(1) << lg_delta;
|
|
size_t delta_mask = delta - 1;
|
|
size_t usize = (size + delta_mask) & ~delta_mask;
|
|
return (usize);
|
|
}
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
s2u_lookup(size_t size)
|
|
{
|
|
size_t ret = index2size_lookup(size2index_lookup(size));
|
|
|
|
assert(ret == s2u_compute(size));
|
|
return (ret);
|
|
}
|
|
|
|
/*
|
|
* Compute usable size that would result from allocating an object with the
|
|
* specified size.
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
s2u(size_t size)
|
|
{
|
|
|
|
assert(size > 0);
|
|
if (likely(size <= LOOKUP_MAXCLASS))
|
|
return (s2u_lookup(size));
|
|
else
|
|
return (s2u_compute(size));
|
|
}
|
|
|
|
/*
|
|
* Compute usable size that would result from allocating an object with the
|
|
* specified size and alignment.
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
sa2u(size_t size, size_t alignment)
|
|
{
|
|
size_t usize;
|
|
|
|
assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
|
|
|
|
/* Try for a small size class. */
|
|
if (size <= SMALL_MAXCLASS && alignment < PAGE) {
|
|
/*
|
|
* Round size up to the nearest multiple of alignment.
|
|
*
|
|
* This done, we can take advantage of the fact that for each
|
|
* small size class, every object is aligned at the smallest
|
|
* power of two that is non-zero in the base two representation
|
|
* of the size. For example:
|
|
*
|
|
* Size | Base 2 | Minimum alignment
|
|
* -----+----------+------------------
|
|
* 96 | 1100000 | 32
|
|
* 144 | 10100000 | 32
|
|
* 192 | 11000000 | 64
|
|
*/
|
|
usize = s2u(ALIGNMENT_CEILING(size, alignment));
|
|
if (usize < LARGE_MINCLASS)
|
|
return (usize);
|
|
}
|
|
|
|
/* Try for a large size class. */
|
|
if (size <= arena_maxclass && alignment < chunksize) {
|
|
/*
|
|
* We can't achieve subpage alignment, so round up alignment
|
|
* to the minimum that can actually be supported.
|
|
*/
|
|
alignment = PAGE_CEILING(alignment);
|
|
|
|
/* Make sure result is a large size class. */
|
|
usize = (size <= LARGE_MINCLASS) ? LARGE_MINCLASS : s2u(size);
|
|
|
|
/*
|
|
* Calculate the size of the over-size run that arena_palloc()
|
|
* would need to allocate in order to guarantee the alignment.
|
|
*/
|
|
if (usize + alignment - PAGE <= arena_maxrun)
|
|
return (usize);
|
|
}
|
|
|
|
/* Huge size class. Beware of size_t overflow. */
|
|
|
|
/*
|
|
* We can't achieve subchunk alignment, so round up alignment to the
|
|
* minimum that can actually be supported.
|
|
*/
|
|
alignment = CHUNK_CEILING(alignment);
|
|
if (alignment == 0) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
|
|
/* Make sure result is a huge size class. */
|
|
if (size <= chunksize)
|
|
usize = chunksize;
|
|
else {
|
|
usize = s2u(size);
|
|
if (usize < size) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Calculate the multi-chunk mapping that huge_palloc() would need in
|
|
* order to guarantee the alignment.
|
|
*/
|
|
if (usize + alignment - PAGE < usize) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
return (usize);
|
|
}
|
|
|
|
/* Choose an arena based on a per-thread value. */
|
|
JEMALLOC_INLINE arena_t *
|
|
arena_choose(tsd_t *tsd, arena_t *arena)
|
|
{
|
|
arena_t *ret;
|
|
|
|
if (arena != NULL)
|
|
return (arena);
|
|
|
|
if (unlikely((ret = tsd_arena_get(tsd)) == NULL))
|
|
ret = arena_choose_hard(tsd);
|
|
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_INLINE arena_t *
|
|
arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing,
|
|
bool refresh_if_missing)
|
|
{
|
|
arena_t *arena;
|
|
arena_t **arenas_cache = tsd_arenas_cache_get(tsd);
|
|
|
|
/* init_if_missing requires refresh_if_missing. */
|
|
assert(!init_if_missing || refresh_if_missing);
|
|
|
|
if (unlikely(arenas_cache == NULL)) {
|
|
/* arenas_cache hasn't been initialized yet. */
|
|
return (arena_get_hard(tsd, ind, init_if_missing));
|
|
}
|
|
if (unlikely(ind >= tsd_narenas_cache_get(tsd))) {
|
|
/*
|
|
* ind is invalid, cache is old (too small), or arena to be
|
|
* initialized.
|
|
*/
|
|
return (refresh_if_missing ? arena_get_hard(tsd, ind,
|
|
init_if_missing) : NULL);
|
|
}
|
|
arena = arenas_cache[ind];
|
|
if (likely(arena != NULL) || !refresh_if_missing)
|
|
return (arena);
|
|
if (init_if_missing)
|
|
return (arena_get_hard(tsd, ind, init_if_missing));
|
|
else
|
|
return (NULL);
|
|
}
|
|
#endif
|
|
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
/*
|
|
* Include portions of arena.h interleaved with tcache.h in order to resolve
|
|
* circular dependencies.
|
|
*/
|
|
#define JEMALLOC_ARENA_INLINE_A
|
|
#include "jemalloc/internal/arena.h"
|
|
#undef JEMALLOC_ARENA_INLINE_A
|
|
#include "jemalloc/internal/tcache.h"
|
|
#define JEMALLOC_ARENA_INLINE_B
|
|
#include "jemalloc/internal/arena.h"
|
|
#undef JEMALLOC_ARENA_INLINE_B
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
|
|
void *imalloc(tsd_t *tsd, size_t size);
|
|
void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
|
|
void *icalloc(tsd_t *tsd, size_t size);
|
|
void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
|
|
bool try_tcache, arena_t *arena);
|
|
void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
|
|
size_t isalloc(const void *ptr, bool demote);
|
|
size_t ivsalloc(const void *ptr, bool demote);
|
|
size_t u2rz(size_t usize);
|
|
size_t p2rz(const void *ptr);
|
|
void idalloct(tsd_t *tsd, void *ptr, bool try_tcache);
|
|
void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
|
|
void idalloc(tsd_t *tsd, void *ptr);
|
|
void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache);
|
|
void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
|
|
void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
|
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
|
|
bool try_tcache_dalloc, arena_t *arena);
|
|
void *iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
|
|
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
|
|
void *iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
|
|
bool zero);
|
|
bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
|
|
bool zero);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
|
|
{
|
|
|
|
assert(size != 0);
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_malloc(tsd, arena, size, false, try_tcache));
|
|
else
|
|
return (huge_malloc(tsd, arena, size, false));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
imalloc(tsd_t *tsd, size_t size)
|
|
{
|
|
|
|
return (imalloct(tsd, size, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
|
|
{
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_malloc(tsd, arena, size, true, try_tcache));
|
|
else
|
|
return (huge_malloc(tsd, arena, size, true));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
icalloc(tsd_t *tsd, size_t size)
|
|
{
|
|
|
|
return (icalloct(tsd, size, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache,
|
|
arena_t *arena)
|
|
{
|
|
void *ret;
|
|
|
|
assert(usize != 0);
|
|
assert(usize == sa2u(usize, alignment));
|
|
|
|
if (usize <= SMALL_MAXCLASS && alignment < PAGE)
|
|
ret = arena_malloc(tsd, arena, usize, zero, try_tcache);
|
|
else {
|
|
if (usize <= arena_maxclass) {
|
|
arena = arena_choose(tsd, arena);
|
|
if (unlikely(arena == NULL))
|
|
return (NULL);
|
|
ret = arena_palloc(arena, usize, alignment, zero);
|
|
} else if (alignment <= chunksize)
|
|
ret = huge_malloc(tsd, arena, usize, zero);
|
|
else
|
|
ret = huge_palloc(tsd, arena, usize, alignment, zero);
|
|
}
|
|
|
|
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero)
|
|
{
|
|
|
|
return (ipalloct(tsd, usize, alignment, zero, true, NULL));
|
|
}
|
|
|
|
/*
|
|
* Typical usage:
|
|
* void *ptr = [...]
|
|
* size_t sz = isalloc(ptr, config_prof);
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
isalloc(const void *ptr, bool demote)
|
|
{
|
|
size_t ret;
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
/* Demotion only makes sense if config_prof is true. */
|
|
assert(config_prof || !demote);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
ret = arena_salloc(ptr, demote);
|
|
else
|
|
ret = huge_salloc(ptr);
|
|
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
ivsalloc(const void *ptr, bool demote)
|
|
{
|
|
|
|
/* Return 0 if ptr is not within a chunk managed by jemalloc. */
|
|
if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0)
|
|
return (0);
|
|
|
|
return (isalloc(ptr, demote));
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
u2rz(size_t usize)
|
|
{
|
|
size_t ret;
|
|
|
|
if (usize <= SMALL_MAXCLASS) {
|
|
index_t binind = size2index(usize);
|
|
ret = arena_bin_info[binind].redzone_size;
|
|
} else
|
|
ret = 0;
|
|
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
p2rz(const void *ptr)
|
|
{
|
|
size_t usize = isalloc(ptr, false);
|
|
|
|
return (u2rz(usize));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
idalloct(tsd_t *tsd, void *ptr, bool try_tcache)
|
|
{
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
arena_dalloc(tsd, chunk, ptr, try_tcache);
|
|
else
|
|
huge_dalloc(ptr);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
|
|
{
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
arena_sdalloc(tsd, chunk, ptr, size, try_tcache);
|
|
else
|
|
huge_dalloc(ptr);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
idalloc(tsd_t *tsd, void *ptr)
|
|
{
|
|
|
|
idalloct(tsd, ptr, true);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
iqalloc(tsd_t *tsd, void *ptr, bool try_tcache)
|
|
{
|
|
|
|
if (config_fill && unlikely(opt_quarantine))
|
|
quarantine(tsd, ptr);
|
|
else
|
|
idalloct(tsd, ptr, try_tcache);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
|
|
{
|
|
|
|
if (config_fill && unlikely(opt_quarantine))
|
|
quarantine(tsd, ptr);
|
|
else
|
|
isdalloct(tsd, ptr, size, try_tcache);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
|
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
|
|
bool try_tcache_dalloc, arena_t *arena)
|
|
{
|
|
void *p;
|
|
size_t usize, copysize;
|
|
|
|
usize = sa2u(size + extra, alignment);
|
|
if (usize == 0)
|
|
return (NULL);
|
|
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, arena);
|
|
if (p == NULL) {
|
|
if (extra == 0)
|
|
return (NULL);
|
|
/* Try again, without extra this time. */
|
|
usize = sa2u(size, alignment);
|
|
if (usize == 0)
|
|
return (NULL);
|
|
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc,
|
|
arena);
|
|
if (p == NULL)
|
|
return (NULL);
|
|
}
|
|
/*
|
|
* Copy at most size bytes (not size+extra), since the caller has no
|
|
* expectation that the extra bytes will be reliably preserved.
|
|
*/
|
|
copysize = (size < oldsize) ? size : oldsize;
|
|
memcpy(p, ptr, copysize);
|
|
iqalloc(tsd, ptr, try_tcache_dalloc);
|
|
return (p);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero,
|
|
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
|
|
{
|
|
size_t oldsize;
|
|
|
|
assert(ptr != NULL);
|
|
assert(size != 0);
|
|
|
|
oldsize = isalloc(ptr, config_prof);
|
|
|
|
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
|
|
!= 0) {
|
|
/*
|
|
* Existing object alignment is inadequate; allocate new space
|
|
* and copy.
|
|
*/
|
|
return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment,
|
|
zero, try_tcache_alloc, try_tcache_dalloc, arena));
|
|
}
|
|
|
|
if (size <= arena_maxclass) {
|
|
return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0,
|
|
alignment, zero, try_tcache_alloc, try_tcache_dalloc));
|
|
} else {
|
|
return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0,
|
|
alignment, zero, try_tcache_dalloc));
|
|
}
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero)
|
|
{
|
|
|
|
return (iralloct(tsd, ptr, size, alignment, zero, true, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE bool
|
|
ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
|
|
{
|
|
size_t oldsize;
|
|
|
|
assert(ptr != NULL);
|
|
assert(size != 0);
|
|
|
|
oldsize = isalloc(ptr, config_prof);
|
|
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
|
|
!= 0) {
|
|
/* Existing object alignment is inadequate. */
|
|
return (true);
|
|
}
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
|
|
else
|
|
return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero));
|
|
}
|
|
#endif
|
|
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_INLINES
|
|
/******************************************************************************/
|
|
#endif /* JEMALLOC_INTERNAL_H */
|