a95018ee81
This adds support for expanding huge allocations in-place by requesting
memory at a specific address from the chunk allocator.
It's currently only implemented for the chunk recycling path, although
in theory it could also be done by optimistically allocating new chunks.
On Linux, it could attempt an in-place mremap. However, that won't work
in practice since the heap is grown downwards and memory is not unmapped
(in a normal build, at least).
Repeated vector reallocation micro-benchmark:
#include <string.h>
#include <stdlib.h>
int main(void) {
for (size_t i = 0; i < 100; i++) {
void *ptr = NULL;
size_t old_size = 0;
for (size_t size = 4; size < (1 << 30); size *= 2) {
ptr = realloc(ptr, size);
if (!ptr) return 1;
memset(ptr + old_size, 0xff, size - old_size);
old_size = size;
}
free(ptr);
}
}
The glibc allocator fails to do any in-place reallocations on this
benchmark once it passes the M_MMAP_THRESHOLD (default 128k) but it
elides the cost of copies via mremap, which is currently not something
that jemalloc can use.
With this improvement, jemalloc still fails to do any in-place huge
reallocations for the first outer loop, but then succeeds 100% of the
time for the remaining 99 iterations. The time spent doing allocations
and copies drops down to under 5%, with nearly all of it spent doing
purging + faulting (when huge pages are disabled) and the array memset.
An improved mremap API (MREMAP_RETAIN - #138) would be far more general
but this is a portable optimization and would still be useful on Linux
for xallocx.
Numbers with transparent huge pages enabled:
glibc (copies elided via MREMAP_MAYMOVE): 8.471s
jemalloc: 17.816s
jemalloc + no-op madvise: 13.236s
jemalloc + this commit: 6.787s
jemalloc + this commit + no-op madvise: 6.144s
Numbers with transparent huge pages disabled:
glibc (copies elided via MREMAP_MAYMOVE): 15.403s
jemalloc: 39.456s
jemalloc + no-op madvise: 12.768s
jemalloc + this commit: 15.534s
jemalloc + this commit + no-op madvise: 6.354s
Closes #137
914 lines
23 KiB
C
914 lines
23 KiB
C
#ifndef JEMALLOC_INTERNAL_H
|
|
#define JEMALLOC_INTERNAL_H
|
|
|
|
#include "jemalloc_internal_defs.h"
|
|
#include "jemalloc/internal/jemalloc_internal_decls.h"
|
|
|
|
#ifdef JEMALLOC_UTRACE
|
|
#include <sys/ktrace.h>
|
|
#endif
|
|
|
|
#define JEMALLOC_NO_DEMANGLE
|
|
#ifdef JEMALLOC_JET
|
|
# define JEMALLOC_N(n) jet_##n
|
|
# include "jemalloc/internal/public_namespace.h"
|
|
# define JEMALLOC_NO_RENAME
|
|
# include "../jemalloc@install_suffix@.h"
|
|
# undef JEMALLOC_NO_RENAME
|
|
#else
|
|
# define JEMALLOC_N(n) @private_namespace@##n
|
|
# include "../jemalloc@install_suffix@.h"
|
|
#endif
|
|
#include "jemalloc/internal/private_namespace.h"
|
|
|
|
static const bool config_debug =
|
|
#ifdef JEMALLOC_DEBUG
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool have_dss =
|
|
#ifdef JEMALLOC_DSS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_fill =
|
|
#ifdef JEMALLOC_FILL
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_lazy_lock =
|
|
#ifdef JEMALLOC_LAZY_LOCK
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof =
|
|
#ifdef JEMALLOC_PROF
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof_libgcc =
|
|
#ifdef JEMALLOC_PROF_LIBGCC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof_libunwind =
|
|
#ifdef JEMALLOC_PROF_LIBUNWIND
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_munmap =
|
|
#ifdef JEMALLOC_MUNMAP
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_stats =
|
|
#ifdef JEMALLOC_STATS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_tcache =
|
|
#ifdef JEMALLOC_TCACHE
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_tls =
|
|
#ifdef JEMALLOC_TLS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_utrace =
|
|
#ifdef JEMALLOC_UTRACE
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_valgrind =
|
|
#ifdef JEMALLOC_VALGRIND
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_xmalloc =
|
|
#ifdef JEMALLOC_XMALLOC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_ivsalloc =
|
|
#ifdef JEMALLOC_IVSALLOC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
|
|
#ifdef JEMALLOC_ATOMIC9
|
|
#include <machine/atomic.h>
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
|
|
#include <libkern/OSAtomic.h>
|
|
#endif
|
|
|
|
#ifdef JEMALLOC_ZONE
|
|
#include <mach/mach_error.h>
|
|
#include <mach/mach_init.h>
|
|
#include <mach/vm_map.h>
|
|
#include <malloc/malloc.h>
|
|
#endif
|
|
|
|
#define RB_COMPACT
|
|
#include "jemalloc/internal/rb.h"
|
|
#include "jemalloc/internal/qr.h"
|
|
#include "jemalloc/internal/ql.h"
|
|
|
|
/*
|
|
* jemalloc can conceptually be broken into components (arena, tcache, etc.),
|
|
* but there are circular dependencies that cannot be broken without
|
|
* substantial performance degradation. In order to reduce the effect on
|
|
* visual code flow, read the header files in multiple passes, with one of the
|
|
* following cpp variables defined during each pass:
|
|
*
|
|
* JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data
|
|
* types.
|
|
* JEMALLOC_H_STRUCTS : Data structures.
|
|
* JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
|
|
* JEMALLOC_H_INLINES : Inline functions.
|
|
*/
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_TYPES
|
|
|
|
#include "jemalloc/internal/jemalloc_internal_macros.h"
|
|
|
|
#define MALLOCX_ARENA_MASK ((int)~0xff)
|
|
#define MALLOCX_LG_ALIGN_MASK ((int)0x3f)
|
|
/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */
|
|
#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \
|
|
(ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
|
|
#define MALLOCX_ALIGN_GET(flags) \
|
|
(MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1))
|
|
#define MALLOCX_ZERO_GET(flags) \
|
|
((bool)(flags & MALLOCX_ZERO))
|
|
#define MALLOCX_ARENA_GET(flags) \
|
|
(((unsigned)(flags >> 8)) - 1)
|
|
|
|
/* Smallest size class to support. */
|
|
#define LG_TINY_MIN 3
|
|
#define TINY_MIN (1U << LG_TINY_MIN)
|
|
|
|
/*
|
|
* Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size
|
|
* classes).
|
|
*/
|
|
#ifndef LG_QUANTUM
|
|
# if (defined(__i386__) || defined(_M_IX86))
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __ia64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __alpha__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __sparc64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64))
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __arm__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __aarch64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __hppa__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __mips__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __or1k__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __powerpc__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __s390__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __SH4__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __tile__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __le32__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifndef LG_QUANTUM
|
|
# error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS"
|
|
# endif
|
|
#endif
|
|
|
|
#define QUANTUM ((size_t)(1U << LG_QUANTUM))
|
|
#define QUANTUM_MASK (QUANTUM - 1)
|
|
|
|
/* Return the smallest quantum multiple that is >= a. */
|
|
#define QUANTUM_CEILING(a) \
|
|
(((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
|
|
|
|
#define LONG ((size_t)(1U << LG_SIZEOF_LONG))
|
|
#define LONG_MASK (LONG - 1)
|
|
|
|
/* Return the smallest long multiple that is >= a. */
|
|
#define LONG_CEILING(a) \
|
|
(((a) + LONG_MASK) & ~LONG_MASK)
|
|
|
|
#define SIZEOF_PTR (1U << LG_SIZEOF_PTR)
|
|
#define PTR_MASK (SIZEOF_PTR - 1)
|
|
|
|
/* Return the smallest (void *) multiple that is >= a. */
|
|
#define PTR_CEILING(a) \
|
|
(((a) + PTR_MASK) & ~PTR_MASK)
|
|
|
|
/*
|
|
* Maximum size of L1 cache line. This is used to avoid cache line aliasing.
|
|
* In addition, this controls the spacing of cacheline-spaced size classes.
|
|
*
|
|
* CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can
|
|
* only handle raw constants.
|
|
*/
|
|
#define LG_CACHELINE 6
|
|
#define CACHELINE 64
|
|
#define CACHELINE_MASK (CACHELINE - 1)
|
|
|
|
/* Return the smallest cacheline multiple that is >= s. */
|
|
#define CACHELINE_CEILING(s) \
|
|
(((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
|
|
|
|
/* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */
|
|
#ifdef PAGE_MASK
|
|
# undef PAGE_MASK
|
|
#endif
|
|
#define LG_PAGE STATIC_PAGE_SHIFT
|
|
#define PAGE ((size_t)(1U << STATIC_PAGE_SHIFT))
|
|
#define PAGE_MASK ((size_t)(PAGE - 1))
|
|
|
|
/* Return the smallest pagesize multiple that is >= s. */
|
|
#define PAGE_CEILING(s) \
|
|
(((s) + PAGE_MASK) & ~PAGE_MASK)
|
|
|
|
/* Return the nearest aligned address at or below a. */
|
|
#define ALIGNMENT_ADDR2BASE(a, alignment) \
|
|
((void *)((uintptr_t)(a) & (-(alignment))))
|
|
|
|
/* Return the offset between a and the nearest aligned address at or below a. */
|
|
#define ALIGNMENT_ADDR2OFFSET(a, alignment) \
|
|
((size_t)((uintptr_t)(a) & (alignment - 1)))
|
|
|
|
/* Return the smallest alignment multiple that is >= s. */
|
|
#define ALIGNMENT_CEILING(s, alignment) \
|
|
(((s) + (alignment - 1)) & (-(alignment)))
|
|
|
|
/* Declare a variable length array */
|
|
#if __STDC_VERSION__ < 199901L
|
|
# ifdef _MSC_VER
|
|
# include <malloc.h>
|
|
# define alloca _alloca
|
|
# else
|
|
# ifdef JEMALLOC_HAS_ALLOCA_H
|
|
# include <alloca.h>
|
|
# else
|
|
# include <stdlib.h>
|
|
# endif
|
|
# endif
|
|
# define VARIABLE_ARRAY(type, name, count) \
|
|
type *name = alloca(sizeof(type) * (count))
|
|
#else
|
|
# define VARIABLE_ARRAY(type, name, count) type name[(count)]
|
|
#endif
|
|
|
|
#include "jemalloc/internal/valgrind.h"
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_TYPES
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_STRUCTS
|
|
|
|
#include "jemalloc/internal/valgrind.h"
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#include "jemalloc/internal/tsd.h"
|
|
|
|
#undef JEMALLOC_H_STRUCTS
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_EXTERNS
|
|
|
|
extern bool opt_abort;
|
|
extern bool opt_junk;
|
|
extern size_t opt_quarantine;
|
|
extern bool opt_redzone;
|
|
extern bool opt_utrace;
|
|
extern bool opt_xmalloc;
|
|
extern bool opt_zero;
|
|
extern size_t opt_narenas;
|
|
|
|
extern bool in_valgrind;
|
|
|
|
/* Number of CPUs. */
|
|
extern unsigned ncpus;
|
|
|
|
/* Protects arenas initialization (arenas, arenas_total). */
|
|
extern malloc_mutex_t arenas_lock;
|
|
/*
|
|
* Arenas that are used to service external requests. Not all elements of the
|
|
* arenas array are necessarily used; arenas are created lazily as needed.
|
|
*
|
|
* arenas[0..narenas_auto) are used for automatic multiplexing of threads and
|
|
* arenas. arenas[narenas_auto..narenas_total) are only used if the application
|
|
* takes some action to create them and allocate from them.
|
|
*/
|
|
extern arena_t **arenas;
|
|
extern unsigned narenas_total;
|
|
extern unsigned narenas_auto; /* Read-only after initialization. */
|
|
|
|
arena_t *arenas_extend(unsigned ind);
|
|
arena_t *choose_arena_hard(tsd_t *tsd);
|
|
void thread_allocated_cleanup(tsd_t *tsd);
|
|
void thread_deallocated_cleanup(tsd_t *tsd);
|
|
void arena_cleanup(tsd_t *tsd);
|
|
void jemalloc_prefork(void);
|
|
void jemalloc_postfork_parent(void);
|
|
void jemalloc_postfork_child(void);
|
|
|
|
#include "jemalloc/internal/valgrind.h"
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
|
|
#undef JEMALLOC_H_EXTERNS
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_INLINES
|
|
|
|
#include "jemalloc/internal/valgrind.h"
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
|
|
/*
|
|
* Include arena.h the first time in order to provide inline functions for this
|
|
* header's inlines.
|
|
*/
|
|
#define JEMALLOC_ARENA_INLINE_A
|
|
#include "jemalloc/internal/arena.h"
|
|
#undef JEMALLOC_ARENA_INLINE_A
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
size_t s2u(size_t size);
|
|
size_t sa2u(size_t size, size_t alignment);
|
|
unsigned narenas_total_get(void);
|
|
arena_t *choose_arena(tsd_t *tsd, arena_t *arena);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
|
/*
|
|
* Compute usable size that would result from allocating an object with the
|
|
* specified size.
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
s2u(size_t size)
|
|
{
|
|
|
|
if (size <= SMALL_MAXCLASS)
|
|
return (small_s2u(size));
|
|
if (size <= arena_maxclass)
|
|
return (PAGE_CEILING(size));
|
|
return (CHUNK_CEILING(size));
|
|
}
|
|
|
|
/*
|
|
* Compute usable size that would result from allocating an object with the
|
|
* specified size and alignment.
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
sa2u(size_t size, size_t alignment)
|
|
{
|
|
size_t usize;
|
|
|
|
assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
|
|
|
|
/*
|
|
* Round size up to the nearest multiple of alignment.
|
|
*
|
|
* This done, we can take advantage of the fact that for each small
|
|
* size class, every object is aligned at the smallest power of two
|
|
* that is non-zero in the base two representation of the size. For
|
|
* example:
|
|
*
|
|
* Size | Base 2 | Minimum alignment
|
|
* -----+----------+------------------
|
|
* 96 | 1100000 | 32
|
|
* 144 | 10100000 | 32
|
|
* 192 | 11000000 | 64
|
|
*/
|
|
usize = ALIGNMENT_CEILING(size, alignment);
|
|
/*
|
|
* (usize < size) protects against the combination of maximal
|
|
* alignment and size greater than maximal alignment.
|
|
*/
|
|
if (usize < size) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
|
|
if (usize <= arena_maxclass && alignment <= PAGE) {
|
|
if (usize <= SMALL_MAXCLASS)
|
|
return (small_s2u(usize));
|
|
return (PAGE_CEILING(usize));
|
|
} else {
|
|
size_t run_size;
|
|
|
|
/*
|
|
* We can't achieve subpage alignment, so round up alignment
|
|
* permanently; it makes later calculations simpler.
|
|
*/
|
|
alignment = PAGE_CEILING(alignment);
|
|
usize = PAGE_CEILING(size);
|
|
/*
|
|
* (usize < size) protects against very large sizes within
|
|
* PAGE of SIZE_T_MAX.
|
|
*
|
|
* (usize + alignment < usize) protects against the
|
|
* combination of maximal alignment and usize large enough
|
|
* to cause overflow. This is similar to the first overflow
|
|
* check above, but it needs to be repeated due to the new
|
|
* usize value, which may now be *equal* to maximal
|
|
* alignment, whereas before we only detected overflow if the
|
|
* original size was *greater* than maximal alignment.
|
|
*/
|
|
if (usize < size || usize + alignment < usize) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
|
|
/*
|
|
* Calculate the size of the over-size run that arena_palloc()
|
|
* would need to allocate in order to guarantee the alignment.
|
|
* If the run wouldn't fit within a chunk, round up to a huge
|
|
* allocation size.
|
|
*/
|
|
run_size = usize + alignment - PAGE;
|
|
if (run_size <= arena_maxclass)
|
|
return (PAGE_CEILING(usize));
|
|
return (CHUNK_CEILING(usize));
|
|
}
|
|
}
|
|
|
|
JEMALLOC_INLINE unsigned
|
|
narenas_total_get(void)
|
|
{
|
|
unsigned narenas;
|
|
|
|
malloc_mutex_lock(&arenas_lock);
|
|
narenas = narenas_total;
|
|
malloc_mutex_unlock(&arenas_lock);
|
|
|
|
return (narenas);
|
|
}
|
|
|
|
/* Choose an arena based on a per-thread value. */
|
|
JEMALLOC_INLINE arena_t *
|
|
choose_arena(tsd_t *tsd, arena_t *arena)
|
|
{
|
|
arena_t *ret;
|
|
|
|
if (arena != NULL)
|
|
return (arena);
|
|
|
|
if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) {
|
|
ret = choose_arena_hard(tsd);
|
|
assert(ret != NULL);
|
|
}
|
|
|
|
return (ret);
|
|
}
|
|
#endif
|
|
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
/*
|
|
* Include arena.h the second and third times in order to resolve circular
|
|
* dependencies with tcache.h.
|
|
*/
|
|
#define JEMALLOC_ARENA_INLINE_B
|
|
#include "jemalloc/internal/arena.h"
|
|
#undef JEMALLOC_ARENA_INLINE_B
|
|
#include "jemalloc/internal/tcache.h"
|
|
#define JEMALLOC_ARENA_INLINE_C
|
|
#include "jemalloc/internal/arena.h"
|
|
#undef JEMALLOC_ARENA_INLINE_C
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
|
|
void *imalloc(tsd_t *tsd, size_t size);
|
|
void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena);
|
|
void *icalloc(tsd_t *tsd, size_t size);
|
|
void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
|
|
bool try_tcache, arena_t *arena);
|
|
void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
|
|
size_t isalloc(const void *ptr, bool demote);
|
|
size_t ivsalloc(const void *ptr, bool demote);
|
|
size_t u2rz(size_t usize);
|
|
size_t p2rz(const void *ptr);
|
|
void idalloct(tsd_t *tsd, void *ptr, bool try_tcache);
|
|
void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
|
|
void idalloc(tsd_t *tsd, void *ptr);
|
|
void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache);
|
|
void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache);
|
|
void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
|
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
|
|
bool try_tcache_dalloc, arena_t *arena);
|
|
void *iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
|
|
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
|
|
void *iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment,
|
|
bool zero);
|
|
bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
|
|
bool zero);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
|
|
{
|
|
|
|
assert(size != 0);
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_malloc(tsd, arena, size, false, try_tcache));
|
|
else
|
|
return (huge_malloc(tsd, arena, size, false));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
imalloc(tsd_t *tsd, size_t size)
|
|
{
|
|
|
|
return (imalloct(tsd, size, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
|
|
{
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_malloc(tsd, arena, size, true, try_tcache));
|
|
else
|
|
return (huge_malloc(tsd, arena, size, true));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
icalloc(tsd_t *tsd, size_t size)
|
|
{
|
|
|
|
return (icalloct(tsd, size, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache,
|
|
arena_t *arena)
|
|
{
|
|
void *ret;
|
|
|
|
assert(usize != 0);
|
|
assert(usize == sa2u(usize, alignment));
|
|
|
|
if (usize <= arena_maxclass && alignment <= PAGE)
|
|
ret = arena_malloc(tsd, arena, usize, zero, try_tcache);
|
|
else {
|
|
if (usize <= arena_maxclass) {
|
|
ret = arena_palloc(choose_arena(tsd, arena), usize,
|
|
alignment, zero);
|
|
} else if (alignment <= chunksize)
|
|
ret = huge_malloc(tsd, arena, usize, zero);
|
|
else
|
|
ret = huge_palloc(tsd, arena, usize, alignment, zero);
|
|
}
|
|
|
|
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero)
|
|
{
|
|
|
|
return (ipalloct(tsd, usize, alignment, zero, true, NULL));
|
|
}
|
|
|
|
/*
|
|
* Typical usage:
|
|
* void *ptr = [...]
|
|
* size_t sz = isalloc(ptr, config_prof);
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
isalloc(const void *ptr, bool demote)
|
|
{
|
|
size_t ret;
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
/* Demotion only makes sense if config_prof is true. */
|
|
assert(config_prof || !demote);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
ret = arena_salloc(ptr, demote);
|
|
else
|
|
ret = huge_salloc(ptr);
|
|
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
ivsalloc(const void *ptr, bool demote)
|
|
{
|
|
|
|
/* Return 0 if ptr is not within a chunk managed by jemalloc. */
|
|
if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0)
|
|
return (0);
|
|
|
|
return (isalloc(ptr, demote));
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
u2rz(size_t usize)
|
|
{
|
|
size_t ret;
|
|
|
|
if (usize <= SMALL_MAXCLASS) {
|
|
size_t binind = small_size2bin(usize);
|
|
ret = arena_bin_info[binind].redzone_size;
|
|
} else
|
|
ret = 0;
|
|
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
p2rz(const void *ptr)
|
|
{
|
|
size_t usize = isalloc(ptr, false);
|
|
|
|
return (u2rz(usize));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
idalloct(tsd_t *tsd, void *ptr, bool try_tcache)
|
|
{
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
arena_dalloc(tsd, chunk, ptr, try_tcache);
|
|
else
|
|
huge_dalloc(ptr);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
|
|
{
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
arena_sdalloc(tsd, chunk, ptr, size, try_tcache);
|
|
else
|
|
huge_dalloc(ptr);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
idalloc(tsd_t *tsd, void *ptr)
|
|
{
|
|
|
|
idalloct(tsd, ptr, true);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
iqalloc(tsd_t *tsd, void *ptr, bool try_tcache)
|
|
{
|
|
|
|
if (config_fill && unlikely(opt_quarantine))
|
|
quarantine(tsd, ptr);
|
|
else
|
|
idalloct(tsd, ptr, try_tcache);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
|
|
{
|
|
|
|
if (config_fill && unlikely(opt_quarantine))
|
|
quarantine(tsd, ptr);
|
|
else
|
|
isdalloct(tsd, ptr, size, try_tcache);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
|
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
|
|
bool try_tcache_dalloc, arena_t *arena)
|
|
{
|
|
void *p;
|
|
size_t usize, copysize;
|
|
|
|
usize = sa2u(size + extra, alignment);
|
|
if (usize == 0)
|
|
return (NULL);
|
|
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, arena);
|
|
if (p == NULL) {
|
|
if (extra == 0)
|
|
return (NULL);
|
|
/* Try again, without extra this time. */
|
|
usize = sa2u(size, alignment);
|
|
if (usize == 0)
|
|
return (NULL);
|
|
p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc,
|
|
arena);
|
|
if (p == NULL)
|
|
return (NULL);
|
|
}
|
|
/*
|
|
* Copy at most size bytes (not size+extra), since the caller has no
|
|
* expectation that the extra bytes will be reliably preserved.
|
|
*/
|
|
copysize = (size < oldsize) ? size : oldsize;
|
|
memcpy(p, ptr, copysize);
|
|
iqalloc(tsd, ptr, try_tcache_dalloc);
|
|
return (p);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero,
|
|
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
|
|
{
|
|
size_t oldsize;
|
|
|
|
assert(ptr != NULL);
|
|
assert(size != 0);
|
|
|
|
oldsize = isalloc(ptr, config_prof);
|
|
|
|
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
|
|
!= 0) {
|
|
/*
|
|
* Existing object alignment is inadequate; allocate new space
|
|
* and copy.
|
|
*/
|
|
return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment,
|
|
zero, try_tcache_alloc, try_tcache_dalloc, arena));
|
|
}
|
|
|
|
if (size <= arena_maxclass) {
|
|
return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0,
|
|
alignment, zero, try_tcache_alloc, try_tcache_dalloc));
|
|
} else {
|
|
return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0,
|
|
alignment, zero, try_tcache_dalloc));
|
|
}
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero)
|
|
{
|
|
|
|
return (iralloct(tsd, ptr, size, alignment, zero, true, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE bool
|
|
ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
|
|
{
|
|
size_t oldsize;
|
|
|
|
assert(ptr != NULL);
|
|
assert(size != 0);
|
|
|
|
oldsize = isalloc(ptr, config_prof);
|
|
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
|
|
!= 0) {
|
|
/* Existing object alignment is inadequate. */
|
|
return (true);
|
|
}
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
|
|
else
|
|
return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero));
|
|
}
|
|
#endif
|
|
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_INLINES
|
|
/******************************************************************************/
|
|
#endif /* JEMALLOC_INTERNAL_H */
|