b2c31660be
Extract profiling code from malloc(), imemalign(), calloc(), realloc(), mallocx(), rallocx(), and xallocx(). This slightly reduces the amount of code compiled into the fast paths, but the primary benefit is the combinatorial complexity reduction. Simplify iralloc[t]() by creating a separate ixalloc() that handles the no-move cases. Further simplify [mrxn]allocx() (and by implication [mrn]allocm()) to make request size overflows due to size class and/or alignment constraints trigger undefined behavior (detected by debug-only assertions). Report ENOMEM rather than EINVAL if an OOM occurs during heap profiling backtrace creation in imemalign(). This bug impacted posix_memalign() and aligned_alloc().
1028 lines
26 KiB
C
1028 lines
26 KiB
C
#ifndef JEMALLOC_INTERNAL_H
|
|
#define JEMALLOC_INTERNAL_H
|
|
#include <math.h>
|
|
#ifdef _WIN32
|
|
# include <windows.h>
|
|
# define ENOENT ERROR_PATH_NOT_FOUND
|
|
# define EINVAL ERROR_BAD_ARGUMENTS
|
|
# define EAGAIN ERROR_OUTOFMEMORY
|
|
# define EPERM ERROR_WRITE_FAULT
|
|
# define EFAULT ERROR_INVALID_ADDRESS
|
|
# define ENOMEM ERROR_NOT_ENOUGH_MEMORY
|
|
# undef ERANGE
|
|
# define ERANGE ERROR_INVALID_DATA
|
|
#else
|
|
# include <sys/param.h>
|
|
# include <sys/mman.h>
|
|
# include <sys/syscall.h>
|
|
# if !defined(SYS_write) && defined(__NR_write)
|
|
# define SYS_write __NR_write
|
|
# endif
|
|
# include <sys/uio.h>
|
|
# include <pthread.h>
|
|
# include <errno.h>
|
|
#endif
|
|
#include <sys/types.h>
|
|
|
|
#include <limits.h>
|
|
#ifndef SIZE_T_MAX
|
|
# define SIZE_T_MAX SIZE_MAX
|
|
#endif
|
|
#include <stdarg.h>
|
|
#include <stdbool.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <stdint.h>
|
|
#include <stddef.h>
|
|
#ifndef offsetof
|
|
# define offsetof(type, member) ((size_t)&(((type *)NULL)->member))
|
|
#endif
|
|
#include <inttypes.h>
|
|
#include <string.h>
|
|
#include <strings.h>
|
|
#include <ctype.h>
|
|
#ifdef _MSC_VER
|
|
# include <io.h>
|
|
typedef intptr_t ssize_t;
|
|
# define PATH_MAX 1024
|
|
# define STDERR_FILENO 2
|
|
# define __func__ __FUNCTION__
|
|
/* Disable warnings about deprecated system functions */
|
|
# pragma warning(disable: 4996)
|
|
#else
|
|
# include <unistd.h>
|
|
#endif
|
|
#include <fcntl.h>
|
|
|
|
#include "jemalloc_internal_defs.h"
|
|
|
|
#ifdef JEMALLOC_UTRACE
|
|
#include <sys/ktrace.h>
|
|
#endif
|
|
|
|
#ifdef JEMALLOC_VALGRIND
|
|
#include <valgrind/valgrind.h>
|
|
#include <valgrind/memcheck.h>
|
|
#endif
|
|
|
|
#define JEMALLOC_NO_DEMANGLE
|
|
#ifdef JEMALLOC_JET
|
|
# define JEMALLOC_N(n) jet_##n
|
|
# include "jemalloc/internal/public_namespace.h"
|
|
# define JEMALLOC_NO_RENAME
|
|
# include "../jemalloc@install_suffix@.h"
|
|
#else
|
|
# define JEMALLOC_N(n) @private_namespace@##n
|
|
# include "../jemalloc@install_suffix@.h"
|
|
#endif
|
|
#include "jemalloc/internal/private_namespace.h"
|
|
|
|
static const bool config_debug =
|
|
#ifdef JEMALLOC_DEBUG
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_dss =
|
|
#ifdef JEMALLOC_DSS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_fill =
|
|
#ifdef JEMALLOC_FILL
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_lazy_lock =
|
|
#ifdef JEMALLOC_LAZY_LOCK
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof =
|
|
#ifdef JEMALLOC_PROF
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof_libgcc =
|
|
#ifdef JEMALLOC_PROF_LIBGCC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof_libunwind =
|
|
#ifdef JEMALLOC_PROF_LIBUNWIND
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_mremap =
|
|
#ifdef JEMALLOC_MREMAP
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_munmap =
|
|
#ifdef JEMALLOC_MUNMAP
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_stats =
|
|
#ifdef JEMALLOC_STATS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_tcache =
|
|
#ifdef JEMALLOC_TCACHE
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_tls =
|
|
#ifdef JEMALLOC_TLS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_utrace =
|
|
#ifdef JEMALLOC_UTRACE
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_valgrind =
|
|
#ifdef JEMALLOC_VALGRIND
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_xmalloc =
|
|
#ifdef JEMALLOC_XMALLOC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_ivsalloc =
|
|
#ifdef JEMALLOC_IVSALLOC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
|
|
#ifdef JEMALLOC_ATOMIC9
|
|
#include <machine/atomic.h>
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
|
|
#include <libkern/OSAtomic.h>
|
|
#endif
|
|
|
|
#ifdef JEMALLOC_ZONE
|
|
#include <mach/mach_error.h>
|
|
#include <mach/mach_init.h>
|
|
#include <mach/vm_map.h>
|
|
#include <malloc/malloc.h>
|
|
#endif
|
|
|
|
#define RB_COMPACT
|
|
#include "jemalloc/internal/rb.h"
|
|
#include "jemalloc/internal/qr.h"
|
|
#include "jemalloc/internal/ql.h"
|
|
|
|
/*
|
|
* jemalloc can conceptually be broken into components (arena, tcache, etc.),
|
|
* but there are circular dependencies that cannot be broken without
|
|
* substantial performance degradation. In order to reduce the effect on
|
|
* visual code flow, read the header files in multiple passes, with one of the
|
|
* following cpp variables defined during each pass:
|
|
*
|
|
* JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data
|
|
* types.
|
|
* JEMALLOC_H_STRUCTS : Data structures.
|
|
* JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
|
|
* JEMALLOC_H_INLINES : Inline functions.
|
|
*/
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_TYPES
|
|
|
|
#include "jemalloc/internal/jemalloc_internal_macros.h"
|
|
|
|
#define MALLOCX_LG_ALIGN_MASK ((int)0x3f)
|
|
#define ALLOCM_LG_ALIGN_MASK ((int)0x3f)
|
|
|
|
/* Smallest size class to support. */
|
|
#define LG_TINY_MIN 3
|
|
#define TINY_MIN (1U << LG_TINY_MIN)
|
|
|
|
/*
|
|
* Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size
|
|
* classes).
|
|
*/
|
|
#ifndef LG_QUANTUM
|
|
# if (defined(__i386__) || defined(_M_IX86))
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __ia64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __alpha__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __sparc64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64))
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __arm__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __aarch64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __hppa__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __mips__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __powerpc__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __s390__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __SH4__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __tile__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifndef LG_QUANTUM
|
|
# error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS"
|
|
# endif
|
|
#endif
|
|
|
|
#define QUANTUM ((size_t)(1U << LG_QUANTUM))
|
|
#define QUANTUM_MASK (QUANTUM - 1)
|
|
|
|
/* Return the smallest quantum multiple that is >= a. */
|
|
#define QUANTUM_CEILING(a) \
|
|
(((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
|
|
|
|
#define LONG ((size_t)(1U << LG_SIZEOF_LONG))
|
|
#define LONG_MASK (LONG - 1)
|
|
|
|
/* Return the smallest long multiple that is >= a. */
|
|
#define LONG_CEILING(a) \
|
|
(((a) + LONG_MASK) & ~LONG_MASK)
|
|
|
|
#define SIZEOF_PTR (1U << LG_SIZEOF_PTR)
|
|
#define PTR_MASK (SIZEOF_PTR - 1)
|
|
|
|
/* Return the smallest (void *) multiple that is >= a. */
|
|
#define PTR_CEILING(a) \
|
|
(((a) + PTR_MASK) & ~PTR_MASK)
|
|
|
|
/*
|
|
* Maximum size of L1 cache line. This is used to avoid cache line aliasing.
|
|
* In addition, this controls the spacing of cacheline-spaced size classes.
|
|
*
|
|
* CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can
|
|
* only handle raw constants.
|
|
*/
|
|
#define LG_CACHELINE 6
|
|
#define CACHELINE 64
|
|
#define CACHELINE_MASK (CACHELINE - 1)
|
|
|
|
/* Return the smallest cacheline multiple that is >= s. */
|
|
#define CACHELINE_CEILING(s) \
|
|
(((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
|
|
|
|
/* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */
|
|
#ifdef PAGE_MASK
|
|
# undef PAGE_MASK
|
|
#endif
|
|
#define LG_PAGE STATIC_PAGE_SHIFT
|
|
#define PAGE ((size_t)(1U << STATIC_PAGE_SHIFT))
|
|
#define PAGE_MASK ((size_t)(PAGE - 1))
|
|
|
|
/* Return the smallest pagesize multiple that is >= s. */
|
|
#define PAGE_CEILING(s) \
|
|
(((s) + PAGE_MASK) & ~PAGE_MASK)
|
|
|
|
/* Return the nearest aligned address at or below a. */
|
|
#define ALIGNMENT_ADDR2BASE(a, alignment) \
|
|
((void *)((uintptr_t)(a) & (-(alignment))))
|
|
|
|
/* Return the offset between a and the nearest aligned address at or below a. */
|
|
#define ALIGNMENT_ADDR2OFFSET(a, alignment) \
|
|
((size_t)((uintptr_t)(a) & (alignment - 1)))
|
|
|
|
/* Return the smallest alignment multiple that is >= s. */
|
|
#define ALIGNMENT_CEILING(s, alignment) \
|
|
(((s) + (alignment - 1)) & (-(alignment)))
|
|
|
|
/* Declare a variable length array */
|
|
#if __STDC_VERSION__ < 199901L
|
|
# ifdef _MSC_VER
|
|
# include <malloc.h>
|
|
# define alloca _alloca
|
|
# else
|
|
# ifdef JEMALLOC_HAS_ALLOCA_H
|
|
# include <alloca.h>
|
|
# else
|
|
# include <stdlib.h>
|
|
# endif
|
|
# endif
|
|
# define VARIABLE_ARRAY(type, name, count) \
|
|
type *name = alloca(sizeof(type) * count)
|
|
#else
|
|
# define VARIABLE_ARRAY(type, name, count) type name[count]
|
|
#endif
|
|
|
|
#ifdef JEMALLOC_VALGRIND
|
|
/*
|
|
* The JEMALLOC_VALGRIND_*() macros must be macros rather than functions
|
|
* so that when Valgrind reports errors, there are no extra stack frames
|
|
* in the backtraces.
|
|
*
|
|
* The size that is reported to valgrind must be consistent through a chain of
|
|
* malloc..realloc..realloc calls. Request size isn't recorded anywhere in
|
|
* jemalloc, so it is critical that all callers of these macros provide usize
|
|
* rather than request size. As a result, buffer overflow detection is
|
|
* technically weakened for the standard API, though it is generally accepted
|
|
* practice to consider any extra bytes reported by malloc_usable_size() as
|
|
* usable space.
|
|
*/
|
|
#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \
|
|
if (config_valgrind && opt_valgrind && cond) \
|
|
VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \
|
|
} while (0)
|
|
#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \
|
|
old_rzsize, zero) do { \
|
|
if (config_valgrind && opt_valgrind) { \
|
|
size_t rzsize = p2rz(ptr); \
|
|
\
|
|
if (ptr == old_ptr) { \
|
|
VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \
|
|
usize, rzsize); \
|
|
if (zero && old_usize < usize) { \
|
|
VALGRIND_MAKE_MEM_DEFINED( \
|
|
(void *)((uintptr_t)ptr + \
|
|
old_usize), usize - old_usize); \
|
|
} \
|
|
} else { \
|
|
if (old_ptr != NULL) { \
|
|
VALGRIND_FREELIKE_BLOCK(old_ptr, \
|
|
old_rzsize); \
|
|
} \
|
|
if (ptr != NULL) { \
|
|
size_t copy_size = (old_usize < usize) \
|
|
? old_usize : usize; \
|
|
size_t tail_size = usize - copy_size; \
|
|
VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \
|
|
rzsize, false); \
|
|
if (copy_size > 0) { \
|
|
VALGRIND_MAKE_MEM_DEFINED(ptr, \
|
|
copy_size); \
|
|
} \
|
|
if (zero && tail_size > 0) { \
|
|
VALGRIND_MAKE_MEM_DEFINED( \
|
|
(void *)((uintptr_t)ptr + \
|
|
copy_size), tail_size); \
|
|
} \
|
|
} \
|
|
} \
|
|
} \
|
|
} while (0)
|
|
#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \
|
|
if (config_valgrind && opt_valgrind) \
|
|
VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \
|
|
} while (0)
|
|
#else
|
|
#define RUNNING_ON_VALGRIND ((unsigned)0)
|
|
#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \
|
|
do {} while (0)
|
|
#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \
|
|
do {} while (0)
|
|
#define VALGRIND_FREELIKE_BLOCK(addr, rzB) do {} while (0)
|
|
#define VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr, _qzz_len) do {} while (0)
|
|
#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) do {} while (0)
|
|
#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) do {} while (0)
|
|
#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0)
|
|
#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \
|
|
old_rzsize, zero) do {} while (0)
|
|
#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0)
|
|
#endif
|
|
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_TYPES
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_STRUCTS
|
|
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
typedef struct {
|
|
uint64_t allocated;
|
|
uint64_t deallocated;
|
|
} thread_allocated_t;
|
|
/*
|
|
* The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro
|
|
* argument.
|
|
*/
|
|
#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_ARG_CONCAT({0, 0})
|
|
|
|
#undef JEMALLOC_H_STRUCTS
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_EXTERNS
|
|
|
|
extern bool opt_abort;
|
|
extern bool opt_junk;
|
|
extern size_t opt_quarantine;
|
|
extern bool opt_redzone;
|
|
extern bool opt_utrace;
|
|
extern bool opt_valgrind;
|
|
extern bool opt_xmalloc;
|
|
extern bool opt_zero;
|
|
extern size_t opt_narenas;
|
|
|
|
/* Number of CPUs. */
|
|
extern unsigned ncpus;
|
|
|
|
/* Protects arenas initialization (arenas, arenas_total). */
|
|
extern malloc_mutex_t arenas_lock;
|
|
/*
|
|
* Arenas that are used to service external requests. Not all elements of the
|
|
* arenas array are necessarily used; arenas are created lazily as needed.
|
|
*
|
|
* arenas[0..narenas_auto) are used for automatic multiplexing of threads and
|
|
* arenas. arenas[narenas_auto..narenas_total) are only used if the application
|
|
* takes some action to create them and allocate from them.
|
|
*/
|
|
extern arena_t **arenas;
|
|
extern unsigned narenas_total;
|
|
extern unsigned narenas_auto; /* Read-only after initialization. */
|
|
|
|
arena_t *arenas_extend(unsigned ind);
|
|
void arenas_cleanup(void *arg);
|
|
arena_t *choose_arena_hard(void);
|
|
void jemalloc_prefork(void);
|
|
void jemalloc_postfork_parent(void);
|
|
void jemalloc_postfork_child(void);
|
|
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_EXTERNS
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_INLINES
|
|
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
|
|
|
|
size_t s2u(size_t size);
|
|
size_t sa2u(size_t size, size_t alignment);
|
|
unsigned narenas_total_get(void);
|
|
arena_t *choose_arena(arena_t *arena);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
|
/*
|
|
* Map of pthread_self() --> arenas[???], used for selecting an arena to use
|
|
* for allocations.
|
|
*/
|
|
malloc_tsd_externs(arenas, arena_t *)
|
|
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL,
|
|
arenas_cleanup)
|
|
|
|
/*
|
|
* Compute usable size that would result from allocating an object with the
|
|
* specified size.
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
s2u(size_t size)
|
|
{
|
|
|
|
if (size <= SMALL_MAXCLASS)
|
|
return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size);
|
|
if (size <= arena_maxclass)
|
|
return (PAGE_CEILING(size));
|
|
return (CHUNK_CEILING(size));
|
|
}
|
|
|
|
/*
|
|
* Compute usable size that would result from allocating an object with the
|
|
* specified size and alignment.
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
sa2u(size_t size, size_t alignment)
|
|
{
|
|
size_t usize;
|
|
|
|
assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
|
|
|
|
/*
|
|
* Round size up to the nearest multiple of alignment.
|
|
*
|
|
* This done, we can take advantage of the fact that for each small
|
|
* size class, every object is aligned at the smallest power of two
|
|
* that is non-zero in the base two representation of the size. For
|
|
* example:
|
|
*
|
|
* Size | Base 2 | Minimum alignment
|
|
* -----+----------+------------------
|
|
* 96 | 1100000 | 32
|
|
* 144 | 10100000 | 32
|
|
* 192 | 11000000 | 64
|
|
*/
|
|
usize = ALIGNMENT_CEILING(size, alignment);
|
|
/*
|
|
* (usize < size) protects against the combination of maximal
|
|
* alignment and size greater than maximal alignment.
|
|
*/
|
|
if (usize < size) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
|
|
if (usize <= arena_maxclass && alignment <= PAGE) {
|
|
if (usize <= SMALL_MAXCLASS)
|
|
return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size);
|
|
return (PAGE_CEILING(usize));
|
|
} else {
|
|
size_t run_size;
|
|
|
|
/*
|
|
* We can't achieve subpage alignment, so round up alignment
|
|
* permanently; it makes later calculations simpler.
|
|
*/
|
|
alignment = PAGE_CEILING(alignment);
|
|
usize = PAGE_CEILING(size);
|
|
/*
|
|
* (usize < size) protects against very large sizes within
|
|
* PAGE of SIZE_T_MAX.
|
|
*
|
|
* (usize + alignment < usize) protects against the
|
|
* combination of maximal alignment and usize large enough
|
|
* to cause overflow. This is similar to the first overflow
|
|
* check above, but it needs to be repeated due to the new
|
|
* usize value, which may now be *equal* to maximal
|
|
* alignment, whereas before we only detected overflow if the
|
|
* original size was *greater* than maximal alignment.
|
|
*/
|
|
if (usize < size || usize + alignment < usize) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
|
|
/*
|
|
* Calculate the size of the over-size run that arena_palloc()
|
|
* would need to allocate in order to guarantee the alignment.
|
|
* If the run wouldn't fit within a chunk, round up to a huge
|
|
* allocation size.
|
|
*/
|
|
run_size = usize + alignment - PAGE;
|
|
if (run_size <= arena_maxclass)
|
|
return (PAGE_CEILING(usize));
|
|
return (CHUNK_CEILING(usize));
|
|
}
|
|
}
|
|
|
|
JEMALLOC_INLINE unsigned
|
|
narenas_total_get(void)
|
|
{
|
|
unsigned narenas;
|
|
|
|
malloc_mutex_lock(&arenas_lock);
|
|
narenas = narenas_total;
|
|
malloc_mutex_unlock(&arenas_lock);
|
|
|
|
return (narenas);
|
|
}
|
|
|
|
/* Choose an arena based on a per-thread value. */
|
|
JEMALLOC_INLINE arena_t *
|
|
choose_arena(arena_t *arena)
|
|
{
|
|
arena_t *ret;
|
|
|
|
if (arena != NULL)
|
|
return (arena);
|
|
|
|
if ((ret = *arenas_tsd_get()) == NULL) {
|
|
ret = choose_arena_hard();
|
|
assert(ret != NULL);
|
|
}
|
|
|
|
return (ret);
|
|
}
|
|
#endif
|
|
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
/*
|
|
* Include arena.h twice in order to resolve circular dependencies with
|
|
* tcache.h.
|
|
*/
|
|
#define JEMALLOC_ARENA_INLINE_A
|
|
#include "jemalloc/internal/arena.h"
|
|
#undef JEMALLOC_ARENA_INLINE_A
|
|
#include "jemalloc/internal/tcache.h"
|
|
#define JEMALLOC_ARENA_INLINE_B
|
|
#include "jemalloc/internal/arena.h"
|
|
#undef JEMALLOC_ARENA_INLINE_B
|
|
#include "jemalloc/internal/hash.h"
|
|
#include "jemalloc/internal/quarantine.h"
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
void *imalloct(size_t size, bool try_tcache, arena_t *arena);
|
|
void *imalloc(size_t size);
|
|
void *icalloct(size_t size, bool try_tcache, arena_t *arena);
|
|
void *icalloc(size_t size);
|
|
void *ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
|
|
arena_t *arena);
|
|
void *ipalloc(size_t usize, size_t alignment, bool zero);
|
|
size_t isalloc(const void *ptr, bool demote);
|
|
size_t ivsalloc(const void *ptr, bool demote);
|
|
size_t u2rz(size_t usize);
|
|
size_t p2rz(const void *ptr);
|
|
void idalloct(void *ptr, bool try_tcache);
|
|
void idalloc(void *ptr);
|
|
void iqalloct(void *ptr, bool try_tcache);
|
|
void iqalloc(void *ptr);
|
|
void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
|
|
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
|
|
arena_t *arena);
|
|
void *iralloct(void *ptr, size_t size, size_t extra, size_t alignment,
|
|
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
|
|
void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
|
|
bool zero);
|
|
bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
|
|
bool zero);
|
|
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
imalloct(size_t size, bool try_tcache, arena_t *arena)
|
|
{
|
|
|
|
assert(size != 0);
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_malloc(arena, size, false, try_tcache));
|
|
else
|
|
return (huge_malloc(size, false));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
imalloc(size_t size)
|
|
{
|
|
|
|
return (imalloct(size, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
icalloct(size_t size, bool try_tcache, arena_t *arena)
|
|
{
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_malloc(arena, size, true, try_tcache));
|
|
else
|
|
return (huge_malloc(size, true));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
icalloc(size_t size)
|
|
{
|
|
|
|
return (icalloct(size, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
|
|
arena_t *arena)
|
|
{
|
|
void *ret;
|
|
|
|
assert(usize != 0);
|
|
assert(usize == sa2u(usize, alignment));
|
|
|
|
if (usize <= arena_maxclass && alignment <= PAGE)
|
|
ret = arena_malloc(arena, usize, zero, try_tcache);
|
|
else {
|
|
if (usize <= arena_maxclass) {
|
|
ret = arena_palloc(choose_arena(arena), usize,
|
|
alignment, zero);
|
|
} else if (alignment <= chunksize)
|
|
ret = huge_malloc(usize, zero);
|
|
else
|
|
ret = huge_palloc(usize, alignment, zero);
|
|
}
|
|
|
|
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
ipalloc(size_t usize, size_t alignment, bool zero)
|
|
{
|
|
|
|
return (ipalloct(usize, alignment, zero, true, NULL));
|
|
}
|
|
|
|
/*
|
|
* Typical usage:
|
|
* void *ptr = [...]
|
|
* size_t sz = isalloc(ptr, config_prof);
|
|
*/
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
isalloc(const void *ptr, bool demote)
|
|
{
|
|
size_t ret;
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
/* Demotion only makes sense if config_prof is true. */
|
|
assert(config_prof || demote == false);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
ret = arena_salloc(ptr, demote);
|
|
else
|
|
ret = huge_salloc(ptr);
|
|
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE size_t
|
|
ivsalloc(const void *ptr, bool demote)
|
|
{
|
|
|
|
/* Return 0 if ptr is not within a chunk managed by jemalloc. */
|
|
if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0)
|
|
return (0);
|
|
|
|
return (isalloc(ptr, demote));
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
u2rz(size_t usize)
|
|
{
|
|
size_t ret;
|
|
|
|
if (usize <= SMALL_MAXCLASS) {
|
|
size_t binind = SMALL_SIZE2BIN(usize);
|
|
ret = arena_bin_info[binind].redzone_size;
|
|
} else
|
|
ret = 0;
|
|
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
p2rz(const void *ptr)
|
|
{
|
|
size_t usize = isalloc(ptr, false);
|
|
|
|
return (u2rz(usize));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
idalloct(void *ptr, bool try_tcache)
|
|
{
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
arena_dalloc(chunk->arena, chunk, ptr, try_tcache);
|
|
else
|
|
huge_dalloc(ptr, true);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
idalloc(void *ptr)
|
|
{
|
|
|
|
idalloct(ptr, true);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
iqalloct(void *ptr, bool try_tcache)
|
|
{
|
|
|
|
if (config_fill && opt_quarantine)
|
|
quarantine(ptr);
|
|
else
|
|
idalloct(ptr, try_tcache);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
iqalloc(void *ptr)
|
|
{
|
|
|
|
iqalloct(ptr, true);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
|
|
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
|
|
arena_t *arena)
|
|
{
|
|
void *p;
|
|
size_t usize, copysize;
|
|
|
|
usize = sa2u(size + extra, alignment);
|
|
if (usize == 0)
|
|
return (NULL);
|
|
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
|
|
if (p == NULL) {
|
|
if (extra == 0)
|
|
return (NULL);
|
|
/* Try again, without extra this time. */
|
|
usize = sa2u(size, alignment);
|
|
if (usize == 0)
|
|
return (NULL);
|
|
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
|
|
if (p == NULL)
|
|
return (NULL);
|
|
}
|
|
/*
|
|
* Copy at most size bytes (not size+extra), since the caller has no
|
|
* expectation that the extra bytes will be reliably preserved.
|
|
*/
|
|
copysize = (size < oldsize) ? size : oldsize;
|
|
memcpy(p, ptr, copysize);
|
|
iqalloct(ptr, try_tcache_dalloc);
|
|
return (p);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
|
|
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
|
|
{
|
|
size_t oldsize;
|
|
|
|
assert(ptr != NULL);
|
|
assert(size != 0);
|
|
|
|
oldsize = isalloc(ptr, config_prof);
|
|
|
|
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
|
|
!= 0) {
|
|
/*
|
|
* Existing object alignment is inadequate; allocate new space
|
|
* and copy.
|
|
*/
|
|
return (iralloct_realign(ptr, oldsize, size, extra, alignment,
|
|
zero, try_tcache_alloc, try_tcache_dalloc, arena));
|
|
}
|
|
|
|
if (size + extra <= arena_maxclass) {
|
|
return (arena_ralloc(arena, ptr, oldsize, size, extra,
|
|
alignment, zero, try_tcache_alloc,
|
|
try_tcache_dalloc));
|
|
} else {
|
|
return (huge_ralloc(ptr, oldsize, size, extra,
|
|
alignment, zero, try_tcache_dalloc));
|
|
}
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void *
|
|
iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
|
|
{
|
|
|
|
return (iralloct(ptr, size, extra, alignment, zero, true, true, NULL));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE bool
|
|
ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
|
|
{
|
|
size_t oldsize;
|
|
|
|
assert(ptr != NULL);
|
|
assert(size != 0);
|
|
|
|
oldsize = isalloc(ptr, config_prof);
|
|
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
|
|
!= 0) {
|
|
/* Existing object alignment is inadequate. */
|
|
return (true);
|
|
}
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
|
|
else
|
|
return (huge_ralloc_no_move(ptr, oldsize, size, extra));
|
|
}
|
|
|
|
malloc_tsd_externs(thread_allocated, thread_allocated_t)
|
|
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t,
|
|
THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
|
|
#endif
|
|
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_INLINES
|
|
/******************************************************************************/
|
|
#endif /* JEMALLOC_INTERNAL_H */
|