41b6afb834
Use FreeBSD-specific functions (_pthread_mutex_init_calloc_cb(),
_malloc_{pre,post}fork()) to avoid bootstrapping issues due to
allocation in libc and libthr.
Add malloc_strtoumax() and use it instead of strtoul(). Disable
validation code in malloc_vsnprintf() and malloc_strtoumax() until
jemalloc is initialized. This is necessary because locale
initialization causes allocation for both vsnprintf() and strtoumax().
Force the lazy-lock feature on in order to avoid pthread_self(),
because it causes allocation.
Use syscall(SYS_write, ...) rather than write(...), because libthr wraps
write() and causes allocation. Without this workaround, it would not be
possible to print error messages in malloc_conf_init() without
substantially reworking bootstrapping.
Fix choose_arena_hard() to look at how many threads are assigned to the
candidate choice, rather than checking whether the arena is
uninitialized. This bug potentially caused more arenas to be
initialized than necessary.
763 lines
19 KiB
C
763 lines
19 KiB
C
#include <sys/mman.h>
|
|
#include <sys/param.h>
|
|
#include <sys/syscall.h>
|
|
#include <sys/time.h>
|
|
#include <sys/types.h>
|
|
#include <sys/uio.h>
|
|
|
|
#include <errno.h>
|
|
#include <limits.h>
|
|
#ifndef SIZE_T_MAX
|
|
# define SIZE_T_MAX SIZE_MAX
|
|
#endif
|
|
#include <pthread.h>
|
|
#include <sched.h>
|
|
#include <stdarg.h>
|
|
#include <stdbool.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <stdint.h>
|
|
#include <stddef.h>
|
|
#ifndef offsetof
|
|
# define offsetof(type, member) ((size_t)&(((type *)NULL)->member))
|
|
#endif
|
|
#include <inttypes.h>
|
|
#include <string.h>
|
|
#include <strings.h>
|
|
#include <ctype.h>
|
|
#include <unistd.h>
|
|
#include <fcntl.h>
|
|
#include <pthread.h>
|
|
#include <math.h>
|
|
|
|
#define JEMALLOC_NO_DEMANGLE
|
|
#include "../jemalloc@install_suffix@.h"
|
|
|
|
#include "jemalloc/internal/private_namespace.h"
|
|
|
|
#ifdef JEMALLOC_CC_SILENCE
|
|
#define UNUSED JEMALLOC_ATTR(unused)
|
|
#else
|
|
#define UNUSED
|
|
#endif
|
|
|
|
static const bool config_debug =
|
|
#ifdef JEMALLOC_DEBUG
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_dss =
|
|
#ifdef JEMALLOC_DSS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_dynamic_page_shift =
|
|
#ifdef JEMALLOC_DYNAMIC_PAGE_SHIFT
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_fill =
|
|
#ifdef JEMALLOC_FILL
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_lazy_lock =
|
|
#ifdef JEMALLOC_LAZY_LOCK
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof =
|
|
#ifdef JEMALLOC_PROF
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof_libgcc =
|
|
#ifdef JEMALLOC_PROF_LIBGCC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_prof_libunwind =
|
|
#ifdef JEMALLOC_PROF_LIBUNWIND
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_stats =
|
|
#ifdef JEMALLOC_STATS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_tcache =
|
|
#ifdef JEMALLOC_TCACHE
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_tls =
|
|
#ifdef JEMALLOC_TLS
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_xmalloc =
|
|
#ifdef JEMALLOC_XMALLOC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
static const bool config_ivsalloc =
|
|
#ifdef JEMALLOC_IVSALLOC
|
|
true
|
|
#else
|
|
false
|
|
#endif
|
|
;
|
|
|
|
#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
|
|
#include <libkern/OSAtomic.h>
|
|
#endif
|
|
|
|
#ifdef JEMALLOC_ZONE
|
|
#include <mach/mach_error.h>
|
|
#include <mach/mach_init.h>
|
|
#include <mach/vm_map.h>
|
|
#include <malloc/malloc.h>
|
|
#endif
|
|
|
|
#define RB_COMPACT
|
|
#include "jemalloc/internal/rb.h"
|
|
#include "jemalloc/internal/qr.h"
|
|
#include "jemalloc/internal/ql.h"
|
|
|
|
/*
|
|
* jemalloc can conceptually be broken into components (arena, tcache, etc.),
|
|
* but there are circular dependencies that cannot be broken without
|
|
* substantial performance degradation. In order to reduce the effect on
|
|
* visual code flow, read the header files in multiple passes, with one of the
|
|
* following cpp variables defined during each pass:
|
|
*
|
|
* JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data
|
|
* types.
|
|
* JEMALLOC_H_STRUCTS : Data structures.
|
|
* JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
|
|
* JEMALLOC_H_INLINES : Inline functions.
|
|
*/
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_TYPES
|
|
|
|
#define ALLOCM_LG_ALIGN_MASK ((int)0x3f)
|
|
|
|
#define ZU(z) ((size_t)z)
|
|
|
|
#ifndef __DECONST
|
|
# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
|
|
#endif
|
|
|
|
#ifdef JEMALLOC_DEBUG
|
|
/* Disable inlining to make debugging easier. */
|
|
# define JEMALLOC_INLINE
|
|
# define inline
|
|
#else
|
|
# define JEMALLOC_ENABLE_INLINE
|
|
# define JEMALLOC_INLINE static inline
|
|
#endif
|
|
|
|
/* Smallest size class to support. */
|
|
#define LG_TINY_MIN 3
|
|
#define TINY_MIN (1U << LG_TINY_MIN)
|
|
|
|
/*
|
|
* Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size
|
|
* classes).
|
|
*/
|
|
#ifndef LG_QUANTUM
|
|
# ifdef __i386__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __ia64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __alpha__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __sparc64__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# if (defined(__amd64__) || defined(__x86_64__))
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __arm__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __mips__
|
|
# define LG_QUANTUM 3
|
|
# endif
|
|
# ifdef __powerpc__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __s390x__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __SH4__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifdef __tile__
|
|
# define LG_QUANTUM 4
|
|
# endif
|
|
# ifndef LG_QUANTUM
|
|
# error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS"
|
|
# endif
|
|
#endif
|
|
|
|
#define QUANTUM ((size_t)(1U << LG_QUANTUM))
|
|
#define QUANTUM_MASK (QUANTUM - 1)
|
|
|
|
/* Return the smallest quantum multiple that is >= a. */
|
|
#define QUANTUM_CEILING(a) \
|
|
(((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
|
|
|
|
#define LONG ((size_t)(1U << LG_SIZEOF_LONG))
|
|
#define LONG_MASK (LONG - 1)
|
|
|
|
/* Return the smallest long multiple that is >= a. */
|
|
#define LONG_CEILING(a) \
|
|
(((a) + LONG_MASK) & ~LONG_MASK)
|
|
|
|
#define SIZEOF_PTR (1U << LG_SIZEOF_PTR)
|
|
#define PTR_MASK (SIZEOF_PTR - 1)
|
|
|
|
/* Return the smallest (void *) multiple that is >= a. */
|
|
#define PTR_CEILING(a) \
|
|
(((a) + PTR_MASK) & ~PTR_MASK)
|
|
|
|
/*
|
|
* Maximum size of L1 cache line. This is used to avoid cache line aliasing.
|
|
* In addition, this controls the spacing of cacheline-spaced size classes.
|
|
*/
|
|
#define LG_CACHELINE 6
|
|
#define CACHELINE ((size_t)(1U << LG_CACHELINE))
|
|
#define CACHELINE_MASK (CACHELINE - 1)
|
|
|
|
/* Return the smallest cacheline multiple that is >= s. */
|
|
#define CACHELINE_CEILING(s) \
|
|
(((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
|
|
|
|
/* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */
|
|
#define STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT))
|
|
#define STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1))
|
|
#ifdef PAGE_SHIFT
|
|
# undef PAGE_SHIFT
|
|
#endif
|
|
#ifdef PAGE_SIZE
|
|
# undef PAGE_SIZE
|
|
#endif
|
|
#ifdef PAGE_MASK
|
|
# undef PAGE_MASK
|
|
#endif
|
|
#define PAGE_SHIFT STATIC_PAGE_SHIFT
|
|
#define PAGE_SIZE STATIC_PAGE_SIZE
|
|
#define PAGE_MASK STATIC_PAGE_MASK
|
|
|
|
/* Return the smallest pagesize multiple that is >= s. */
|
|
#define PAGE_CEILING(s) \
|
|
(((s) + PAGE_MASK) & ~PAGE_MASK)
|
|
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#ifdef JEMALLOC_ZONE
|
|
#include "jemalloc/internal/zone.h"
|
|
#endif
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_TYPES
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_STRUCTS
|
|
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#ifdef JEMALLOC_ZONE
|
|
#include "jemalloc/internal/zone.h"
|
|
#endif
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
typedef struct {
|
|
uint64_t allocated;
|
|
uint64_t deallocated;
|
|
} thread_allocated_t;
|
|
/*
|
|
* The JEMALLOC_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro
|
|
* argument.
|
|
*/
|
|
#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_CONCAT({0, 0})
|
|
|
|
#undef JEMALLOC_H_STRUCTS
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_EXTERNS
|
|
|
|
extern bool opt_abort;
|
|
extern bool opt_junk;
|
|
extern bool opt_xmalloc;
|
|
extern bool opt_zero;
|
|
extern size_t opt_narenas;
|
|
|
|
#ifdef DYNAMIC_PAGE_SHIFT
|
|
extern size_t pagesize;
|
|
extern size_t pagesize_mask;
|
|
extern size_t lg_pagesize;
|
|
#endif
|
|
|
|
/* Number of CPUs. */
|
|
extern unsigned ncpus;
|
|
|
|
extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */
|
|
/*
|
|
* Arenas that are used to service external requests. Not all elements of the
|
|
* arenas array are necessarily used; arenas are created lazily as needed.
|
|
*/
|
|
extern arena_t **arenas;
|
|
extern unsigned narenas;
|
|
|
|
extern bool malloc_initialized;
|
|
|
|
arena_t *arenas_extend(unsigned ind);
|
|
void arenas_cleanup(void *arg);
|
|
arena_t *choose_arena_hard(void);
|
|
void jemalloc_prefork(void);
|
|
void jemalloc_postfork_parent(void);
|
|
void jemalloc_postfork_child(void);
|
|
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#ifdef JEMALLOC_ZONE
|
|
#include "jemalloc/internal/zone.h"
|
|
#endif
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_EXTERNS
|
|
/******************************************************************************/
|
|
#define JEMALLOC_H_INLINES
|
|
|
|
#include "jemalloc/internal/util.h"
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/prng.h"
|
|
#include "jemalloc/internal/ckh.h"
|
|
#include "jemalloc/internal/size_classes.h"
|
|
#include "jemalloc/internal/stats.h"
|
|
#include "jemalloc/internal/ctl.h"
|
|
#include "jemalloc/internal/mutex.h"
|
|
#include "jemalloc/internal/tsd.h"
|
|
#include "jemalloc/internal/mb.h"
|
|
#include "jemalloc/internal/extent.h"
|
|
#include "jemalloc/internal/base.h"
|
|
#include "jemalloc/internal/chunk.h"
|
|
#include "jemalloc/internal/huge.h"
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
|
|
|
|
size_t s2u(size_t size);
|
|
size_t sa2u(size_t size, size_t alignment, size_t *run_size_p);
|
|
arena_t *choose_arena(void);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
|
/*
|
|
* Map of pthread_self() --> arenas[???], used for selecting an arena to use
|
|
* for allocations.
|
|
*/
|
|
malloc_tsd_externs(arenas, arena_t *)
|
|
malloc_tsd_funcs(JEMALLOC_INLINE, arenas, arena_t *, NULL, arenas_cleanup)
|
|
|
|
/*
|
|
* Compute usable size that would result from allocating an object with the
|
|
* specified size.
|
|
*/
|
|
JEMALLOC_INLINE size_t
|
|
s2u(size_t size)
|
|
{
|
|
|
|
if (size <= SMALL_MAXCLASS)
|
|
return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size);
|
|
if (size <= arena_maxclass)
|
|
return (PAGE_CEILING(size));
|
|
return (CHUNK_CEILING(size));
|
|
}
|
|
|
|
/*
|
|
* Compute usable size that would result from allocating an object with the
|
|
* specified size and alignment.
|
|
*/
|
|
JEMALLOC_INLINE size_t
|
|
sa2u(size_t size, size_t alignment, size_t *run_size_p)
|
|
{
|
|
size_t usize;
|
|
|
|
/*
|
|
* Round size up to the nearest multiple of alignment.
|
|
*
|
|
* This done, we can take advantage of the fact that for each small
|
|
* size class, every object is aligned at the smallest power of two
|
|
* that is non-zero in the base two representation of the size. For
|
|
* example:
|
|
*
|
|
* Size | Base 2 | Minimum alignment
|
|
* -----+----------+------------------
|
|
* 96 | 1100000 | 32
|
|
* 144 | 10100000 | 32
|
|
* 192 | 11000000 | 64
|
|
*/
|
|
usize = (size + (alignment - 1)) & (-alignment);
|
|
/*
|
|
* (usize < size) protects against the combination of maximal
|
|
* alignment and size greater than maximal alignment.
|
|
*/
|
|
if (usize < size) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
|
|
if (usize <= arena_maxclass && alignment <= PAGE_SIZE) {
|
|
if (usize <= SMALL_MAXCLASS)
|
|
return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size);
|
|
return (PAGE_CEILING(usize));
|
|
} else {
|
|
size_t run_size;
|
|
|
|
/*
|
|
* We can't achieve subpage alignment, so round up alignment
|
|
* permanently; it makes later calculations simpler.
|
|
*/
|
|
alignment = PAGE_CEILING(alignment);
|
|
usize = PAGE_CEILING(size);
|
|
/*
|
|
* (usize < size) protects against very large sizes within
|
|
* PAGE_SIZE of SIZE_T_MAX.
|
|
*
|
|
* (usize + alignment < usize) protects against the
|
|
* combination of maximal alignment and usize large enough
|
|
* to cause overflow. This is similar to the first overflow
|
|
* check above, but it needs to be repeated due to the new
|
|
* usize value, which may now be *equal* to maximal
|
|
* alignment, whereas before we only detected overflow if the
|
|
* original size was *greater* than maximal alignment.
|
|
*/
|
|
if (usize < size || usize + alignment < usize) {
|
|
/* size_t overflow. */
|
|
return (0);
|
|
}
|
|
|
|
/*
|
|
* Calculate the size of the over-size run that arena_palloc()
|
|
* would need to allocate in order to guarantee the alignment.
|
|
*/
|
|
if (usize >= alignment)
|
|
run_size = usize + alignment - PAGE_SIZE;
|
|
else {
|
|
/*
|
|
* It is possible that (alignment << 1) will cause
|
|
* overflow, but it doesn't matter because we also
|
|
* subtract PAGE_SIZE, which in the case of overflow
|
|
* leaves us with a very large run_size. That causes
|
|
* the first conditional below to fail, which means
|
|
* that the bogus run_size value never gets used for
|
|
* anything important.
|
|
*/
|
|
run_size = (alignment << 1) - PAGE_SIZE;
|
|
}
|
|
if (run_size_p != NULL)
|
|
*run_size_p = run_size;
|
|
|
|
if (run_size <= arena_maxclass)
|
|
return (PAGE_CEILING(usize));
|
|
return (CHUNK_CEILING(usize));
|
|
}
|
|
}
|
|
|
|
/* Choose an arena based on a per-thread value. */
|
|
JEMALLOC_INLINE arena_t *
|
|
choose_arena(void)
|
|
{
|
|
arena_t *ret;
|
|
|
|
if ((ret = *arenas_tsd_get()) == NULL) {
|
|
ret = choose_arena_hard();
|
|
assert(ret != NULL);
|
|
}
|
|
|
|
return (ret);
|
|
}
|
|
#endif
|
|
|
|
#include "jemalloc/internal/bitmap.h"
|
|
#include "jemalloc/internal/rtree.h"
|
|
#include "jemalloc/internal/tcache.h"
|
|
#include "jemalloc/internal/arena.h"
|
|
#include "jemalloc/internal/hash.h"
|
|
#ifdef JEMALLOC_ZONE
|
|
#include "jemalloc/internal/zone.h"
|
|
#endif
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
void *imalloc(size_t size);
|
|
void *icalloc(size_t size);
|
|
void *ipalloc(size_t usize, size_t alignment, bool zero);
|
|
size_t isalloc(const void *ptr);
|
|
size_t ivsalloc(const void *ptr);
|
|
void idalloc(void *ptr);
|
|
void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
|
|
bool zero, bool no_move);
|
|
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
|
JEMALLOC_INLINE void *
|
|
imalloc(size_t size)
|
|
{
|
|
|
|
assert(size != 0);
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_malloc(size, false));
|
|
else
|
|
return (huge_malloc(size, false));
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
icalloc(size_t size)
|
|
{
|
|
|
|
if (size <= arena_maxclass)
|
|
return (arena_malloc(size, true));
|
|
else
|
|
return (huge_malloc(size, true));
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
ipalloc(size_t usize, size_t alignment, bool zero)
|
|
{
|
|
void *ret;
|
|
|
|
assert(usize != 0);
|
|
assert(usize == sa2u(usize, alignment, NULL));
|
|
|
|
if (usize <= arena_maxclass && alignment <= PAGE_SIZE)
|
|
ret = arena_malloc(usize, zero);
|
|
else {
|
|
size_t run_size JEMALLOC_CC_SILENCE_INIT(0);
|
|
|
|
/*
|
|
* Ideally we would only ever call sa2u() once per aligned
|
|
* allocation request, and the caller of this function has
|
|
* already done so once. However, it's rather burdensome to
|
|
* require every caller to pass in run_size, especially given
|
|
* that it's only relevant to large allocations. Therefore,
|
|
* just call it again here in order to get run_size.
|
|
*/
|
|
sa2u(usize, alignment, &run_size);
|
|
if (run_size <= arena_maxclass) {
|
|
ret = arena_palloc(choose_arena(), usize, run_size,
|
|
alignment, zero);
|
|
} else if (alignment <= chunksize)
|
|
ret = huge_malloc(usize, zero);
|
|
else
|
|
ret = huge_palloc(usize, alignment, zero);
|
|
}
|
|
|
|
assert(((uintptr_t)ret & (alignment - 1)) == 0);
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
isalloc(const void *ptr)
|
|
{
|
|
size_t ret;
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr) {
|
|
/* Region. */
|
|
if (config_prof)
|
|
ret = arena_salloc_demote(ptr);
|
|
else
|
|
ret = arena_salloc(ptr);
|
|
} else
|
|
ret = huge_salloc(ptr);
|
|
|
|
return (ret);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
ivsalloc(const void *ptr)
|
|
{
|
|
|
|
/* Return 0 if ptr is not within a chunk managed by jemalloc. */
|
|
if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL)
|
|
return (0);
|
|
|
|
return (isalloc(ptr));
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
idalloc(void *ptr)
|
|
{
|
|
arena_chunk_t *chunk;
|
|
|
|
assert(ptr != NULL);
|
|
|
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
if (chunk != ptr)
|
|
arena_dalloc(chunk->arena, chunk, ptr);
|
|
else
|
|
huge_dalloc(ptr, true);
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
|
|
bool no_move)
|
|
{
|
|
void *ret;
|
|
size_t oldsize;
|
|
|
|
assert(ptr != NULL);
|
|
assert(size != 0);
|
|
|
|
oldsize = isalloc(ptr);
|
|
|
|
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
|
|
!= 0) {
|
|
size_t usize, copysize;
|
|
|
|
/*
|
|
* Existing object alignment is inadquate; allocate new space
|
|
* and copy.
|
|
*/
|
|
if (no_move)
|
|
return (NULL);
|
|
usize = sa2u(size + extra, alignment, NULL);
|
|
if (usize == 0)
|
|
return (NULL);
|
|
ret = ipalloc(usize, alignment, zero);
|
|
if (ret == NULL) {
|
|
if (extra == 0)
|
|
return (NULL);
|
|
/* Try again, without extra this time. */
|
|
usize = sa2u(size, alignment, NULL);
|
|
if (usize == 0)
|
|
return (NULL);
|
|
ret = ipalloc(usize, alignment, zero);
|
|
if (ret == NULL)
|
|
return (NULL);
|
|
}
|
|
/*
|
|
* Copy at most size bytes (not size+extra), since the caller
|
|
* has no expectation that the extra bytes will be reliably
|
|
* preserved.
|
|
*/
|
|
copysize = (size < oldsize) ? size : oldsize;
|
|
memcpy(ret, ptr, copysize);
|
|
idalloc(ptr);
|
|
return (ret);
|
|
}
|
|
|
|
if (no_move) {
|
|
if (size <= arena_maxclass) {
|
|
return (arena_ralloc_no_move(ptr, oldsize, size,
|
|
extra, zero));
|
|
} else {
|
|
return (huge_ralloc_no_move(ptr, oldsize, size,
|
|
extra));
|
|
}
|
|
} else {
|
|
if (size + extra <= arena_maxclass) {
|
|
return (arena_ralloc(ptr, oldsize, size, extra,
|
|
alignment, zero));
|
|
} else {
|
|
return (huge_ralloc(ptr, oldsize, size, extra,
|
|
alignment, zero));
|
|
}
|
|
}
|
|
}
|
|
|
|
malloc_tsd_externs(thread_allocated, thread_allocated_t)
|
|
malloc_tsd_funcs(JEMALLOC_INLINE, thread_allocated, thread_allocated_t,
|
|
THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
|
|
#endif
|
|
|
|
#include "jemalloc/internal/prof.h"
|
|
|
|
#undef JEMALLOC_H_INLINES
|
|
/******************************************************************************/
|