#include #include #include #include #include #include #include #include #ifndef SIZE_T_MAX # define SIZE_T_MAX SIZE_MAX #endif #include #include #include #include #include #include #include #include #ifndef offsetof # define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) #endif #include #include #include #include #include #include #include #include #define JEMALLOC_NO_DEMANGLE #include "../jemalloc@install_suffix@.h" #include "jemalloc/internal/private_namespace.h" #ifdef JEMALLOC_CC_SILENCE #define UNUSED JEMALLOC_ATTR(unused) #else #define UNUSED #endif static const bool config_debug = #ifdef JEMALLOC_DEBUG true #else false #endif ; static const bool config_dss = #ifdef JEMALLOC_DSS true #else false #endif ; static const bool config_dynamic_page_shift = #ifdef JEMALLOC_DYNAMIC_PAGE_SHIFT true #else false #endif ; static const bool config_fill = #ifdef JEMALLOC_FILL true #else false #endif ; static const bool config_lazy_lock = #ifdef JEMALLOC_LAZY_LOCK true #else false #endif ; static const bool config_prof = #ifdef JEMALLOC_PROF true #else false #endif ; static const bool config_prof_libgcc = #ifdef JEMALLOC_PROF_LIBGCC true #else false #endif ; static const bool config_prof_libunwind = #ifdef JEMALLOC_PROF_LIBUNWIND true #else false #endif ; static const bool config_stats = #ifdef JEMALLOC_STATS true #else false #endif ; static const bool config_tcache = #ifdef JEMALLOC_TCACHE true #else false #endif ; static const bool config_tls = #ifdef JEMALLOC_TLS true #else false #endif ; static const bool config_xmalloc = #ifdef JEMALLOC_XMALLOC true #else false #endif ; static const bool config_ivsalloc = #ifdef JEMALLOC_IVSALLOC true #else false #endif ; #if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) #include #endif #ifdef JEMALLOC_ZONE #include #include #include #include #endif #define RB_COMPACT #include "jemalloc/internal/rb.h" #include "jemalloc/internal/qr.h" #include "jemalloc/internal/ql.h" /* * jemalloc can conceptually be broken into components (arena, tcache, etc.), * but there are circular dependencies that cannot be broken without * substantial performance degradation. In order to reduce the effect on * visual code flow, read the header files in multiple passes, with one of the * following cpp variables defined during each pass: * * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data * types. * JEMALLOC_H_STRUCTS : Data structures. * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes. * JEMALLOC_H_INLINES : Inline functions. */ /******************************************************************************/ #define JEMALLOC_H_TYPES #define ALLOCM_LG_ALIGN_MASK ((int)0x3f) #define ZU(z) ((size_t)z) #ifndef __DECONST # define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) #endif #ifdef JEMALLOC_DEBUG /* Disable inlining to make debugging easier. */ # define JEMALLOC_INLINE # define inline #else # define JEMALLOC_ENABLE_INLINE # define JEMALLOC_INLINE static inline #endif /* Smallest size class to support. */ #define LG_TINY_MIN 3 #define TINY_MIN (1U << LG_TINY_MIN) /* * Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size * classes). */ #ifndef LG_QUANTUM # ifdef __i386__ # define LG_QUANTUM 4 # endif # ifdef __ia64__ # define LG_QUANTUM 4 # endif # ifdef __alpha__ # define LG_QUANTUM 4 # endif # ifdef __sparc64__ # define LG_QUANTUM 4 # endif # if (defined(__amd64__) || defined(__x86_64__)) # define LG_QUANTUM 4 # endif # ifdef __arm__ # define LG_QUANTUM 3 # endif # ifdef __mips__ # define LG_QUANTUM 3 # endif # ifdef __powerpc__ # define LG_QUANTUM 4 # endif # ifdef __s390x__ # define LG_QUANTUM 4 # endif # ifdef __SH4__ # define LG_QUANTUM 4 # endif # ifdef __tile__ # define LG_QUANTUM 4 # endif # ifndef LG_QUANTUM # error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS" # endif #endif #define QUANTUM ((size_t)(1U << LG_QUANTUM)) #define QUANTUM_MASK (QUANTUM - 1) /* Return the smallest quantum multiple that is >= a. */ #define QUANTUM_CEILING(a) \ (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) #define LONG ((size_t)(1U << LG_SIZEOF_LONG)) #define LONG_MASK (LONG - 1) /* Return the smallest long multiple that is >= a. */ #define LONG_CEILING(a) \ (((a) + LONG_MASK) & ~LONG_MASK) #define SIZEOF_PTR (1U << LG_SIZEOF_PTR) #define PTR_MASK (SIZEOF_PTR - 1) /* Return the smallest (void *) multiple that is >= a. */ #define PTR_CEILING(a) \ (((a) + PTR_MASK) & ~PTR_MASK) /* * Maximum size of L1 cache line. This is used to avoid cache line aliasing. * In addition, this controls the spacing of cacheline-spaced size classes. */ #define LG_CACHELINE 6 #define CACHELINE ((size_t)(1U << LG_CACHELINE)) #define CACHELINE_MASK (CACHELINE - 1) /* Return the smallest cacheline multiple that is >= s. */ #define CACHELINE_CEILING(s) \ (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) /* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */ #define STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT)) #define STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1)) #ifdef PAGE_SHIFT # undef PAGE_SHIFT #endif #ifdef PAGE_SIZE # undef PAGE_SIZE #endif #ifdef PAGE_MASK # undef PAGE_MASK #endif #define PAGE_SHIFT STATIC_PAGE_SHIFT #define PAGE_SIZE STATIC_PAGE_SIZE #define PAGE_MASK STATIC_PAGE_MASK /* Return the smallest pagesize multiple that is >= s. */ #define PAGE_CEILING(s) \ (((s) + PAGE_MASK) & ~PAGE_MASK) #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" #include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/arena.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" #ifdef JEMALLOC_ZONE #include "jemalloc/internal/zone.h" #endif #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_TYPES /******************************************************************************/ #define JEMALLOC_H_STRUCTS #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" #include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/arena.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" #ifdef JEMALLOC_ZONE #include "jemalloc/internal/zone.h" #endif #include "jemalloc/internal/prof.h" typedef struct { uint64_t allocated; uint64_t deallocated; } thread_allocated_t; #undef JEMALLOC_H_STRUCTS /******************************************************************************/ #define JEMALLOC_H_EXTERNS extern bool opt_abort; extern bool opt_junk; extern bool opt_xmalloc; extern bool opt_zero; extern size_t opt_narenas; #ifdef DYNAMIC_PAGE_SHIFT extern size_t pagesize; extern size_t pagesize_mask; extern size_t lg_pagesize; #endif /* Number of CPUs. */ extern unsigned ncpus; extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ extern pthread_key_t arenas_tsd; #ifndef NO_TLS /* * Map of pthread_self() --> arenas[???], used for selecting an arena to use * for allocations. */ extern __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); # define ARENA_GET() arenas_tls # define ARENA_SET(v) do { \ arenas_tls = (v); \ pthread_setspecific(arenas_tsd, (void *)(v)); \ } while (0) #else # define ARENA_GET() ((arena_t *)pthread_getspecific(arenas_tsd)) # define ARENA_SET(v) do { \ pthread_setspecific(arenas_tsd, (void *)(v)); \ } while (0) #endif /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. */ extern arena_t **arenas; extern unsigned narenas; #ifndef NO_TLS extern __thread thread_allocated_t thread_allocated_tls; # define ALLOCATED_GET() (thread_allocated_tls.allocated) # define ALLOCATEDP_GET() (&thread_allocated_tls.allocated) # define DEALLOCATED_GET() (thread_allocated_tls.deallocated) # define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated) # define ALLOCATED_ADD(a, d) do { \ thread_allocated_tls.allocated += a; \ thread_allocated_tls.deallocated += d; \ } while (0) #else # define ALLOCATED_GET() (thread_allocated_get()->allocated) # define ALLOCATEDP_GET() (&thread_allocated_get()->allocated) # define DEALLOCATED_GET() (thread_allocated_get()->deallocated) # define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated) # define ALLOCATED_ADD(a, d) do { \ thread_allocated_t *thread_allocated = thread_allocated_get(); \ thread_allocated->allocated += (a); \ thread_allocated->deallocated += (d); \ } while (0) #endif extern pthread_key_t thread_allocated_tsd; thread_allocated_t *thread_allocated_get_hard(void); arena_t *arenas_extend(unsigned ind); arena_t *choose_arena_hard(void); void jemalloc_prefork(void); void jemalloc_postfork(void); #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" #include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/arena.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" #ifdef JEMALLOC_ZONE #include "jemalloc/internal/zone.h" #endif #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_EXTERNS /******************************************************************************/ #define JEMALLOC_H_INLINES #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" #include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" #ifndef JEMALLOC_ENABLE_INLINE size_t s2u(size_t size); size_t sa2u(size_t size, size_t alignment, size_t *run_size_p); arena_t *choose_arena(void); thread_allocated_t *thread_allocated_get(void); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) /* * Compute usable size that would result from allocating an object with the * specified size. */ JEMALLOC_INLINE size_t s2u(size_t size) { if (size <= SMALL_MAXCLASS) return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size); if (size <= arena_maxclass) return (PAGE_CEILING(size)); return (CHUNK_CEILING(size)); } /* * Compute usable size that would result from allocating an object with the * specified size and alignment. */ JEMALLOC_INLINE size_t sa2u(size_t size, size_t alignment, size_t *run_size_p) { size_t usize; /* * Round size up to the nearest multiple of alignment. * * This done, we can take advantage of the fact that for each small * size class, every object is aligned at the smallest power of two * that is non-zero in the base two representation of the size. For * example: * * Size | Base 2 | Minimum alignment * -----+----------+------------------ * 96 | 1100000 | 32 * 144 | 10100000 | 32 * 192 | 11000000 | 64 */ usize = (size + (alignment - 1)) & (-alignment); /* * (usize < size) protects against the combination of maximal * alignment and size greater than maximal alignment. */ if (usize < size) { /* size_t overflow. */ return (0); } if (usize <= arena_maxclass && alignment <= PAGE_SIZE) { if (usize <= SMALL_MAXCLASS) return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size); return (PAGE_CEILING(usize)); } else { size_t run_size; /* * We can't achieve subpage alignment, so round up alignment * permanently; it makes later calculations simpler. */ alignment = PAGE_CEILING(alignment); usize = PAGE_CEILING(size); /* * (usize < size) protects against very large sizes within * PAGE_SIZE of SIZE_T_MAX. * * (usize + alignment < usize) protects against the * combination of maximal alignment and usize large enough * to cause overflow. This is similar to the first overflow * check above, but it needs to be repeated due to the new * usize value, which may now be *equal* to maximal * alignment, whereas before we only detected overflow if the * original size was *greater* than maximal alignment. */ if (usize < size || usize + alignment < usize) { /* size_t overflow. */ return (0); } /* * Calculate the size of the over-size run that arena_palloc() * would need to allocate in order to guarantee the alignment. */ if (usize >= alignment) run_size = usize + alignment - PAGE_SIZE; else { /* * It is possible that (alignment << 1) will cause * overflow, but it doesn't matter because we also * subtract PAGE_SIZE, which in the case of overflow * leaves us with a very large run_size. That causes * the first conditional below to fail, which means * that the bogus run_size value never gets used for * anything important. */ run_size = (alignment << 1) - PAGE_SIZE; } if (run_size_p != NULL) *run_size_p = run_size; if (run_size <= arena_maxclass) return (PAGE_CEILING(usize)); return (CHUNK_CEILING(usize)); } } /* * Choose an arena based on a per-thread value (fast-path code, calls slow-path * code if necessary). */ JEMALLOC_INLINE arena_t * choose_arena(void) { arena_t *ret; ret = ARENA_GET(); if (ret == NULL) { ret = choose_arena_hard(); assert(ret != NULL); } return (ret); } JEMALLOC_INLINE thread_allocated_t * thread_allocated_get(void) { thread_allocated_t *thread_allocated = (thread_allocated_t *) pthread_getspecific(thread_allocated_tsd); if (thread_allocated == NULL) return (thread_allocated_get_hard()); return (thread_allocated); } #endif #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/arena.h" #include "jemalloc/internal/hash.h" #ifdef JEMALLOC_ZONE #include "jemalloc/internal/zone.h" #endif #ifndef JEMALLOC_ENABLE_INLINE void *imalloc(size_t size); void *icalloc(size_t size); void *ipalloc(size_t usize, size_t alignment, bool zero); size_t isalloc(const void *ptr); size_t ivsalloc(const void *ptr); void idalloc(void *ptr); void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, bool no_move); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) JEMALLOC_INLINE void * imalloc(size_t size) { assert(size != 0); if (size <= arena_maxclass) return (arena_malloc(size, false)); else return (huge_malloc(size, false)); } JEMALLOC_INLINE void * icalloc(size_t size) { if (size <= arena_maxclass) return (arena_malloc(size, true)); else return (huge_malloc(size, true)); } JEMALLOC_INLINE void * ipalloc(size_t usize, size_t alignment, bool zero) { void *ret; assert(usize != 0); assert(usize == sa2u(usize, alignment, NULL)); if (usize <= arena_maxclass && alignment <= PAGE_SIZE) ret = arena_malloc(usize, zero); else { size_t run_size #ifdef JEMALLOC_CC_SILENCE = 0 #endif ; /* * Ideally we would only ever call sa2u() once per aligned * allocation request, and the caller of this function has * already done so once. However, it's rather burdensome to * require every caller to pass in run_size, especially given * that it's only relevant to large allocations. Therefore, * just call it again here in order to get run_size. */ sa2u(usize, alignment, &run_size); if (run_size <= arena_maxclass) { ret = arena_palloc(choose_arena(), usize, run_size, alignment, zero); } else if (alignment <= chunksize) ret = huge_malloc(usize, zero); else ret = huge_palloc(usize, alignment, zero); } assert(((uintptr_t)ret & (alignment - 1)) == 0); return (ret); } JEMALLOC_INLINE size_t isalloc(const void *ptr) { size_t ret; arena_chunk_t *chunk; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ if (config_prof) ret = arena_salloc_demote(ptr); else ret = arena_salloc(ptr); } else ret = huge_salloc(ptr); return (ret); } JEMALLOC_INLINE size_t ivsalloc(const void *ptr) { /* Return 0 if ptr is not within a chunk managed by jemalloc. */ if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) return (0); return (isalloc(ptr)); } JEMALLOC_INLINE void idalloc(void *ptr) { arena_chunk_t *chunk; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) arena_dalloc(chunk->arena, chunk, ptr); else huge_dalloc(ptr, true); } JEMALLOC_INLINE void * iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, bool no_move) { void *ret; size_t oldsize; assert(ptr != NULL); assert(size != 0); oldsize = isalloc(ptr); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { size_t usize, copysize; /* * Existing object alignment is inadquate; allocate new space * and copy. */ if (no_move) return (NULL); usize = sa2u(size + extra, alignment, NULL); if (usize == 0) return (NULL); ret = ipalloc(usize, alignment, zero); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, without extra this time. */ usize = sa2u(size, alignment, NULL); if (usize == 0) return (NULL); ret = ipalloc(usize, alignment, zero); if (ret == NULL) return (NULL); } /* * Copy at most size bytes (not size+extra), since the caller * has no expectation that the extra bytes will be reliably * preserved. */ copysize = (size < oldsize) ? size : oldsize; memcpy(ret, ptr, copysize); idalloc(ptr); return (ret); } if (no_move) { if (size <= arena_maxclass) { return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero)); } else { return (huge_ralloc_no_move(ptr, oldsize, size, extra)); } } else { if (size + extra <= arena_maxclass) { return (arena_ralloc(ptr, oldsize, size, extra, alignment, zero)); } else { return (huge_ralloc(ptr, oldsize, size, extra, alignment, zero)); } } } #endif #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_INLINES /******************************************************************************/