project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
6ffbbeb5d6
server-skynet-source-3rd-je.../src/jemalloc.c
Jason Evans 6ffbbeb5d6 Silence compiler warnings.
2012-02-13 12:31:30 -08:00

1673 lines
37 KiB
C
Raw Blame History

#define JEMALLOC_C_
#include "jemalloc/internal/jemalloc_internal.h"
/******************************************************************************/
/* Data. */
malloc_mutex_t arenas_lock;
arena_t **arenas;
unsigned narenas;
pthread_key_t arenas_tsd;
#ifndef NO_TLS
__thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec"));
#endif
#ifndef NO_TLS
__thread thread_allocated_t thread_allocated_tls;
#endif
pthread_key_t thread_allocated_tsd;
/* Set to true once the allocator has been initialized. */
static bool malloc_initialized = false;
/* Used to let the initializing thread recursively allocate. */
static pthread_t malloc_initializer = (unsigned long)0;
/* Used to avoid initialization races. */
static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
#ifdef DYNAMIC_PAGE_SHIFT
size_t pagesize;
size_t pagesize_mask;
size_t lg_pagesize;
#endif
unsigned ncpus;
/* Runtime configuration options. */
const char *JEMALLOC_P(malloc_conf) JEMALLOC_ATTR(visibility("default"));
#ifdef JEMALLOC_DEBUG
bool opt_abort = true;
# ifdef JEMALLOC_FILL
bool opt_junk = true;
# else
bool opt_junk = false;
# endif
#else
bool opt_abort = false;
bool opt_junk = false;
#endif
bool opt_sysv = false;
bool opt_xmalloc = false;
bool opt_zero = false;
size_t opt_narenas = 0;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static void wrtmessage(void *cbopaque, const char *s);
static void stats_print_atexit(void);
static unsigned malloc_ncpus(void);
static void arenas_cleanup(void *arg);
#ifdef NO_TLS
static void thread_allocated_cleanup(void *arg);
#endif
static bool malloc_conf_next(char const **opts_p, char const **k_p,
size_t *klen_p, char const **v_p, size_t *vlen_p);
static void malloc_conf_error(const char *msg, const char *k, size_t klen,
const char *v, size_t vlen);
static void malloc_conf_init(void);
static bool malloc_init_hard(void);
static int imemalign(void **memptr, size_t alignment, size_t size);
/******************************************************************************/
/* malloc_message() setup. */
JEMALLOC_CATTR(visibility("hidden"), static)
void
wrtmessage(void *cbopaque, const char *s)
{
UNUSED int result = write(STDERR_FILENO, s, strlen(s));
}
void (*JEMALLOC_P(malloc_message))(void *, const char *s)
JEMALLOC_ATTR(visibility("default")) = wrtmessage;
/******************************************************************************/
/*
* Begin miscellaneous support functions.
*/
/* Create a new arena and insert it into the arenas array at index ind. */
arena_t *
arenas_extend(unsigned ind)
{
arena_t *ret;
/* Allocate enough space for trailing bins. */
ret = (arena_t *)base_alloc(offsetof(arena_t, bins)
+ (sizeof(arena_bin_t) * nbins));
if (ret != NULL && arena_new(ret, ind) == false) {
arenas[ind] = ret;
return (ret);
}
/* Only reached if there is an OOM error. */
/*
* OOM here is quite inconvenient to propagate, since dealing with it
* would require a check for failure in the fast path. Instead, punt
* by using arenas[0]. In practice, this is an extremely unlikely
* failure.
*/
malloc_write("<jemalloc>: Error initializing arena\n");
if (opt_abort)
abort();
return (arenas[0]);
}
/*
* Choose an arena based on a per-thread value (slow-path code only, called
* only by choose_arena()).
*/
arena_t *
choose_arena_hard(void)
{
arena_t *ret;
if (narenas > 1) {
unsigned i, choose, first_null;
choose = 0;
first_null = narenas;
malloc_mutex_lock(&arenas_lock);
assert(arenas[0] != NULL);
for (i = 1; i < narenas; i++) {
if (arenas[i] != NULL) {
/*
* Choose the first arena that has the lowest
* number of threads assigned to it.
*/
if (arenas[i]->nthreads <
arenas[choose]->nthreads)
choose = i;
} else if (first_null == narenas) {
/*
* Record the index of the first uninitialized
* arena, in case all extant arenas are in use.
*
* NB: It is possible for there to be
* discontinuities in terms of initialized
* versus uninitialized arenas, due to the
* "thread.arena" mallctl.
*/
first_null = i;
}
}
if (arenas[choose] == 0 || first_null == narenas) {
/*
* Use an unloaded arena, or the least loaded arena if
* all arenas are already initialized.
*/
ret = arenas[choose];
} else {
/* Initialize a new arena. */
ret = arenas_extend(first_null);
}
ret->nthreads++;
malloc_mutex_unlock(&arenas_lock);
} else {
ret = arenas[0];
malloc_mutex_lock(&arenas_lock);
ret->nthreads++;
malloc_mutex_unlock(&arenas_lock);
}
ARENA_SET(ret);
return (ret);
}
/*
* glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
* provide a wrapper.
*/
int
buferror(int errnum, char *buf, size_t buflen)
{
#ifdef _GNU_SOURCE
char *b = strerror_r(errno, buf, buflen);
if (b != buf) {
strncpy(buf, b, buflen);
buf[buflen-1] = '\0';
}
return (0);
#else
return (strerror_r(errno, buf, buflen));
#endif
}
static void
stats_print_atexit(void)
{
if (config_tcache && config_stats) {
unsigned i;
/*
* Merge stats from extant threads. This is racy, since
* individual threads do not lock when recording tcache stats
* events. As a consequence, the final stats may be slightly
* out of date by the time they are reported, if other threads
* continue to allocate.
*/
for (i = 0; i < narenas; i++) {
arena_t *arena = arenas[i];
if (arena != NULL) {
tcache_t *tcache;
/*
* tcache_stats_merge() locks bins, so if any
* code is introduced that acquires both arena
* and bin locks in the opposite order,
* deadlocks may result.
*/
malloc_mutex_lock(&arena->lock);
ql_foreach(tcache, &arena->tcache_ql, link) {
tcache_stats_merge(tcache, arena);
}
malloc_mutex_unlock(&arena->lock);
}
}
}
JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL);
}
thread_allocated_t *
thread_allocated_get_hard(void)
{
thread_allocated_t *thread_allocated = (thread_allocated_t *)
imalloc(sizeof(thread_allocated_t));
if (thread_allocated == NULL) {
static thread_allocated_t static_thread_allocated = {0, 0};
malloc_write("<jemalloc>: Error allocating TSD;"
" mallctl(\"thread.{de,}allocated[p]\", ...)"
" will be inaccurate\n");
if (opt_abort)
abort();
return (&static_thread_allocated);
}
pthread_setspecific(thread_allocated_tsd, thread_allocated);
thread_allocated->allocated = 0;
thread_allocated->deallocated = 0;
return (thread_allocated);
}
/*
* End miscellaneous support functions.
*/
/******************************************************************************/
/*
* Begin initialization functions.
*/
static unsigned
malloc_ncpus(void)
{
unsigned ret;
long result;
result = sysconf(_SC_NPROCESSORS_ONLN);
if (result == -1) {
/* Error. */
ret = 1;
}
ret = (unsigned)result;
return (ret);
}
static void
arenas_cleanup(void *arg)
{
arena_t *arena = (arena_t *)arg;
malloc_mutex_lock(&arenas_lock);
arena->nthreads--;
malloc_mutex_unlock(&arenas_lock);
}
#ifdef NO_TLS
static void
thread_allocated_cleanup(void *arg)
{
uint64_t *allocated = (uint64_t *)arg;
if (allocated != NULL)
idalloc(allocated);
}
#endif
/*
* FreeBSD's pthreads implementation calls malloc(3), so the malloc
* implementation has to take pains to avoid infinite recursion during
* initialization.
*/
static inline bool
malloc_init(void)
{
if (malloc_initialized == false)
return (malloc_init_hard());
return (false);
}
static bool
malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
char const **v_p, size_t *vlen_p)
{
bool accept;
const char *opts = *opts_p;
*k_p = opts;
for (accept = false; accept == false;) {
switch (*opts) {
case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F': case 'G': case 'H': case 'I': case 'J':
case 'K': case 'L': case 'M': case 'N': case 'O':
case 'P': case 'Q': case 'R': case 'S': case 'T':
case 'U': case 'V': case 'W': case 'X': case 'Y':
case 'Z':
case 'a': case 'b': case 'c': case 'd': case 'e':
case 'f': case 'g': case 'h': case 'i': case 'j':
case 'k': case 'l': case 'm': case 'n': case 'o':
case 'p': case 'q': case 'r': case 's': case 't':
case 'u': case 'v': case 'w': case 'x': case 'y':
case 'z':
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
case '_':
opts++;
break;
case ':':
opts++;
*klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
*v_p = opts;
accept = true;
break;
case '\0':
if (opts != *opts_p) {
malloc_write("<jemalloc>: Conf string "
"ends with key\n");
}
return (true);
default:
malloc_write("<jemalloc>: Malformed conf "
"string\n");
return (true);
}
}
for (accept = false; accept == false;) {
switch (*opts) {
case ',':
opts++;
/*
* Look ahead one character here, because the
* next time this function is called, it will
* assume that end of input has been cleanly
* reached if no input remains, but we have
* optimistically already consumed the comma if
* one exists.
*/
if (*opts == '\0') {
malloc_write("<jemalloc>: Conf string "
"ends with comma\n");
}
*vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
accept = true;
break;
case '\0':
*vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
accept = true;
break;
default:
opts++;
break;
}
}
*opts_p = opts;
return (false);
}
static void
malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
size_t vlen)
{
char buf[PATH_MAX + 1];
malloc_write("<jemalloc>: ");
malloc_write(msg);
malloc_write(": ");
memcpy(buf, k, klen);
memcpy(&buf[klen], ":", 1);
memcpy(&buf[klen+1], v, vlen);
buf[klen+1+vlen] = '\0';
malloc_write(buf);
malloc_write("\n");
}
static void
malloc_conf_init(void)
{
unsigned i;
char buf[PATH_MAX + 1];
const char *opts, *k, *v;
size_t klen, vlen;
for (i = 0; i < 3; i++) {
/* Get runtime configuration. */
switch (i) {
case 0:
if (JEMALLOC_P(malloc_conf) != NULL) {
/*
* Use options that were compiled into the
* program.
*/
opts = JEMALLOC_P(malloc_conf);
} else {
/* No configuration specified. */
buf[0] = '\0';
opts = buf;
}
break;
case 1: {
int linklen;
const char *linkname =
#ifdef JEMALLOC_PREFIX
"/etc/"JEMALLOC_PREFIX"malloc.conf"
#else
"/etc/malloc.conf"
#endif
;
if ((linklen = readlink(linkname, buf,
sizeof(buf) - 1)) != -1) {
/*
* Use the contents of the "/etc/malloc.conf"
* symbolic link's name.
*/
buf[linklen] = '\0';
opts = buf;
} else {
/* No configuration specified. */
buf[0] = '\0';
opts = buf;
}
break;
}
case 2: {
const char *envname =
#ifdef JEMALLOC_PREFIX
JEMALLOC_CPREFIX"MALLOC_CONF"
#else
"MALLOC_CONF"
#endif
;
if ((opts = getenv(envname)) != NULL) {
/*
* Do nothing; opts is already initialized to
* the value of the MALLOC_CONF environment
* variable.
*/
} else {
/* No configuration specified. */
buf[0] = '\0';
opts = buf;
}
break;
}
default:
/* NOTREACHED */
assert(false);
buf[0] = '\0';
opts = buf;
}
while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
&vlen) == false) {
#define CONF_HANDLE_BOOL(n) \
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
klen) == 0) { \
if (strncmp("true", v, vlen) == 0 && \
vlen == sizeof("true")-1) \
opt_##n = true; \
else if (strncmp("false", v, vlen) == \
0 && vlen == sizeof("false")-1) \
opt_##n = false; \
else { \
malloc_conf_error( \
"Invalid conf value", \
k, klen, v, vlen); \
} \
continue; \
}
#define CONF_HANDLE_SIZE_T(n, min, max) \
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
klen) == 0) { \
unsigned long ul; \
char *end; \
\
errno = 0; \
ul = strtoul(v, &end, 0); \
if (errno != 0 || (uintptr_t)end - \
(uintptr_t)v != vlen) { \
malloc_conf_error( \
"Invalid conf value", \
k, klen, v, vlen); \
} else if (ul < min || ul > max) { \
malloc_conf_error( \
"Out-of-range conf value", \
k, klen, v, vlen); \
} else \
opt_##n = ul; \
continue; \
}
#define CONF_HANDLE_SSIZE_T(n, min, max) \
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
klen) == 0) { \
long l; \
char *end; \
\
errno = 0; \
l = strtol(v, &end, 0); \
if (errno != 0 || (uintptr_t)end - \
(uintptr_t)v != vlen) { \
malloc_conf_error( \
"Invalid conf value", \
k, klen, v, vlen); \
} else if (l < (ssize_t)min || l > \
(ssize_t)max) { \
malloc_conf_error( \
"Out-of-range conf value", \
k, klen, v, vlen); \
} else \
opt_##n = l; \
continue; \
}
#define CONF_HANDLE_CHAR_P(n, d) \
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
klen) == 0) { \
size_t cpylen = (vlen <= \
sizeof(opt_##n)-1) ? vlen : \
sizeof(opt_##n)-1; \
strncpy(opt_##n, v, cpylen); \
opt_##n[cpylen] = '\0'; \
continue; \
}
CONF_HANDLE_BOOL(abort)
CONF_HANDLE_SIZE_T(lg_qspace_max, LG_QUANTUM,
PAGE_SHIFT-1)
CONF_HANDLE_SIZE_T(lg_cspace_max, LG_QUANTUM,
PAGE_SHIFT-1)
/*
* Chunks always require at least one * header page,
* plus one data page.
*/
CONF_HANDLE_SIZE_T(lg_chunk, PAGE_SHIFT+1,
(sizeof(size_t) << 3) - 1)
CONF_HANDLE_SIZE_T(narenas, 1, SIZE_T_MAX)
CONF_HANDLE_SSIZE_T(lg_dirty_mult, -1,
(sizeof(size_t) << 3) - 1)
CONF_HANDLE_BOOL(stats_print)
if (config_fill) {
CONF_HANDLE_BOOL(junk)
CONF_HANDLE_BOOL(zero)
}
if (config_sysv) {
CONF_HANDLE_BOOL(sysv)
}
if (config_xmalloc) {
CONF_HANDLE_BOOL(xmalloc)
}
if (config_tcache) {
CONF_HANDLE_BOOL(tcache)
CONF_HANDLE_SSIZE_T(lg_tcache_gc_sweep, -1,
(sizeof(size_t) << 3) - 1)
CONF_HANDLE_SSIZE_T(lg_tcache_max, -1,
(sizeof(size_t) << 3) - 1)
}
if (config_prof) {
CONF_HANDLE_BOOL(prof)
CONF_HANDLE_CHAR_P(prof_prefix, "jeprof")
CONF_HANDLE_SIZE_T(lg_prof_bt_max, 0,
LG_PROF_BT_MAX)
CONF_HANDLE_BOOL(prof_active)
CONF_HANDLE_SSIZE_T(lg_prof_sample, 0,
(sizeof(uint64_t) << 3) - 1)
CONF_HANDLE_BOOL(prof_accum)
CONF_HANDLE_SSIZE_T(lg_prof_tcmax, -1,
(sizeof(size_t) << 3) - 1)
CONF_HANDLE_SSIZE_T(lg_prof_interval, -1,
(sizeof(uint64_t) << 3) - 1)
CONF_HANDLE_BOOL(prof_gdump)
CONF_HANDLE_BOOL(prof_leak)
}
malloc_conf_error("Invalid conf pair", k, klen, v,
vlen);
#undef CONF_HANDLE_BOOL
#undef CONF_HANDLE_SIZE_T
#undef CONF_HANDLE_SSIZE_T
#undef CONF_HANDLE_CHAR_P
}
/* Validate configuration of options that are inter-related. */
if (opt_lg_qspace_max+1 >= opt_lg_cspace_max) {
malloc_write("<jemalloc>: Invalid lg_[qc]space_max "
"relationship; restoring defaults\n");
opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT;
opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT;
}
}
}
static bool
malloc_init_hard(void)
{
arena_t *init_arenas[1];
malloc_mutex_lock(&init_lock);
if (malloc_initialized || malloc_initializer == pthread_self()) {
/*
* Another thread initialized the allocator before this one
* acquired init_lock, or this thread is the initializing
* thread, and it is recursively allocating.
*/
malloc_mutex_unlock(&init_lock);
return (false);
}
if (malloc_initializer != (unsigned long)0) {
/* Busy-wait until the initializing thread completes. */
do {
malloc_mutex_unlock(&init_lock);
CPU_SPINWAIT;
malloc_mutex_lock(&init_lock);
} while (malloc_initialized == false);
malloc_mutex_unlock(&init_lock);
return (false);
}
#ifdef DYNAMIC_PAGE_SHIFT
/* Get page size. */
{
long result;
result = sysconf(_SC_PAGESIZE);
assert(result != -1);
pagesize = (size_t)result;
/*
* We assume that pagesize is a power of 2 when calculating
* pagesize_mask and lg_pagesize.
*/
assert(((result - 1) & result) == 0);
pagesize_mask = result - 1;
lg_pagesize = ffs((int)result) - 1;
}
#endif
if (config_prof)
prof_boot0();
malloc_conf_init();
/* Register fork handlers. */
if (pthread_atfork(jemalloc_prefork, jemalloc_postfork,
jemalloc_postfork) != 0) {
malloc_write("<jemalloc>: Error in pthread_atfork()\n");
if (opt_abort)
abort();
}
if (ctl_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (opt_stats_print) {
/* Print statistics at exit. */
if (atexit(stats_print_atexit) != 0) {
malloc_write("<jemalloc>: Error in atexit()\n");
if (opt_abort)
abort();
}
}
if (chunk_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (base_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_prof)
prof_boot1();
if (arena_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_tcache && tcache_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (huge_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
#ifdef NO_TLS
/* Initialize allocation counters before any allocations can occur. */
if (config_stats && pthread_key_create(&thread_allocated_tsd,
thread_allocated_cleanup) != 0) {
malloc_mutex_unlock(&init_lock);
return (true);
}
#endif
if (malloc_mutex_init(&arenas_lock))
return (true);
if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) {
malloc_mutex_unlock(&init_lock);
return (true);
}
/*
* Create enough scaffolding to allow recursive allocation in
* malloc_ncpus().
*/
narenas = 1;
arenas = init_arenas;
memset(arenas, 0, sizeof(arena_t *) * narenas);
/*
* Initialize one arena here. The rest are lazily created in
* choose_arena_hard().
*/
arenas_extend(0);
if (arenas[0] == NULL) {
malloc_mutex_unlock(&init_lock);
return (true);
}
/*
* Assign the initial arena to the initial thread, in order to avoid
* spurious creation of an extra arena if the application switches to
* threaded mode.
*/
ARENA_SET(arenas[0]);
arenas[0]->nthreads++;
if (config_prof && prof_boot2()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
/* Get number of CPUs. */
malloc_initializer = pthread_self();
malloc_mutex_unlock(&init_lock);
ncpus = malloc_ncpus();
malloc_mutex_lock(&init_lock);
if (opt_narenas == 0) {
/*
* For SMP systems, create more than one arena per CPU by
* default.
*/
if (ncpus > 1)
opt_narenas = ncpus << 2;
else
opt_narenas = 1;
}
narenas = opt_narenas;
/*
* Make sure that the arenas array can be allocated. In practice, this
* limit is enough to allow the allocator to function, but the ctl
* machinery will fail to allocate memory at far lower limits.
*/
if (narenas > chunksize / sizeof(arena_t *)) {
char buf[UMAX2S_BUFSIZE];
narenas = chunksize / sizeof(arena_t *);
malloc_write("<jemalloc>: Reducing narenas to limit (");
malloc_write(u2s(narenas, 10, buf));
malloc_write(")\n");
}
/* Allocate and initialize arenas. */
arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
if (arenas == NULL) {
malloc_mutex_unlock(&init_lock);
return (true);
}
/*
* Zero the array. In practice, this should always be pre-zeroed,
* since it was just mmap()ed, but let's be sure.
*/
memset(arenas, 0, sizeof(arena_t *) * narenas);
/* Copy the pointer to the one arena that was already initialized. */
arenas[0] = init_arenas[0];
#ifdef JEMALLOC_ZONE
/* Register the custom zone. */
malloc_zone_register(create_zone());
/*
* Convert the default szone to an "overlay zone" that is capable of
* deallocating szone-allocated objects, but allocating new objects
* from jemalloc.
*/
szone2ozone(malloc_default_zone());
#endif
malloc_initialized = true;
malloc_mutex_unlock(&init_lock);
return (false);
}
#ifdef JEMALLOC_ZONE
JEMALLOC_ATTR(constructor)
void
jemalloc_darwin_init(void)
{
if (malloc_init_hard())
abort();
}
#endif
/*
* End initialization functions.
*/
/******************************************************************************/
/*
* Begin malloc(3)-compatible functions.
*/
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void *
JEMALLOC_P(malloc)(size_t size)
{
void *ret;
size_t usize;
prof_thr_cnt_t *cnt
#ifdef JEMALLOC_CC_SILENCE
= NULL
#endif
;
if (malloc_init()) {
ret = NULL;
goto OOM;
}
if (size == 0) {
if (config_sysv == false || opt_sysv == false)
size = 1;
else {
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in malloc(): "
"invalid size 0\n");
abort();
}
ret = NULL;
goto RETURN;
}
}
if (config_prof && opt_prof) {
usize = s2u(size);
PROF_ALLOC_PREP(1, usize, cnt);
if (cnt == NULL) {
ret = NULL;
goto OOM;
}
if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <=
small_maxclass) {
ret = imalloc(small_maxclass+1);
if (ret != NULL)
arena_prof_promoted(ret, usize);
} else
ret = imalloc(size);
} else {
if (config_stats)
usize = s2u(size);
ret = imalloc(size);
}
OOM:
if (ret == NULL) {
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in malloc(): "
"out of memory\n");
abort();
}
errno = ENOMEM;
}
RETURN:
if (config_prof && opt_prof && ret != NULL)
prof_malloc(ret, usize, cnt);
if (config_stats && ret != NULL) {
assert(usize == isalloc(ret));
ALLOCATED_ADD(usize, 0);
}
return (ret);
}
JEMALLOC_ATTR(nonnull(1))
#ifdef JEMALLOC_PROF
/*
* Avoid any uncertainty as to how many backtrace frames to ignore in
* PROF_ALLOC_PREP().
*/
JEMALLOC_ATTR(noinline)
#endif
static int
imemalign(void **memptr, size_t alignment, size_t size)
{
int ret;
size_t usize;
void *result;
prof_thr_cnt_t *cnt
#ifdef JEMALLOC_CC_SILENCE
= NULL
#endif
;
if (malloc_init())
result = NULL;
else {
if (size == 0) {
if (config_sysv == false || opt_sysv == false)
size = 1;
else {
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in "
"posix_memalign(): invalid size "
"0\n");
abort();
}
result = NULL;
*memptr = NULL;
ret = 0;
goto RETURN;
}
}
/* Make sure that alignment is a large enough power of 2. */
if (((alignment - 1) & alignment) != 0
|| alignment < sizeof(void *)) {
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in "
"posix_memalign(): invalid alignment\n");
abort();
}
result = NULL;
ret = EINVAL;
goto RETURN;
}
usize = sa2u(size, alignment, NULL);
if (usize == 0) {
result = NULL;
ret = ENOMEM;
goto RETURN;
}
if (config_prof && opt_prof) {
PROF_ALLOC_PREP(2, usize, cnt);
if (cnt == NULL) {
result = NULL;
ret = EINVAL;
} else {
if (prof_promote && (uintptr_t)cnt !=
(uintptr_t)1U && usize <= small_maxclass) {
assert(sa2u(small_maxclass+1,
alignment, NULL) != 0);
result = ipalloc(sa2u(small_maxclass+1,
alignment, NULL), alignment, false);
if (result != NULL) {
arena_prof_promoted(result,
usize);
}
} else {
result = ipalloc(usize, alignment,
false);
}
}
} else
result = ipalloc(usize, alignment, false);
}
if (result == NULL) {
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in posix_memalign(): "
"out of memory\n");
abort();
}
ret = ENOMEM;
goto RETURN;
}
*memptr = result;
ret = 0;
RETURN:
if (config_stats && result != NULL) {
assert(usize == isalloc(result));
ALLOCATED_ADD(usize, 0);
}
if (config_prof && opt_prof && result != NULL)
prof_malloc(result, usize, cnt);
return (ret);
}
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size)
{
return imemalign(memptr, alignment, size);
}
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void *
JEMALLOC_P(calloc)(size_t num, size_t size)
{
void *ret;
size_t num_size;
size_t usize;
prof_thr_cnt_t *cnt
#ifdef JEMALLOC_CC_SILENCE
= NULL
#endif
;
if (malloc_init()) {
num_size = 0;
ret = NULL;
goto RETURN;
}
num_size = num * size;
if (num_size == 0) {
if ((config_sysv == false || opt_sysv == false)
&& ((num == 0) || (size == 0)))
num_size = 1;
else {
ret = NULL;
goto RETURN;
}
/*
* Try to avoid division here. We know that it isn't possible to
* overflow during multiplication if neither operand uses any of the
* most significant half of the bits in a size_t.
*/
} else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
&& (num_size / size != num)) {
/* size_t overflow. */
ret = NULL;
goto RETURN;
}
if (config_prof && opt_prof) {
usize = s2u(num_size);
PROF_ALLOC_PREP(1, usize, cnt);
if (cnt == NULL) {
ret = NULL;
goto RETURN;
}
if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize
<= small_maxclass) {
ret = icalloc(small_maxclass+1);
if (ret != NULL)
arena_prof_promoted(ret, usize);
} else
ret = icalloc(num_size);
} else {
if (config_stats)
usize = s2u(num_size);
ret = icalloc(num_size);
}
RETURN:
if (ret == NULL) {
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in calloc(): out of "
"memory\n");
abort();
}
errno = ENOMEM;
}
if (config_prof && opt_prof && ret != NULL)
prof_malloc(ret, usize, cnt);
if (config_stats && ret != NULL) {
assert(usize == isalloc(ret));
ALLOCATED_ADD(usize, 0);
}
return (ret);
}
JEMALLOC_ATTR(visibility("default"))
void *
JEMALLOC_P(realloc)(void *ptr, size_t size)
{
void *ret;
size_t usize;
size_t old_size = 0;
prof_thr_cnt_t *cnt
#ifdef JEMALLOC_CC_SILENCE
= NULL
#endif
;
prof_ctx_t *old_ctx
#ifdef JEMALLOC_CC_SILENCE
= NULL
#endif
;
if (size == 0) {
if (config_sysv == false || opt_sysv == false)
size = 1;
else {
if (ptr != NULL) {
if (config_prof || config_stats)
old_size = isalloc(ptr);
if (config_prof && opt_prof) {
old_ctx = prof_ctx_get(ptr);
cnt = NULL;
}
idalloc(ptr);
} else if (config_prof && opt_prof) {
old_ctx = NULL;
cnt = NULL;
}
ret = NULL;
goto RETURN;
}
}
if (ptr != NULL) {
assert(malloc_initialized || malloc_initializer ==
pthread_self());
if (config_prof || config_stats)
old_size = isalloc(ptr);
if (config_prof && opt_prof) {
usize = s2u(size);
old_ctx = prof_ctx_get(ptr);
PROF_ALLOC_PREP(1, usize, cnt);
if (cnt == NULL) {
old_ctx = NULL;
ret = NULL;
goto OOM;
}
if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U &&
usize <= small_maxclass) {
ret = iralloc(ptr, small_maxclass+1, 0, 0,
false, false);
if (ret != NULL)
arena_prof_promoted(ret, usize);
else
old_ctx = NULL;
} else {
ret = iralloc(ptr, size, 0, 0, false, false);
if (ret == NULL)
old_ctx = NULL;
}
} else {
if (config_stats)
usize = s2u(size);
ret = iralloc(ptr, size, 0, 0, false, false);
}
OOM:
if (ret == NULL) {
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in realloc(): "
"out of memory\n");
abort();
}
errno = ENOMEM;
}
} else {
if (config_prof && opt_prof)
old_ctx = NULL;
if (malloc_init()) {
if (config_prof && opt_prof)
cnt = NULL;
ret = NULL;
} else {
if (config_prof && opt_prof) {
usize = s2u(size);
PROF_ALLOC_PREP(1, usize, cnt);
if (cnt == NULL)
ret = NULL;
else {
if (prof_promote && (uintptr_t)cnt !=
(uintptr_t)1U && usize <=
small_maxclass) {
ret = imalloc(small_maxclass+1);
if (ret != NULL) {
arena_prof_promoted(ret,
usize);
}
} else
ret = imalloc(size);
}
} else {
if (config_stats)
usize = s2u(size);
ret = imalloc(size);
}
}
if (ret == NULL) {
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in realloc(): "
"out of memory\n");
abort();
}
errno = ENOMEM;
}
}
RETURN:
if (config_prof && opt_prof)
prof_realloc(ret, usize, cnt, old_size, old_ctx);
if (config_stats && ret != NULL) {
assert(usize == isalloc(ret));
ALLOCATED_ADD(usize, old_size);
}
return (ret);
}
JEMALLOC_ATTR(visibility("default"))
void
JEMALLOC_P(free)(void *ptr)
{
if (ptr != NULL) {
size_t usize;
assert(malloc_initialized || malloc_initializer ==
pthread_self());
if (config_prof && opt_prof) {
usize = isalloc(ptr);
prof_free(ptr, usize);
} else if (config_stats) {
usize = isalloc(ptr);
}
if (config_stats)
ALLOCATED_ADD(0, usize);
idalloc(ptr);
}
}
/*
* End malloc(3)-compatible functions.
*/
/******************************************************************************/
/*
* Begin non-standard override functions.
*
* These overrides are omitted if the JEMALLOC_PREFIX is defined, since the
* entire point is to avoid accidental mixed allocator usage.
*/
#ifndef JEMALLOC_PREFIX
#ifdef JEMALLOC_OVERRIDE_MEMALIGN
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void *
JEMALLOC_P(memalign)(size_t alignment, size_t size)
{
void *ret
#ifdef JEMALLOC_CC_SILENCE
= NULL
#endif
;
imemalign(&ret, alignment, size);
return (ret);
}
#endif
#ifdef JEMALLOC_OVERRIDE_VALLOC
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void *
JEMALLOC_P(valloc)(size_t size)
{
void *ret
#ifdef JEMALLOC_CC_SILENCE
= NULL
#endif
;
imemalign(&ret, PAGE_SIZE, size);
return (ret);
}
#endif
#endif /* JEMALLOC_PREFIX */
/*
* End non-standard override functions.
*/
/******************************************************************************/
/*
* Begin non-standard functions.
*/
JEMALLOC_ATTR(visibility("default"))
size_t
JEMALLOC_P(malloc_usable_size)(const void *ptr)
{
size_t ret;
assert(malloc_initialized || malloc_initializer == pthread_self());
if (config_ivsalloc)
ret = ivsalloc(ptr);
else {
assert(ptr != NULL);
ret = isalloc(ptr);
}
return (ret);
}
JEMALLOC_ATTR(visibility("default"))
void
JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *),
void *cbopaque, const char *opts)
{
stats_print(write_cb, cbopaque, opts);
}
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, void *newp,
size_t newlen)
{
if (malloc_init())
return (EAGAIN);
return (ctl_byname(name, oldp, oldlenp, newp, newlen));
}
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, size_t *miblenp)
{
if (malloc_init())
return (EAGAIN);
return (ctl_nametomib(name, mibp, miblenp));
}
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
if (malloc_init())
return (EAGAIN);
return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen));
}
JEMALLOC_INLINE void *
iallocm(size_t usize, size_t alignment, bool zero)
{
assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, alignment,
NULL)));
if (alignment != 0)
return (ipalloc(usize, alignment, zero));
else if (zero)
return (icalloc(usize));
else
return (imalloc(usize));
}
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags)
{
void *p;
size_t usize;
size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
& (SIZE_T_MAX-1));
bool zero = flags & ALLOCM_ZERO;
prof_thr_cnt_t *cnt;
assert(ptr != NULL);
assert(size != 0);
if (malloc_init())
goto OOM;
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment, NULL);
if (usize == 0)
goto OOM;
if (config_prof && opt_prof) {
PROF_ALLOC_PREP(1, usize, cnt);
if (cnt == NULL)
goto OOM;
if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <=
small_maxclass) {
size_t usize_promoted = (alignment == 0) ?
s2u(small_maxclass+1) : sa2u(small_maxclass+1,
alignment, NULL);
assert(usize_promoted != 0);
p = iallocm(usize_promoted, alignment, zero);
if (p == NULL)
goto OOM;
arena_prof_promoted(p, usize);
} else {
p = iallocm(usize, alignment, zero);
if (p == NULL)
goto OOM;
}
prof_malloc(p, usize, cnt);
} else {
p = iallocm(usize, alignment, zero);
if (p == NULL)
goto OOM;
}
if (rsize != NULL)
*rsize = usize;
*ptr = p;
if (config_stats) {
assert(usize == isalloc(p));
ALLOCATED_ADD(usize, 0);
}
return (ALLOCM_SUCCESS);
OOM:
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in allocm(): "
"out of memory\n");
abort();
}
*ptr = NULL;
return (ALLOCM_ERR_OOM);
}
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra,
int flags)
{
void *p, *q;
size_t usize;
size_t old_size;
size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
& (SIZE_T_MAX-1));
bool zero = flags & ALLOCM_ZERO;
bool no_move = flags & ALLOCM_NO_MOVE;
prof_thr_cnt_t *cnt;
assert(ptr != NULL);
assert(*ptr != NULL);
assert(size != 0);
assert(SIZE_T_MAX - size >= extra);
assert(malloc_initialized || malloc_initializer == pthread_self());
p = *ptr;
if (config_prof && opt_prof) {
/*
* usize isn't knowable before iralloc() returns when extra is
* non-zero. Therefore, compute its maximum possible value and
* use that in PROF_ALLOC_PREP() to decide whether to capture a
* backtrace. prof_realloc() will use the actual usize to
* decide whether to sample.
*/
size_t max_usize = (alignment == 0) ? s2u(size+extra) :
sa2u(size+extra, alignment, NULL);
prof_ctx_t *old_ctx = prof_ctx_get(p);
old_size = isalloc(p);
PROF_ALLOC_PREP(1, max_usize, cnt);
if (cnt == NULL)
goto OOM;
/*
* Use minimum usize to determine whether promotion may happen.
*/
if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U
&& ((alignment == 0) ? s2u(size) : sa2u(size,
alignment, NULL)) <= small_maxclass) {
q = iralloc(p, small_maxclass+1, (small_maxclass+1 >=
size+extra) ? 0 : size+extra - (small_maxclass+1),
alignment, zero, no_move);
if (q == NULL)
goto ERR;
if (max_usize < PAGE_SIZE) {
usize = max_usize;
arena_prof_promoted(q, usize);
} else
usize = isalloc(q);
} else {
q = iralloc(p, size, extra, alignment, zero, no_move);
if (q == NULL)
goto ERR;
usize = isalloc(q);
}
prof_realloc(q, usize, cnt, old_size, old_ctx);
if (rsize != NULL)
*rsize = usize;
} else {
if (config_stats)
old_size = isalloc(p);
q = iralloc(p, size, extra, alignment, zero, no_move);
if (q == NULL)
goto ERR;
if (config_stats)
usize = isalloc(q);
if (rsize != NULL) {
if (config_stats == false)
usize = isalloc(q);
*rsize = usize;
}
}
*ptr = q;
if (config_stats)
ALLOCATED_ADD(usize, old_size);
return (ALLOCM_SUCCESS);
ERR:
if (no_move)
return (ALLOCM_ERR_NOT_MOVED);
OOM:
if (config_xmalloc && opt_xmalloc) {
malloc_write("<jemalloc>: Error in rallocm(): "
"out of memory\n");
abort();
}
return (ALLOCM_ERR_OOM);
}
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags)
{
size_t sz;
assert(malloc_initialized || malloc_initializer == pthread_self());
if (config_ivsalloc)
sz = ivsalloc(ptr);
else {
assert(ptr != NULL);
sz = isalloc(ptr);
}
assert(rsize != NULL);
*rsize = sz;
return (ALLOCM_SUCCESS);
}
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(dallocm)(void *ptr, int flags)
{
size_t usize;
assert(ptr != NULL);
assert(malloc_initialized || malloc_initializer == pthread_self());
if (config_stats)
usize = isalloc(ptr);
if (config_prof && opt_prof) {
if (config_stats == false)
usize = isalloc(ptr);
prof_free(ptr, usize);
}
if (config_stats)
ALLOCATED_ADD(0, usize);
idalloc(ptr);
return (ALLOCM_SUCCESS);
}
/*
* End non-standard functions.
*/
/******************************************************************************/
/*
* The following functions are used by threading libraries for protection of
* malloc during fork().
*/
void
jemalloc_prefork(void)
{
unsigned i;
/* Acquire all mutexes in a safe order. */
malloc_mutex_lock(&arenas_lock);
for (i = 0; i < narenas; i++) {
if (arenas[i] != NULL)
malloc_mutex_lock(&arenas[i]->lock);
}
malloc_mutex_lock(&base_mtx);
malloc_mutex_lock(&huge_mtx);
if (config_dss)
malloc_mutex_lock(&dss_mtx);
}
void
jemalloc_postfork(void)
{
unsigned i;
/* Release all mutexes, now that fork() has completed. */
if (config_dss)
malloc_mutex_unlock(&dss_mtx);
malloc_mutex_unlock(&huge_mtx);
malloc_mutex_unlock(&base_mtx);
for (i = 0; i < narenas; i++) {
if (arenas[i] != NULL)
malloc_mutex_unlock(&arenas[i]->lock);
}
malloc_mutex_unlock(&arenas_lock);
}
/******************************************************************************/
Reference in New Issue View Git Blame Copy Permalink