diff --git a/README b/README index 4d7b552b..a7864f33 100644 --- a/README +++ b/README @@ -1,10 +1,10 @@ jemalloc is a general-purpose scalable concurrent malloc(3) implementation. This distribution is a stand-alone "portable" implementation that currently -targets Linux and Apple OS X. jemalloc is included as the default allocator in -the FreeBSD and NetBSD operating systems, and it is used by the Mozilla Firefox -web browser on Microsoft Windows-related platforms. Depending on your needs, -one of the other divergent versions may suit your needs better than this -distribution. +targets FreeBSD, Linux and Apple OS X. jemalloc is included as the default +allocator in the FreeBSD and NetBSD operating systems, and it is used by the +Mozilla Firefox web browser on Microsoft Windows-related platforms. Depending +on your needs, one of the other divergent versions may suit your needs better +than this distribution. The COPYING file contains copyright and licensing information. diff --git a/configure.ac b/configure.ac index 7e4f2211..478ae9d4 100644 --- a/configure.ac +++ b/configure.ac @@ -777,6 +777,17 @@ if test "x$have__malloc_thread_cleanup" = "x1" ; then force_tls="1" fi +dnl Check whether the BSD-specific _pthread_mutex_init_calloc_cb() exists. If +dnl so, mutex initialization causes allocation, and we need to implement this +dnl callback function in order to prevent recursive allocation. +AC_CHECK_FUNC([_pthread_mutex_init_calloc_cb], + [have__pthread_mutex_init_calloc_cb="1"], + [have__pthread_mutex_init_calloc_cb="0"] + ) +if test "x$have__pthread_mutex_init_calloc_cb" = "x1" ; then + AC_DEFINE([JEMALLOC_MUTEX_INIT_CB]) +fi + dnl Disable lazy locking by default. AC_ARG_ENABLE([lazy_lock], [AS_HELP_STRING([--enable-lazy-lock], diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h index 796a2835..9cf75ffb 100644 --- a/include/jemalloc/internal/base.h +++ b/include/jemalloc/internal/base.h @@ -10,6 +10,7 @@ #ifdef JEMALLOC_H_EXTERNS void *base_alloc(size_t size); +void *base_calloc(size_t number, size_t size); extent_node_t *base_node_alloc(void); void base_node_dealloc(extent_node_t *node); bool base_boot(void); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index e0558140..4f557794 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -370,6 +371,8 @@ extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ extern arena_t **arenas; extern unsigned narenas; +extern bool malloc_initialized; + arena_t *arenas_extend(unsigned ind); void arenas_cleanup(void *arg); arena_t *choose_arena_hard(void); diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index 10637e92..98f2cba5 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -6,9 +6,12 @@ typedef OSSpinLock malloc_mutex_t; #define MALLOC_MUTEX_INITIALIZER 0 #else typedef pthread_mutex_t malloc_mutex_t; -# ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +# if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) && \ + defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP)) +# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP # define MALLOC_MUTEX_INITIALIZER PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP # else +# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT # define MALLOC_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER # endif #endif diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 5a174acd..0e32c612 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -71,7 +71,7 @@ a_name##_tsd_set(a_type *val); #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP #define malloc_tsd_externs(a_name, a_type) \ extern __thread a_type a_name##_tls; \ -extern __thread bool *a_name##_initialized; \ +extern __thread bool a_name##_initialized; \ extern bool a_name##_booted; #elif (defined(JEMALLOC_TLS)) #define malloc_tsd_externs(a_name, a_type) \ diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index 5156399f..3d3ea3ab 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -85,6 +85,7 @@ extern void (*je_malloc_message)(void *wcbopaque, const char *s); int buferror(int errnum, char *buf, size_t buflen); +uintmax_t malloc_strtoumax(const char *nptr, char **endptr, int base); /* * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in index 838f5615..f150413e 100644 --- a/include/jemalloc/jemalloc_defs.h.in +++ b/include/jemalloc/jemalloc_defs.h.in @@ -68,6 +68,20 @@ */ #undef JEMALLOC_MALLOC_THREAD_CLEANUP +/* + * Defined if threaded initialization is known to be safe on this platform. + * Among other things, it must be possible to initialize a mutex without + * triggering allocation in order for threaded allocation to be safe. + */ +#undef JEMALLOC_THREADED_INIT + +/* + * Defined if the pthreads implementation defines + * _pthread_mutex_init_calloc_cb(), in which case the function is used in order + * to avoid recursive allocation during mutex initialization. + */ +#undef JEMALLOC_MUTEX_INIT_CB + /* Defined if __attribute__((...)) syntax is supported. */ #undef JEMALLOC_HAVE_ATTR #ifdef JEMALLOC_HAVE_ATTR diff --git a/src/base.c b/src/base.c index eb68334b..696c362a 100644 --- a/src/base.c +++ b/src/base.c @@ -66,6 +66,17 @@ base_alloc(size_t size) return (ret); } +void * +base_calloc(size_t number, size_t size) +{ + void *ret = base_alloc(number * size); + + if (ret != NULL) + memset(ret, 0, number * size); + + return (ret); +} + extent_node_t * base_node_alloc(void) { diff --git a/src/ctl.c b/src/ctl.c index e17e5034..943c2925 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -615,19 +615,19 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, goto RETURN; } } else { - unsigned long index; + uintmax_t index; const ctl_node_t *inode; /* Children are indexed. */ - index = strtoul(elm, NULL, 10); - if (index == ULONG_MAX) { + index = malloc_strtoumax(elm, NULL, 10); + if (index == UINTMAX_MAX || index > SIZE_T_MAX) { ret = ENOENT; goto RETURN; } inode = &node->u.named.children[0]; node = inode->u.indexed.index(mibp, *depthp, - index); + (size_t)index); if (node == NULL) { ret = ENOENT; goto RETURN; diff --git a/src/jemalloc.c b/src/jemalloc.c index b3e898c1..3e168fd0 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -38,10 +38,18 @@ arena_t **arenas; unsigned narenas; /* Set to true once the allocator has been initialized. */ -static bool malloc_initialized = false; +bool malloc_initialized = false; +#ifdef JEMALLOC_THREADED_INIT /* Used to let the initializing thread recursively allocate. */ static pthread_t malloc_initializer = (unsigned long)0; +# define INITIALIZER pthread_self() +# define IS_INITIALIZER (malloc_initializer == pthread_self()) +#else +static bool malloc_initializer = false; +# define INITIALIZER true +# define IS_INITIALIZER malloc_initializer +#endif /* Used to avoid initialization races. */ static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; @@ -127,7 +135,7 @@ choose_arena_hard(void) } } - if (arenas[choose] == 0 || first_null == narenas) { + if (arenas[choose]->nthreads == 0 || first_null == narenas) { /* * Use an unloaded arena, or the least loaded arena if * all arenas are already initialized. @@ -413,22 +421,22 @@ malloc_conf_init(void) #define CONF_HANDLE_SIZE_T(o, n, min, max) \ if (sizeof(#n)-1 == klen && strncmp(#n, k, \ klen) == 0) { \ - unsigned long ul; \ + uintmax_t um; \ char *end; \ \ errno = 0; \ - ul = strtoul(v, &end, 0); \ + um = malloc_strtoumax(v, &end, 0); \ if (errno != 0 || (uintptr_t)end - \ (uintptr_t)v != vlen) { \ malloc_conf_error( \ "Invalid conf value", \ k, klen, v, vlen); \ - } else if (ul < min || ul > max) { \ + } else if (um < min || um > max) { \ malloc_conf_error( \ "Out-of-range conf value", \ k, klen, v, vlen); \ } else \ - o = ul; \ + o = um; \ continue; \ } #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ @@ -519,7 +527,7 @@ malloc_init_hard(void) arena_t *init_arenas[1]; malloc_mutex_lock(&init_lock); - if (malloc_initialized || malloc_initializer == pthread_self()) { + if (malloc_initialized || IS_INITIALIZER) { /* * Another thread initialized the allocator before this one * acquired init_lock, or this thread is the initializing @@ -528,7 +536,8 @@ malloc_init_hard(void) malloc_mutex_unlock(&init_lock); return (false); } - if (malloc_initializer != (unsigned long)0) { +#ifdef JEMALLOC_THREADED_INIT + if (IS_INITIALIZER == false) { /* Busy-wait until the initializing thread completes. */ do { malloc_mutex_unlock(&init_lock); @@ -538,6 +547,8 @@ malloc_init_hard(void) malloc_mutex_unlock(&init_lock); return (false); } +#endif + malloc_initializer = INITIALIZER; #ifdef DYNAMIC_PAGE_SHIFT /* Get page size. */ @@ -564,6 +575,7 @@ malloc_init_hard(void) malloc_conf_init(); +#ifndef JEMALLOC_MUTEX_INIT_CB /* Register fork handlers. */ if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, jemalloc_postfork_child) != 0) { @@ -571,11 +583,7 @@ malloc_init_hard(void) if (opt_abort) abort(); } - - if (ctl_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } +#endif if (opt_stats_print) { /* Print statistics at exit. */ @@ -596,6 +604,11 @@ malloc_init_hard(void) return (true); } + if (ctl_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + if (config_prof) prof_boot1(); @@ -654,7 +667,6 @@ malloc_init_hard(void) } /* Get number of CPUs. */ - malloc_initializer = pthread_self(); malloc_mutex_unlock(&init_lock); ncpus = malloc_ncpus(); malloc_mutex_lock(&init_lock); @@ -1018,8 +1030,7 @@ je_realloc(void *ptr, size_t size) } if (ptr != NULL) { - assert(malloc_initialized || malloc_initializer == - pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); if (config_prof || config_stats) old_size = isalloc(ptr); @@ -1124,8 +1135,7 @@ je_free(void *ptr) if (ptr != NULL) { size_t usize; - assert(malloc_initialized || malloc_initializer == - pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); if (config_prof && opt_prof) { usize = isalloc(ptr); @@ -1208,7 +1218,7 @@ je_malloc_usable_size(const void *ptr) { size_t ret; - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); if (config_ivsalloc) ret = ivsalloc(ptr); @@ -1372,7 +1382,7 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) assert(*ptr != NULL); assert(size != 0); assert(SIZE_T_MAX - size >= extra); - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); p = *ptr; if (config_prof && opt_prof) { @@ -1457,7 +1467,7 @@ je_sallocm(const void *ptr, size_t *rsize, int flags) { size_t sz; - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); if (config_ivsalloc) sz = ivsalloc(ptr); @@ -1479,7 +1489,7 @@ je_dallocm(void *ptr, int flags) size_t usize; assert(ptr != NULL); - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); if (config_stats) usize = isalloc(ptr); @@ -1528,8 +1538,13 @@ je_nallocm(size_t *rsize, size_t size, int flags) * malloc during fork(). */ +#ifndef JEMALLOC_MUTEX_INIT_CB void jemalloc_prefork(void) +#else +void +_malloc_prefork(void) +#endif { unsigned i; @@ -1544,8 +1559,13 @@ jemalloc_prefork(void) chunk_dss_prefork(); } +#ifndef JEMALLOC_MUTEX_INIT_CB void jemalloc_postfork_parent(void) +#else +void +_malloc_postfork(void) +#endif { unsigned i; diff --git a/src/mutex.c b/src/mutex.c index 07d2a033..0b20bbf3 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -56,21 +56,25 @@ pthread_create(pthread_t *__restrict thread, /******************************************************************************/ +#ifdef JEMALLOC_MUTEX_INIT_CB +int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, + void *(calloc_cb)(size_t, size_t)); +#endif + bool malloc_mutex_init(malloc_mutex_t *mutex) { #ifdef JEMALLOC_OSSPIN *mutex = 0; +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) + if (_pthread_mutex_init_calloc_cb(mutex, base_calloc) != 0) + return (true); #else pthread_mutexattr_t attr; if (pthread_mutexattr_init(&attr) != 0) return (true); -#ifdef PTHREAD_MUTEX_ADAPTIVE_NP - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP); -#else - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); -#endif + pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE); if (pthread_mutex_init(mutex, &attr) != 0) { pthread_mutexattr_destroy(&attr); return (true); @@ -99,10 +103,14 @@ void malloc_mutex_postfork_child(malloc_mutex_t *mutex) { +#ifdef JEMALLOC_MUTEX_INIT_CB + malloc_mutex_unlock(mutex); +#else if (malloc_mutex_init(mutex)) { malloc_printf(": Error re-initializing mutex in " "child\n"); if (opt_abort) abort(); } +#endif } diff --git a/src/util.c b/src/util.c index 698b53a9..090e1f06 100644 --- a/src/util.c +++ b/src/util.c @@ -44,7 +44,7 @@ JEMALLOC_CATTR(visibility("hidden"), static) void wrtmessage(void *cbopaque, const char *s) { - UNUSED int result = write(STDERR_FILENO, s, strlen(s)); + UNUSED int result = syscall(SYS_write, STDERR_FILENO, s, strlen(s)); } void (*je_malloc_message)(void *, const char *s) @@ -69,6 +69,123 @@ buferror(int errnum, char *buf, size_t buflen) #endif } +uintmax_t +malloc_strtoumax(const char *nptr, char **endptr, int base) +{ + uintmax_t ret, digit; + int b; + bool neg; + const char *p, *ns; + + if (base < 0 || base == 1 || base > 36) { + errno = EINVAL; + return (UINTMAX_MAX); + } + b = base; + + /* Swallow leading whitespace and get sign, if any. */ + neg = false; + p = nptr; + while (true) { + switch (*p) { + case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': + p++; + break; + case '-': + neg = true; + /* Fall through. */ + case '+': + p++; + /* Fall through. */ + default: + goto PREFIX; + } + } + + /* Get prefix, if any. */ + PREFIX: + /* + * Note where the first non-whitespace/sign character is so that it is + * possible to tell whether any digits are consumed (e.g., " 0" vs. + * " -x"). + */ + ns = p; + if (*p == '0') { + switch (p[1]) { + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': + if (b == 0) + b = 8; + if (b == 8) + p++; + break; + case 'x': + switch (p[2]) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + if (b == 0) + b = 16; + if (b == 16) + p += 2; + break; + default: + break; + } + break; + default: + break; + } + } + if (b == 0) + b = 10; + + /* Convert. */ + ret = 0; + while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b) + || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b) + || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) { + uintmax_t pret = ret; + ret *= b; + ret += digit; + if (ret < pret) { + /* Overflow. */ + errno = ERANGE; + return (UINTMAX_MAX); + } + p++; + } + if (neg) + ret = -ret; + + if (endptr != NULL) { + if (p == ns) { + /* No characters were converted. */ + *endptr = (char *)nptr; + } else + *endptr = (char *)p; + } + + if (config_debug && malloc_initialized) { + uintmax_t tret; + int perrno; + char *pend; + + perrno = errno; + if (endptr != NULL) + pend = *endptr; + tret = strtoumax(nptr, endptr, base); + assert(tret == ret); + assert(errno == perrno); + assert(endptr == NULL || *endptr == pend); + } + + return (ret); +} + static char * u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { @@ -220,7 +337,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) val = va_arg(ap, ptrdiff_t); \ break; \ case 'z': \ - val = va_arg(ap, size_t); \ + val = va_arg(ap, ssize_t); \ break; \ case 'p': /* Synthetic; used for %p. */ \ val = va_arg(ap, uintptr_t); \ @@ -289,10 +406,11 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - unsigned long uwidth; + uintmax_t uwidth; errno = 0; - uwidth = strtoul(f, (char **)&f, 10); - assert(uwidth != ULONG_MAX || errno != ERANGE); + uwidth = malloc_strtoumax(f, (char **)&f, 10); + assert(uwidth != UINTMAX_MAX || errno != + ERANGE); width = (int)uwidth; if (*f == '.') { f++; @@ -314,10 +432,10 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - unsigned long uprec; + uintmax_t uprec; errno = 0; - uprec = strtoul(f, (char **)&f, 10); - assert(uprec != ULONG_MAX || errno != ERANGE); + uprec = malloc_strtoumax(f, (char **)&f, 10); + assert(uprec != UINTMAX_MAX || errno != ERANGE); prec = (int)uprec; break; } @@ -435,7 +553,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) str[size - 1] = '\0'; ret = i; - if (config_debug) { + if (config_debug && malloc_initialized) { char buf[MALLOC_PRINTF_BUFSIZE]; int tret;