project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Port to FreeBSD.

Browse Source 
Use FreeBSD-specific functions (_pthread_mutex_init_calloc_cb(),
_malloc_{pre,post}fork()) to avoid bootstrapping issues due to
allocation in libc and libthr.

Add malloc_strtoumax() and use it instead of strtoul().  Disable
validation code in malloc_vsnprintf() and malloc_strtoumax() until
jemalloc is initialized.  This is necessary because locale
initialization causes allocation for both vsnprintf() and strtoumax().

Force the lazy-lock feature on in order to avoid pthread_self(),
because it causes allocation.

Use syscall(SYS_write, ...) rather than write(...), because libthr wraps
write() and causes allocation.  Without this workaround, it would not be
possible to print error messages in malloc_conf_init() without
substantially reworking bootstrapping.

Fix choose_arena_hard() to look at how many threads are assigned to the
candidate choice, rather than checking whether the arena is
uninitialized.  This bug potentially caused more arenas to be
initialized than necessary.
This commit is contained in:
Jason Evans
2012-02-02 22:04:57 -08:00
parent 6da5418ded
commit 41b6afb834
13 changed files with 237 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/base.c
View File

@@ -66,6 +66,17 @@ base_alloc(size_t size)
return (ret);
}
void *
base_calloc(size_t number, size_t size)
{
void *ret = base_alloc(number * size);
if (ret != NULL)
memset(ret, 0, number * size);
return (ret);
}
extent_node_t *
base_node_alloc(void)
{

8
src/ctl.c
View File

@@ -615,19 +615,19 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp,
goto RETURN;
}
} else {
unsigned long index;
uintmax_t index;
const ctl_node_t *inode;
/* Children are indexed. */
index = strtoul(elm, NULL, 10);
if (index == ULONG_MAX) {
index = malloc_strtoumax(elm, NULL, 10);
if (index == UINTMAX_MAX || index > SIZE_T_MAX) {
ret = ENOENT;
goto RETURN;
}
inode = &node->u.named.children[0];
node = inode->u.indexed.index(mibp, *depthp,
index);
(size_t)index);
if (node == NULL) {
ret = ENOENT;
goto RETURN;

64
src/jemalloc.c
View File

@@ -38,10 +38,18 @@ arena_t **arenas;
unsigned narenas;
/* Set to true once the allocator has been initialized. */
static bool malloc_initialized = false;
bool malloc_initialized = false;
#ifdef JEMALLOC_THREADED_INIT
/* Used to let the initializing thread recursively allocate. */
static pthread_t malloc_initializer = (unsigned long)0;
# define INITIALIZER pthread_self()
# define IS_INITIALIZER (malloc_initializer == pthread_self())
#else
static bool malloc_initializer = false;
# define INITIALIZER true
# define IS_INITIALIZER malloc_initializer
#endif
/* Used to avoid initialization races. */
static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
@@ -127,7 +135,7 @@ choose_arena_hard(void)
}
}
if (arenas[choose] == 0 || first_null == narenas) {
if (arenas[choose]->nthreads == 0 || first_null == narenas) {
/*
* Use an unloaded arena, or the least loaded arena if
* all arenas are already initialized.
@@ -413,22 +421,22 @@ malloc_conf_init(void)
#define CONF_HANDLE_SIZE_T(o, n, min, max) \
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
klen) == 0) { \
unsigned long ul; \
uintmax_t um; \
char *end; \
\
errno = 0; \
ul = strtoul(v, &end, 0); \
um = malloc_strtoumax(v, &end, 0); \
if (errno != 0 || (uintptr_t)end - \
(uintptr_t)v != vlen) { \
malloc_conf_error( \
"Invalid conf value", \
k, klen, v, vlen); \
} else if (ul < min || ul > max) { \
} else if (um < min || um > max) { \
malloc_conf_error( \
"Out-of-range conf value", \
k, klen, v, vlen); \
} else \
o = ul; \
o = um; \
continue; \
}
#define CONF_HANDLE_SSIZE_T(o, n, min, max) \
@@ -519,7 +527,7 @@ malloc_init_hard(void)
arena_t *init_arenas[1];
malloc_mutex_lock(&init_lock);
if (malloc_initialized || malloc_initializer == pthread_self()) {
if (malloc_initialized || IS_INITIALIZER) {
/*
* Another thread initialized the allocator before this one
* acquired init_lock, or this thread is the initializing
@@ -528,7 +536,8 @@ malloc_init_hard(void)
malloc_mutex_unlock(&init_lock);
return (false);
}
if (malloc_initializer != (unsigned long)0) {
#ifdef JEMALLOC_THREADED_INIT
if (IS_INITIALIZER == false) {
/* Busy-wait until the initializing thread completes. */
do {
malloc_mutex_unlock(&init_lock);
@@ -538,6 +547,8 @@ malloc_init_hard(void)
malloc_mutex_unlock(&init_lock);
return (false);
}
#endif
malloc_initializer = INITIALIZER;
#ifdef DYNAMIC_PAGE_SHIFT
/* Get page size. */
@@ -564,6 +575,7 @@ malloc_init_hard(void)
malloc_conf_init();
#ifndef JEMALLOC_MUTEX_INIT_CB
/* Register fork handlers. */
if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
jemalloc_postfork_child) != 0) {
@@ -571,11 +583,7 @@ malloc_init_hard(void)
if (opt_abort)
abort();
}
if (ctl_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
#endif
if (opt_stats_print) {
/* Print statistics at exit. */
@@ -596,6 +604,11 @@ malloc_init_hard(void)
return (true);
}
if (ctl_boot()) {
malloc_mutex_unlock(&init_lock);
return (true);
}
if (config_prof)
prof_boot1();
@@ -654,7 +667,6 @@ malloc_init_hard(void)
}
/* Get number of CPUs. */
malloc_initializer = pthread_self();
malloc_mutex_unlock(&init_lock);
ncpus = malloc_ncpus();
malloc_mutex_lock(&init_lock);
@@ -1018,8 +1030,7 @@ je_realloc(void *ptr, size_t size)
}
if (ptr != NULL) {
assert(malloc_initialized || malloc_initializer ==
pthread_self());
assert(malloc_initialized || IS_INITIALIZER);
if (config_prof || config_stats)
old_size = isalloc(ptr);
@@ -1124,8 +1135,7 @@ je_free(void *ptr)
if (ptr != NULL) {
size_t usize;
assert(malloc_initialized || malloc_initializer ==
pthread_self());
assert(malloc_initialized || IS_INITIALIZER);
if (config_prof && opt_prof) {
usize = isalloc(ptr);
@@ -1208,7 +1218,7 @@ je_malloc_usable_size(const void *ptr)
{
size_t ret;
assert(malloc_initialized || malloc_initializer == pthread_self());
assert(malloc_initialized || IS_INITIALIZER);
if (config_ivsalloc)
ret = ivsalloc(ptr);
@@ -1372,7 +1382,7 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
assert(*ptr != NULL);
assert(size != 0);
assert(SIZE_T_MAX - size >= extra);
assert(malloc_initialized || malloc_initializer == pthread_self());
assert(malloc_initialized || IS_INITIALIZER);
p = *ptr;
if (config_prof && opt_prof) {
@@ -1457,7 +1467,7 @@ je_sallocm(const void *ptr, size_t *rsize, int flags)
{
size_t sz;
assert(malloc_initialized || malloc_initializer == pthread_self());
assert(malloc_initialized || IS_INITIALIZER);
if (config_ivsalloc)
sz = ivsalloc(ptr);
@@ -1479,7 +1489,7 @@ je_dallocm(void *ptr, int flags)
size_t usize;
assert(ptr != NULL);
assert(malloc_initialized || malloc_initializer == pthread_self());
assert(malloc_initialized || IS_INITIALIZER);
if (config_stats)
usize = isalloc(ptr);
@@ -1528,8 +1538,13 @@ je_nallocm(size_t *rsize, size_t size, int flags)
* malloc during fork().
*/
#ifndef JEMALLOC_MUTEX_INIT_CB
void
jemalloc_prefork(void)
#else
void
_malloc_prefork(void)
#endif
{
unsigned i;
@@ -1544,8 +1559,13 @@ jemalloc_prefork(void)
chunk_dss_prefork();
}
#ifndef JEMALLOC_MUTEX_INIT_CB
void
jemalloc_postfork_parent(void)
#else
void
_malloc_postfork(void)
#endif
{
unsigned i;

18
src/mutex.c
View File

@@ -56,21 +56,25 @@ pthread_create(pthread_t *__restrict thread,
/******************************************************************************/
#ifdef JEMALLOC_MUTEX_INIT_CB
int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void *(calloc_cb)(size_t, size_t));
#endif
bool
malloc_mutex_init(malloc_mutex_t *mutex)
{
#ifdef JEMALLOC_OSSPIN
*mutex = 0;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
if (_pthread_mutex_init_calloc_cb(mutex, base_calloc) != 0)
return (true);
#else
pthread_mutexattr_t attr;
if (pthread_mutexattr_init(&attr) != 0)
return (true);
#ifdef PTHREAD_MUTEX_ADAPTIVE_NP
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
#else
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT);
#endif
pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE);
if (pthread_mutex_init(mutex, &attr) != 0) {
pthread_mutexattr_destroy(&attr);
return (true);
@@ -99,10 +103,14 @@ void
malloc_mutex_postfork_child(malloc_mutex_t *mutex)
{
#ifdef JEMALLOC_MUTEX_INIT_CB
malloc_mutex_unlock(mutex);
#else
if (malloc_mutex_init(mutex)) {
malloc_printf("<jemalloc>: Error re-initializing mutex in "
"child\n");
if (opt_abort)
abort();
}
#endif
}

136
src/util.c
View File

@@ -44,7 +44,7 @@ JEMALLOC_CATTR(visibility("hidden"), static)
void
wrtmessage(void *cbopaque, const char *s)
{
UNUSED int result = write(STDERR_FILENO, s, strlen(s));
UNUSED int result = syscall(SYS_write, STDERR_FILENO, s, strlen(s));
}
void (*je_malloc_message)(void *, const char *s)
@@ -69,6 +69,123 @@ buferror(int errnum, char *buf, size_t buflen)
#endif
}
uintmax_t
malloc_strtoumax(const char *nptr, char **endptr, int base)
{
uintmax_t ret, digit;
int b;
bool neg;
const char *p, *ns;
if (base < 0 || base == 1 || base > 36) {
errno = EINVAL;
return (UINTMAX_MAX);
}
b = base;
/* Swallow leading whitespace and get sign, if any. */
neg = false;
p = nptr;
while (true) {
switch (*p) {
case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
p++;
break;
case '-':
neg = true;
/* Fall through. */
case '+':
p++;
/* Fall through. */
default:
goto PREFIX;
}
}
/* Get prefix, if any. */
PREFIX:
/*
* Note where the first non-whitespace/sign character is so that it is
* possible to tell whether any digits are consumed (e.g., " 0" vs.
* " -x").
*/
ns = p;
if (*p == '0') {
switch (p[1]) {
case '0': case '1': case '2': case '3': case '4': case '5':
case '6': case '7':
if (b == 0)
b = 8;
if (b == 8)
p++;
break;
case 'x':
switch (p[2]) {
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F':
case 'a': case 'b': case 'c': case 'd': case 'e':
case 'f':
if (b == 0)
b = 16;
if (b == 16)
p += 2;
break;
default:
break;
}
break;
default:
break;
}
}
if (b == 0)
b = 10;
/* Convert. */
ret = 0;
while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b)
|| (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b)
|| (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) {
uintmax_t pret = ret;
ret *= b;
ret += digit;
if (ret < pret) {
/* Overflow. */
errno = ERANGE;
return (UINTMAX_MAX);
}
p++;
}
if (neg)
ret = -ret;
if (endptr != NULL) {
if (p == ns) {
/* No characters were converted. */
*endptr = (char *)nptr;
} else
*endptr = (char *)p;
}
if (config_debug && malloc_initialized) {
uintmax_t tret;
int perrno;
char *pend;
perrno = errno;
if (endptr != NULL)
pend = *endptr;
tret = strtoumax(nptr, endptr, base);
assert(tret == ret);
assert(errno == perrno);
assert(endptr == NULL || *endptr == pend);
}
return (ret);
}
static char *
u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p)
{
@@ -220,7 +337,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
val = va_arg(ap, ptrdiff_t); \
break; \
case 'z': \
val = va_arg(ap, size_t); \
val = va_arg(ap, ssize_t); \
break; \
case 'p': /* Synthetic; used for %p. */ \
val = va_arg(ap, uintptr_t); \
@@ -289,10 +406,11 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
break;
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': {
unsigned long uwidth;
uintmax_t uwidth;
errno = 0;
uwidth = strtoul(f, (char **)&f, 10);
assert(uwidth != ULONG_MAX || errno != ERANGE);
uwidth = malloc_strtoumax(f, (char **)&f, 10);
assert(uwidth != UINTMAX_MAX || errno !=
ERANGE);
width = (int)uwidth;
if (*f == '.') {
f++;
@@ -314,10 +432,10 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
break;
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': {
unsigned long uprec;
uintmax_t uprec;
errno = 0;
uprec = strtoul(f, (char **)&f, 10);
assert(uprec != ULONG_MAX || errno != ERANGE);
uprec = malloc_strtoumax(f, (char **)&f, 10);
assert(uprec != UINTMAX_MAX || errno != ERANGE);
prec = (int)uprec;
break;
}
@@ -435,7 +553,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
str[size - 1] = '\0';
ret = i;
if (config_debug) {
if (config_debug && malloc_initialized) {
char buf[MALLOC_PRINTF_BUFSIZE];
int tret;