2010-01-24 18:53:40 +08:00
|
|
|
#define JEMALLOC_CHUNK_DSS_C_
|
2010-02-12 06:45:59 +08:00
|
|
|
#include "jemalloc/internal/jemalloc_internal.h"
|
2010-01-24 18:53:40 +08:00
|
|
|
/******************************************************************************/
|
|
|
|
/* Data. */
|
|
|
|
|
2012-10-12 04:53:15 +08:00
|
|
|
const char *dss_prec_names[] = {
|
|
|
|
"disabled",
|
|
|
|
"primary",
|
|
|
|
"secondary",
|
|
|
|
"N/A"
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Current dss precedence default, used when creating new arenas. */
|
|
|
|
static dss_prec_t dss_prec_default = DSS_PREC_DEFAULT;
|
|
|
|
|
2012-03-14 07:31:41 +08:00
|
|
|
/*
|
|
|
|
* Protects sbrk() calls. This avoids malloc races among threads, though it
|
|
|
|
* does not protect against races with threads that call sbrk() directly.
|
|
|
|
*/
|
|
|
|
static malloc_mutex_t dss_mtx;
|
2010-01-24 18:53:40 +08:00
|
|
|
|
|
|
|
/* Base address of the DSS. */
|
2012-03-14 07:31:41 +08:00
|
|
|
static void *dss_base;
|
2010-01-24 18:53:40 +08:00
|
|
|
/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */
|
2012-03-14 07:31:41 +08:00
|
|
|
static void *dss_prev;
|
2010-01-24 18:53:40 +08:00
|
|
|
/* Current upper limit on DSS addresses. */
|
2012-03-14 07:31:41 +08:00
|
|
|
static void *dss_max;
|
2010-01-24 18:53:40 +08:00
|
|
|
|
|
|
|
/******************************************************************************/
|
|
|
|
|
|
|
|
static void *
|
2013-12-04 13:49:36 +08:00
|
|
|
chunk_dss_sbrk(intptr_t increment)
|
2010-01-24 18:53:40 +08:00
|
|
|
{
|
|
|
|
|
2014-04-16 03:09:48 +08:00
|
|
|
#ifdef JEMALLOC_DSS
|
2013-12-04 13:49:36 +08:00
|
|
|
return (sbrk(increment));
|
|
|
|
#else
|
2012-04-13 11:20:58 +08:00
|
|
|
not_implemented();
|
|
|
|
return (NULL);
|
|
|
|
#endif
|
2013-12-04 13:49:36 +08:00
|
|
|
}
|
2010-01-24 18:53:40 +08:00
|
|
|
|
2012-10-12 04:53:15 +08:00
|
|
|
dss_prec_t
|
2016-05-11 13:21:10 +08:00
|
|
|
chunk_dss_prec_get(tsdn_t *tsdn)
|
2012-10-12 04:53:15 +08:00
|
|
|
{
|
|
|
|
dss_prec_t ret;
|
|
|
|
|
2014-10-04 01:16:09 +08:00
|
|
|
if (!have_dss)
|
2012-10-12 04:53:15 +08:00
|
|
|
return (dss_prec_disabled);
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_lock(tsdn, &dss_mtx);
|
2012-10-12 04:53:15 +08:00
|
|
|
ret = dss_prec_default;
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_unlock(tsdn, &dss_mtx);
|
2012-10-12 04:53:15 +08:00
|
|
|
return (ret);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
2016-05-11 13:21:10 +08:00
|
|
|
chunk_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec)
|
2012-10-12 04:53:15 +08:00
|
|
|
{
|
|
|
|
|
2014-10-04 01:16:09 +08:00
|
|
|
if (!have_dss)
|
2014-04-16 03:09:48 +08:00
|
|
|
return (dss_prec != dss_prec_disabled);
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_lock(tsdn, &dss_mtx);
|
2012-10-12 04:53:15 +08:00
|
|
|
dss_prec_default = dss_prec;
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_unlock(tsdn, &dss_mtx);
|
2012-10-12 04:53:15 +08:00
|
|
|
return (false);
|
|
|
|
}
|
|
|
|
|
2010-01-24 18:53:40 +08:00
|
|
|
void *
|
2016-05-11 13:21:10 +08:00
|
|
|
chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
2016-04-14 14:36:15 +08:00
|
|
|
size_t alignment, bool *zero, bool *commit)
|
2010-01-24 18:53:40 +08:00
|
|
|
{
|
2016-05-27 13:12:38 +08:00
|
|
|
void *ret;
|
|
|
|
extent_t *pad;
|
|
|
|
|
2014-04-16 03:09:48 +08:00
|
|
|
cassert(have_dss);
|
2016-05-27 13:12:38 +08:00
|
|
|
assert(size > 0);
|
|
|
|
assert(alignment > 0);
|
2012-02-11 12:22:09 +08:00
|
|
|
|
2010-01-24 18:53:40 +08:00
|
|
|
/*
|
|
|
|
* sbrk() uses a signed increment argument, so take care not to
|
2016-06-01 05:50:21 +08:00
|
|
|
* interpret a large allocation request as a negative increment.
|
2010-01-24 18:53:40 +08:00
|
|
|
*/
|
|
|
|
if ((intptr_t)size < 0)
|
|
|
|
return (NULL);
|
|
|
|
|
2016-05-27 13:12:38 +08:00
|
|
|
pad = extent_alloc(tsdn, arena);
|
|
|
|
if (pad == NULL)
|
|
|
|
return (NULL);
|
|
|
|
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_lock(tsdn, &dss_mtx);
|
2010-01-24 18:53:40 +08:00
|
|
|
if (dss_prev != (void *)-1) {
|
|
|
|
/*
|
|
|
|
* The loop is necessary to recover from races with other
|
|
|
|
* threads that are using the DSS for something other than
|
|
|
|
* malloc.
|
|
|
|
*/
|
2016-05-27 13:12:38 +08:00
|
|
|
while (true) {
|
|
|
|
void *pad_addr, *dss_next;
|
|
|
|
size_t pad_size;
|
2015-09-08 20:09:20 +08:00
|
|
|
intptr_t incr;
|
2016-05-27 13:12:38 +08:00
|
|
|
|
2014-11-04 03:02:52 +08:00
|
|
|
/* Avoid an unnecessary system call. */
|
|
|
|
if (new_addr != NULL && dss_max != new_addr)
|
|
|
|
break;
|
|
|
|
|
2010-01-24 18:53:40 +08:00
|
|
|
/* Get the current end of the DSS. */
|
2013-12-04 13:49:36 +08:00
|
|
|
dss_max = chunk_dss_sbrk(0);
|
2014-11-04 03:02:52 +08:00
|
|
|
|
|
|
|
/* Make sure the earlier condition still holds. */
|
|
|
|
if (new_addr != NULL && dss_max != new_addr)
|
|
|
|
break;
|
|
|
|
|
2010-01-24 18:53:40 +08:00
|
|
|
/*
|
2016-05-27 13:12:38 +08:00
|
|
|
* Compute how much pad space (if any) is necessary to
|
|
|
|
* satisfy alignment. This space can be recycled for
|
|
|
|
* later use.
|
2010-01-24 18:53:40 +08:00
|
|
|
*/
|
2016-05-27 13:12:38 +08:00
|
|
|
pad_addr = (void *)((uintptr_t)dss_max);
|
2012-04-12 09:13:45 +08:00
|
|
|
ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max,
|
|
|
|
alignment);
|
2016-05-27 13:12:38 +08:00
|
|
|
pad_size = (uintptr_t)ret - (uintptr_t)pad_addr;
|
|
|
|
if (pad_size != 0) {
|
|
|
|
extent_init(pad, arena, pad_addr, pad_size,
|
2016-05-30 09:34:50 +08:00
|
|
|
pad_size, false, false, true, false);
|
2016-05-24 05:56:35 +08:00
|
|
|
}
|
2012-04-11 01:50:33 +08:00
|
|
|
dss_next = (void *)((uintptr_t)ret + size);
|
|
|
|
if ((uintptr_t)ret < (uintptr_t)dss_max ||
|
2016-05-27 13:12:38 +08:00
|
|
|
(uintptr_t)dss_next < (uintptr_t)dss_max)
|
|
|
|
break; /* Wrap-around. */
|
|
|
|
incr = pad_size + size;
|
2013-12-04 13:49:36 +08:00
|
|
|
dss_prev = chunk_dss_sbrk(incr);
|
2016-05-27 13:12:38 +08:00
|
|
|
if (dss_prev == (void *)-1)
|
|
|
|
break;
|
2010-01-24 18:53:40 +08:00
|
|
|
if (dss_prev == dss_max) {
|
|
|
|
/* Success. */
|
2012-04-11 01:50:33 +08:00
|
|
|
dss_max = dss_next;
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_unlock(tsdn, &dss_mtx);
|
2016-05-27 13:12:38 +08:00
|
|
|
if (pad_size != 0) {
|
2016-06-01 06:03:51 +08:00
|
|
|
extent_hooks_t extent_hooks =
|
2016-06-02 02:56:45 +08:00
|
|
|
EXTENT_HOOKS_INITIALIZER;
|
2016-05-11 13:21:10 +08:00
|
|
|
chunk_dalloc_wrapper(tsdn, arena,
|
2016-06-01 06:03:51 +08:00
|
|
|
&extent_hooks, pad);
|
2016-05-27 13:12:38 +08:00
|
|
|
} else
|
|
|
|
extent_dalloc(tsdn, arena, pad);
|
2016-04-06 07:25:44 +08:00
|
|
|
if (*zero)
|
2012-04-22 07:04:51 +08:00
|
|
|
memset(ret, 0, size);
|
2015-08-13 01:26:54 +08:00
|
|
|
if (!*commit)
|
|
|
|
*commit = pages_decommit(ret, size);
|
2010-01-24 18:53:40 +08:00
|
|
|
return (ret);
|
|
|
|
}
|
2016-05-27 13:12:38 +08:00
|
|
|
}
|
2010-01-24 18:53:40 +08:00
|
|
|
}
|
2016-05-27 13:12:38 +08:00
|
|
|
/* OOM. */
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_unlock(tsdn, &dss_mtx);
|
2016-05-27 13:12:38 +08:00
|
|
|
extent_dalloc(tsdn, arena, pad);
|
2010-01-24 18:53:40 +08:00
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
2010-12-01 08:50:58 +08:00
|
|
|
bool
|
2016-05-11 13:21:10 +08:00
|
|
|
chunk_in_dss(tsdn_t *tsdn, void *chunk)
|
2010-12-01 08:50:58 +08:00
|
|
|
{
|
|
|
|
bool ret;
|
|
|
|
|
2014-04-16 03:09:48 +08:00
|
|
|
cassert(have_dss);
|
2012-02-11 12:22:09 +08:00
|
|
|
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_lock(tsdn, &dss_mtx);
|
2010-12-01 08:50:58 +08:00
|
|
|
if ((uintptr_t)chunk >= (uintptr_t)dss_base
|
|
|
|
&& (uintptr_t)chunk < (uintptr_t)dss_max)
|
|
|
|
ret = true;
|
|
|
|
else
|
|
|
|
ret = false;
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_unlock(tsdn, &dss_mtx);
|
2010-12-01 08:50:58 +08:00
|
|
|
|
|
|
|
return (ret);
|
|
|
|
}
|
|
|
|
|
2010-01-24 18:53:40 +08:00
|
|
|
bool
|
|
|
|
chunk_dss_boot(void)
|
|
|
|
{
|
|
|
|
|
2014-04-16 03:09:48 +08:00
|
|
|
cassert(have_dss);
|
2012-02-11 12:22:09 +08:00
|
|
|
|
2016-04-14 14:36:15 +08:00
|
|
|
if (malloc_mutex_init(&dss_mtx, "dss", WITNESS_RANK_DSS))
|
2010-01-24 18:53:40 +08:00
|
|
|
return (true);
|
2013-12-04 13:49:36 +08:00
|
|
|
dss_base = chunk_dss_sbrk(0);
|
2010-01-24 18:53:40 +08:00
|
|
|
dss_prev = dss_base;
|
|
|
|
dss_max = dss_base;
|
|
|
|
|
|
|
|
return (false);
|
|
|
|
}
|
|
|
|
|
2012-03-14 07:31:41 +08:00
|
|
|
void
|
2016-05-11 13:21:10 +08:00
|
|
|
chunk_dss_prefork(tsdn_t *tsdn)
|
2012-03-14 07:31:41 +08:00
|
|
|
{
|
|
|
|
|
2014-04-16 03:09:48 +08:00
|
|
|
if (have_dss)
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_prefork(tsdn, &dss_mtx);
|
2012-03-14 07:31:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2016-05-11 13:21:10 +08:00
|
|
|
chunk_dss_postfork_parent(tsdn_t *tsdn)
|
2012-03-14 07:31:41 +08:00
|
|
|
{
|
|
|
|
|
2014-04-16 03:09:48 +08:00
|
|
|
if (have_dss)
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_postfork_parent(tsdn, &dss_mtx);
|
2012-03-14 07:31:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2016-05-11 13:21:10 +08:00
|
|
|
chunk_dss_postfork_child(tsdn_t *tsdn)
|
2012-03-14 07:31:41 +08:00
|
|
|
{
|
|
|
|
|
2014-04-16 03:09:48 +08:00
|
|
|
if (have_dss)
|
2016-05-11 13:21:10 +08:00
|
|
|
malloc_mutex_postfork_child(tsdn, &dss_mtx);
|
2012-03-14 07:31:41 +08:00
|
|
|
}
|
|
|
|
|
2010-01-24 18:53:40 +08:00
|
|
|
/******************************************************************************/
|