2010-01-17 01:53:50 +08:00
|
|
|
#define JEMALLOC_BASE_C_
|
2010-02-12 06:45:59 +08:00
|
|
|
#include "jemalloc/internal/jemalloc_internal.h"
|
2010-01-17 01:53:50 +08:00
|
|
|
|
2010-01-24 18:53:40 +08:00
|
|
|
/******************************************************************************/
|
|
|
|
/* Data. */
|
2010-01-17 01:53:50 +08:00
|
|
|
|
2012-03-14 07:31:41 +08:00
|
|
|
static malloc_mutex_t base_mtx;
|
2015-01-31 13:49:19 +08:00
|
|
|
static extent_tree_t base_avail_szad;
|
2010-01-17 01:53:50 +08:00
|
|
|
static extent_node_t *base_nodes;
|
2014-11-28 03:22:36 +08:00
|
|
|
static size_t base_allocated;
|
|
|
|
|
2010-01-24 18:53:40 +08:00
|
|
|
/******************************************************************************/
|
2010-01-17 01:53:50 +08:00
|
|
|
|
2015-01-31 13:49:19 +08:00
|
|
|
static extent_node_t *
|
|
|
|
base_node_try_alloc_locked(void)
|
2010-01-17 01:53:50 +08:00
|
|
|
{
|
2015-01-31 13:49:19 +08:00
|
|
|
extent_node_t *node;
|
2010-01-17 01:53:50 +08:00
|
|
|
|
2015-01-31 13:49:19 +08:00
|
|
|
if (base_nodes == NULL)
|
|
|
|
return (NULL);
|
|
|
|
node = base_nodes;
|
|
|
|
base_nodes = *(extent_node_t **)node;
|
|
|
|
JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
|
|
|
|
return (node);
|
|
|
|
}
|
2010-01-17 01:53:50 +08:00
|
|
|
|
2015-01-31 13:49:19 +08:00
|
|
|
static void
|
|
|
|
base_node_dalloc_locked(extent_node_t *node)
|
|
|
|
{
|
|
|
|
|
|
|
|
JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
|
|
|
|
*(extent_node_t **)node = base_nodes;
|
|
|
|
base_nodes = node;
|
2010-01-17 01:53:50 +08:00
|
|
|
}
|
|
|
|
|
2015-01-31 13:49:19 +08:00
|
|
|
/* base_mtx must be held. */
|
|
|
|
static extent_node_t *
|
|
|
|
base_chunk_alloc(size_t minsize)
|
|
|
|
{
|
|
|
|
extent_node_t *node;
|
|
|
|
size_t csize, nsize;
|
|
|
|
void *addr;
|
|
|
|
|
|
|
|
assert(minsize != 0);
|
|
|
|
node = base_node_try_alloc_locked();
|
|
|
|
/* Allocate enough space to also carve a node out if necessary. */
|
|
|
|
nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0;
|
|
|
|
csize = CHUNK_CEILING(minsize + nsize);
|
|
|
|
addr = chunk_alloc_base(csize);
|
|
|
|
if (addr == NULL) {
|
|
|
|
if (node != NULL)
|
|
|
|
base_node_dalloc_locked(node);
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
if (node == NULL) {
|
|
|
|
csize -= nsize;
|
|
|
|
node = (extent_node_t *)((uintptr_t)addr + csize);
|
|
|
|
if (config_stats)
|
|
|
|
base_allocated += nsize;
|
|
|
|
}
|
|
|
|
node->addr = addr;
|
|
|
|
node->size = csize;
|
|
|
|
return (node);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *
|
|
|
|
base_alloc_locked(size_t size)
|
2010-01-17 01:53:50 +08:00
|
|
|
{
|
|
|
|
void *ret;
|
|
|
|
size_t csize;
|
2015-01-31 13:49:19 +08:00
|
|
|
extent_node_t *node;
|
|
|
|
extent_node_t key;
|
2010-01-17 01:53:50 +08:00
|
|
|
|
2015-01-31 13:49:19 +08:00
|
|
|
/*
|
|
|
|
* Round size up to nearest multiple of the cacheline size, so that
|
|
|
|
* there is no chance of false cache line sharing.
|
|
|
|
*/
|
2010-01-17 01:53:50 +08:00
|
|
|
csize = CACHELINE_CEILING(size);
|
|
|
|
|
2015-01-31 13:49:19 +08:00
|
|
|
key.addr = NULL;
|
|
|
|
key.size = csize;
|
|
|
|
node = extent_tree_szad_nsearch(&base_avail_szad, &key);
|
|
|
|
if (node != NULL) {
|
|
|
|
/* Use existing space. */
|
|
|
|
extent_tree_szad_remove(&base_avail_szad, node);
|
|
|
|
} else {
|
|
|
|
/* Try to allocate more space. */
|
|
|
|
node = base_chunk_alloc(csize);
|
2010-01-17 01:53:50 +08:00
|
|
|
}
|
2015-01-31 13:49:19 +08:00
|
|
|
if (node == NULL)
|
|
|
|
return (NULL);
|
|
|
|
|
|
|
|
ret = node->addr;
|
|
|
|
if (node->size > csize) {
|
|
|
|
node->addr = (void *)((uintptr_t)ret + csize);
|
|
|
|
node->size -= csize;
|
|
|
|
extent_tree_szad_insert(&base_avail_szad, node);
|
|
|
|
} else
|
|
|
|
base_node_dalloc_locked(node);
|
2014-11-28 03:22:36 +08:00
|
|
|
if (config_stats)
|
|
|
|
base_allocated += csize;
|
2014-04-16 07:35:08 +08:00
|
|
|
JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize);
|
2010-01-17 01:53:50 +08:00
|
|
|
return (ret);
|
|
|
|
}
|
|
|
|
|
2015-01-31 13:49:19 +08:00
|
|
|
/*
|
|
|
|
* base_alloc() guarantees demand-zeroed memory, in order to make multi-page
|
|
|
|
* sparse data structures such as radix tree nodes efficient with respect to
|
|
|
|
* physical memory usage.
|
|
|
|
*/
|
2012-02-03 14:04:57 +08:00
|
|
|
void *
|
2015-01-31 13:49:19 +08:00
|
|
|
base_alloc(size_t size)
|
2012-02-03 14:04:57 +08:00
|
|
|
{
|
2015-01-31 13:49:19 +08:00
|
|
|
void *ret;
|
2012-02-03 14:04:57 +08:00
|
|
|
|
2015-01-31 13:49:19 +08:00
|
|
|
malloc_mutex_lock(&base_mtx);
|
|
|
|
ret = base_alloc_locked(size);
|
|
|
|
malloc_mutex_unlock(&base_mtx);
|
2012-02-03 14:04:57 +08:00
|
|
|
return (ret);
|
|
|
|
}
|
|
|
|
|
2010-01-17 01:53:50 +08:00
|
|
|
extent_node_t *
|
|
|
|
base_node_alloc(void)
|
|
|
|
{
|
|
|
|
extent_node_t *ret;
|
|
|
|
|
|
|
|
malloc_mutex_lock(&base_mtx);
|
2015-01-31 13:49:19 +08:00
|
|
|
if ((ret = base_node_try_alloc_locked()) == NULL)
|
|
|
|
ret = (extent_node_t *)base_alloc_locked(sizeof(extent_node_t));
|
|
|
|
malloc_mutex_unlock(&base_mtx);
|
2010-01-17 01:53:50 +08:00
|
|
|
return (ret);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2014-05-16 13:22:27 +08:00
|
|
|
base_node_dalloc(extent_node_t *node)
|
2010-01-17 01:53:50 +08:00
|
|
|
{
|
|
|
|
|
|
|
|
malloc_mutex_lock(&base_mtx);
|
2015-01-31 13:49:19 +08:00
|
|
|
base_node_dalloc_locked(node);
|
2010-01-17 01:53:50 +08:00
|
|
|
malloc_mutex_unlock(&base_mtx);
|
|
|
|
}
|
|
|
|
|
2014-11-28 03:22:36 +08:00
|
|
|
size_t
|
|
|
|
base_allocated_get(void)
|
|
|
|
{
|
|
|
|
size_t ret;
|
|
|
|
|
|
|
|
malloc_mutex_lock(&base_mtx);
|
|
|
|
ret = base_allocated;
|
|
|
|
malloc_mutex_unlock(&base_mtx);
|
|
|
|
return (ret);
|
|
|
|
}
|
|
|
|
|
2010-01-17 01:53:50 +08:00
|
|
|
bool
|
|
|
|
base_boot(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
if (malloc_mutex_init(&base_mtx))
|
|
|
|
return (true);
|
2015-01-31 13:49:19 +08:00
|
|
|
extent_tree_szad_new(&base_avail_szad);
|
|
|
|
base_nodes = NULL;
|
2010-01-17 01:53:50 +08:00
|
|
|
|
|
|
|
return (false);
|
|
|
|
}
|
2012-03-14 07:31:41 +08:00
|
|
|
|
|
|
|
void
|
|
|
|
base_prefork(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
malloc_mutex_prefork(&base_mtx);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
base_postfork_parent(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
malloc_mutex_postfork_parent(&base_mtx);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
base_postfork_child(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
malloc_mutex_postfork_child(&base_mtx);
|
|
|
|
}
|