7372b15a31
Convert configuration-related cpp conditional logic to use static constant variables, e.g.: #ifdef JEMALLOC_DEBUG [...] #endif becomes: if (config_debug) { [...] } The advantage is clearer, more concise code. The main disadvantage is that data structures no longer have conditionally defined fields, so they pay the cost of all fields regardless of whether they are used. In practice, this is only a minor concern; config_stats will go away in an upcoming change, and config_prof is the only other major feature that depends on more than a few special-purpose fields.
365 lines
8.2 KiB
C
365 lines
8.2 KiB
C
#define JEMALLOC_HUGE_C_
|
|
#include "jemalloc/internal/jemalloc_internal.h"
|
|
|
|
/******************************************************************************/
|
|
/* Data. */
|
|
|
|
uint64_t huge_nmalloc;
|
|
uint64_t huge_ndalloc;
|
|
size_t huge_allocated;
|
|
|
|
malloc_mutex_t huge_mtx;
|
|
|
|
/******************************************************************************/
|
|
|
|
/* Tree of chunks that are stand-alone huge allocations. */
|
|
static extent_tree_t huge;
|
|
|
|
void *
|
|
huge_malloc(size_t size, bool zero)
|
|
{
|
|
void *ret;
|
|
size_t csize;
|
|
extent_node_t *node;
|
|
|
|
/* Allocate one or more contiguous chunks for this request. */
|
|
|
|
csize = CHUNK_CEILING(size);
|
|
if (csize == 0) {
|
|
/* size is large enough to cause size_t wrap-around. */
|
|
return (NULL);
|
|
}
|
|
|
|
/* Allocate an extent node with which to track the chunk. */
|
|
node = base_node_alloc();
|
|
if (node == NULL)
|
|
return (NULL);
|
|
|
|
ret = chunk_alloc(csize, false, &zero);
|
|
if (ret == NULL) {
|
|
base_node_dealloc(node);
|
|
return (NULL);
|
|
}
|
|
|
|
/* Insert node into huge. */
|
|
node->addr = ret;
|
|
node->size = csize;
|
|
|
|
malloc_mutex_lock(&huge_mtx);
|
|
extent_tree_ad_insert(&huge, node);
|
|
if (config_stats) {
|
|
stats_cactive_add(csize);
|
|
huge_nmalloc++;
|
|
huge_allocated += csize;
|
|
}
|
|
malloc_mutex_unlock(&huge_mtx);
|
|
|
|
if (config_fill && zero == false) {
|
|
if (opt_junk)
|
|
memset(ret, 0xa5, csize);
|
|
else if (opt_zero)
|
|
memset(ret, 0, csize);
|
|
}
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/* Only handles large allocations that require more than chunk alignment. */
|
|
void *
|
|
huge_palloc(size_t size, size_t alignment, bool zero)
|
|
{
|
|
void *ret;
|
|
size_t alloc_size, chunk_size, offset;
|
|
extent_node_t *node;
|
|
|
|
/*
|
|
* This allocation requires alignment that is even larger than chunk
|
|
* alignment. This means that huge_malloc() isn't good enough.
|
|
*
|
|
* Allocate almost twice as many chunks as are demanded by the size or
|
|
* alignment, in order to assure the alignment can be achieved, then
|
|
* unmap leading and trailing chunks.
|
|
*/
|
|
assert(alignment > chunksize);
|
|
|
|
chunk_size = CHUNK_CEILING(size);
|
|
|
|
if (size >= alignment)
|
|
alloc_size = chunk_size + alignment - chunksize;
|
|
else
|
|
alloc_size = (alignment << 1) - chunksize;
|
|
|
|
/* Allocate an extent node with which to track the chunk. */
|
|
node = base_node_alloc();
|
|
if (node == NULL)
|
|
return (NULL);
|
|
|
|
ret = chunk_alloc(alloc_size, false, &zero);
|
|
if (ret == NULL) {
|
|
base_node_dealloc(node);
|
|
return (NULL);
|
|
}
|
|
|
|
offset = (uintptr_t)ret & (alignment - 1);
|
|
assert((offset & chunksize_mask) == 0);
|
|
assert(offset < alloc_size);
|
|
if (offset == 0) {
|
|
/* Trim trailing space. */
|
|
chunk_dealloc((void *)((uintptr_t)ret + chunk_size), alloc_size
|
|
- chunk_size, true);
|
|
} else {
|
|
size_t trailsize;
|
|
|
|
/* Trim leading space. */
|
|
chunk_dealloc(ret, alignment - offset, true);
|
|
|
|
ret = (void *)((uintptr_t)ret + (alignment - offset));
|
|
|
|
trailsize = alloc_size - (alignment - offset) - chunk_size;
|
|
if (trailsize != 0) {
|
|
/* Trim trailing space. */
|
|
assert(trailsize < alloc_size);
|
|
chunk_dealloc((void *)((uintptr_t)ret + chunk_size),
|
|
trailsize, true);
|
|
}
|
|
}
|
|
|
|
/* Insert node into huge. */
|
|
node->addr = ret;
|
|
node->size = chunk_size;
|
|
|
|
malloc_mutex_lock(&huge_mtx);
|
|
extent_tree_ad_insert(&huge, node);
|
|
if (config_stats) {
|
|
stats_cactive_add(chunk_size);
|
|
huge_nmalloc++;
|
|
huge_allocated += chunk_size;
|
|
}
|
|
malloc_mutex_unlock(&huge_mtx);
|
|
|
|
if (config_fill && zero == false) {
|
|
if (opt_junk)
|
|
memset(ret, 0xa5, chunk_size);
|
|
else if (opt_zero)
|
|
memset(ret, 0, chunk_size);
|
|
}
|
|
|
|
return (ret);
|
|
}
|
|
|
|
void *
|
|
huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
|
|
{
|
|
|
|
/*
|
|
* Avoid moving the allocation if the size class can be left the same.
|
|
*/
|
|
if (oldsize > arena_maxclass
|
|
&& CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
|
|
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
|
|
assert(CHUNK_CEILING(oldsize) == oldsize);
|
|
if (config_fill && opt_junk && size < oldsize) {
|
|
memset((void *)((uintptr_t)ptr + size), 0x5a,
|
|
oldsize - size);
|
|
}
|
|
return (ptr);
|
|
}
|
|
|
|
/* Reallocation would require a move. */
|
|
return (NULL);
|
|
}
|
|
|
|
void *
|
|
huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
|
|
size_t alignment, bool zero)
|
|
{
|
|
void *ret;
|
|
size_t copysize;
|
|
|
|
/* Try to avoid moving the allocation. */
|
|
ret = huge_ralloc_no_move(ptr, oldsize, size, extra);
|
|
if (ret != NULL)
|
|
return (ret);
|
|
|
|
/*
|
|
* size and oldsize are different enough that we need to use a
|
|
* different size class. In that case, fall back to allocating new
|
|
* space and copying.
|
|
*/
|
|
if (alignment > chunksize)
|
|
ret = huge_palloc(size + extra, alignment, zero);
|
|
else
|
|
ret = huge_malloc(size + extra, zero);
|
|
|
|
if (ret == NULL) {
|
|
if (extra == 0)
|
|
return (NULL);
|
|
/* Try again, this time without extra. */
|
|
if (alignment > chunksize)
|
|
ret = huge_palloc(size, alignment, zero);
|
|
else
|
|
ret = huge_malloc(size, zero);
|
|
|
|
if (ret == NULL)
|
|
return (NULL);
|
|
}
|
|
|
|
/*
|
|
* Copy at most size bytes (not size+extra), since the caller has no
|
|
* expectation that the extra bytes will be reliably preserved.
|
|
*/
|
|
copysize = (size < oldsize) ? size : oldsize;
|
|
|
|
/*
|
|
* Use mremap(2) if this is a huge-->huge reallocation, and neither the
|
|
* source nor the destination are in swap or dss.
|
|
*/
|
|
#ifdef JEMALLOC_MREMAP_FIXED
|
|
if (oldsize >= chunksize && (config_swap == false || swap_enabled ==
|
|
false || (chunk_in_swap(ptr) == false && chunk_in_swap(ret) ==
|
|
false)) && (config_dss == false || (chunk_in_dss(ptr) == false &&
|
|
chunk_in_dss(ret) == false))) {
|
|
size_t newsize = huge_salloc(ret);
|
|
|
|
/*
|
|
* Remove ptr from the tree of huge allocations before
|
|
* performing the remap operation, in order to avoid the
|
|
* possibility of another thread acquiring that mapping before
|
|
* this one removes it from the tree.
|
|
*/
|
|
huge_dalloc(ptr, false);
|
|
if (mremap(ptr, oldsize, newsize, MREMAP_MAYMOVE|MREMAP_FIXED,
|
|
ret) == MAP_FAILED) {
|
|
/*
|
|
* Assuming no chunk management bugs in the allocator,
|
|
* the only documented way an error can occur here is
|
|
* if the application changed the map type for a
|
|
* portion of the old allocation. This is firmly in
|
|
* undefined behavior territory, so write a diagnostic
|
|
* message, and optionally abort.
|
|
*/
|
|
char buf[BUFERROR_BUF];
|
|
|
|
buferror(errno, buf, sizeof(buf));
|
|
malloc_write("<jemalloc>: Error in mremap(): ");
|
|
malloc_write(buf);
|
|
malloc_write("\n");
|
|
if (opt_abort)
|
|
abort();
|
|
memcpy(ret, ptr, copysize);
|
|
chunk_dealloc_mmap(ptr, oldsize);
|
|
}
|
|
} else
|
|
#endif
|
|
{
|
|
memcpy(ret, ptr, copysize);
|
|
idalloc(ptr);
|
|
}
|
|
return (ret);
|
|
}
|
|
|
|
void
|
|
huge_dalloc(void *ptr, bool unmap)
|
|
{
|
|
extent_node_t *node, key;
|
|
|
|
malloc_mutex_lock(&huge_mtx);
|
|
|
|
/* Extract from tree of huge allocations. */
|
|
key.addr = ptr;
|
|
node = extent_tree_ad_search(&huge, &key);
|
|
assert(node != NULL);
|
|
assert(node->addr == ptr);
|
|
extent_tree_ad_remove(&huge, node);
|
|
|
|
if (config_stats) {
|
|
stats_cactive_sub(node->size);
|
|
huge_ndalloc++;
|
|
huge_allocated -= node->size;
|
|
}
|
|
|
|
malloc_mutex_unlock(&huge_mtx);
|
|
|
|
if (unmap && config_fill && (config_swap || config_dss) && opt_junk)
|
|
memset(node->addr, 0x5a, node->size);
|
|
|
|
chunk_dealloc(node->addr, node->size, unmap);
|
|
|
|
base_node_dealloc(node);
|
|
}
|
|
|
|
size_t
|
|
huge_salloc(const void *ptr)
|
|
{
|
|
size_t ret;
|
|
extent_node_t *node, key;
|
|
|
|
malloc_mutex_lock(&huge_mtx);
|
|
|
|
/* Extract from tree of huge allocations. */
|
|
key.addr = __DECONST(void *, ptr);
|
|
node = extent_tree_ad_search(&huge, &key);
|
|
assert(node != NULL);
|
|
|
|
ret = node->size;
|
|
|
|
malloc_mutex_unlock(&huge_mtx);
|
|
|
|
return (ret);
|
|
}
|
|
|
|
prof_ctx_t *
|
|
huge_prof_ctx_get(const void *ptr)
|
|
{
|
|
prof_ctx_t *ret;
|
|
extent_node_t *node, key;
|
|
|
|
malloc_mutex_lock(&huge_mtx);
|
|
|
|
/* Extract from tree of huge allocations. */
|
|
key.addr = __DECONST(void *, ptr);
|
|
node = extent_tree_ad_search(&huge, &key);
|
|
assert(node != NULL);
|
|
|
|
ret = node->prof_ctx;
|
|
|
|
malloc_mutex_unlock(&huge_mtx);
|
|
|
|
return (ret);
|
|
}
|
|
|
|
void
|
|
huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
|
|
{
|
|
extent_node_t *node, key;
|
|
|
|
malloc_mutex_lock(&huge_mtx);
|
|
|
|
/* Extract from tree of huge allocations. */
|
|
key.addr = __DECONST(void *, ptr);
|
|
node = extent_tree_ad_search(&huge, &key);
|
|
assert(node != NULL);
|
|
|
|
node->prof_ctx = ctx;
|
|
|
|
malloc_mutex_unlock(&huge_mtx);
|
|
}
|
|
|
|
bool
|
|
huge_boot(void)
|
|
{
|
|
|
|
/* Initialize chunks data. */
|
|
if (malloc_mutex_init(&huge_mtx))
|
|
return (true);
|
|
extent_tree_ad_new(&huge);
|
|
|
|
if (config_stats) {
|
|
huge_nmalloc = 0;
|
|
huge_ndalloc = 0;
|
|
huge_allocated = 0;
|
|
}
|
|
|
|
return (false);
|
|
}
|