#define	JEMALLOC_CHUNK_C_
#include "jemalloc/internal/jemalloc_internal.h"

/******************************************************************************/
/* Data. */

const char	*opt_dss = DSS_DEFAULT;
size_t		opt_lg_chunk = 0;

/* Used exclusively for gdump triggering. */
static size_t	curchunks;
static size_t	highchunks;

rtree_t		chunks_rtree;

/* Various chunk-related settings. */
size_t		chunksize;
size_t		chunksize_mask; /* (chunksize - 1). */
size_t		chunk_npages;

static void	*chunk_alloc_default(void *new_addr, size_t size,
    size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
static bool	chunk_dalloc_default(void *chunk, size_t size, bool committed,
    unsigned arena_ind);
static bool	chunk_commit_default(void *chunk, size_t size, size_t offset,
    size_t length, unsigned arena_ind);
static bool	chunk_decommit_default(void *chunk, size_t size, size_t offset,
    size_t length, unsigned arena_ind);
static bool	chunk_purge_default(void *chunk, size_t size, size_t offset,
    size_t length, unsigned arena_ind);
static bool	chunk_split_default(void *chunk, size_t size, size_t size_a,
    size_t size_b, bool committed, unsigned arena_ind);
static bool	chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b,
    size_t size_b, bool committed, unsigned arena_ind);

const chunk_hooks_t	chunk_hooks_default = {
	chunk_alloc_default,
	chunk_dalloc_default,
	chunk_commit_default,
	chunk_decommit_default,
	chunk_purge_default,
	chunk_split_default,
	chunk_merge_default
};

/******************************************************************************/
/*
 * Function prototypes for static functions that are referenced prior to
 * definition.
 */

static void	chunk_record(tsdn_t *tsdn, arena_t *arena,
    chunk_hooks_t *chunk_hooks, extent_heap_t extent_heaps[NPSIZES], bool cache,
    void *chunk, size_t size, bool zeroed, bool committed);

/******************************************************************************/

static void
extent_heaps_insert(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
{
	size_t psz = extent_size_quantize_floor(extent_size_get(extent));
	pszind_t pind = psz2ind(psz);
	extent_heap_insert(&extent_heaps[pind], extent);
}

static void
extent_heaps_remove(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
{
	size_t psz = extent_size_quantize_floor(extent_size_get(extent));
	pszind_t pind = psz2ind(psz);
	extent_heap_remove(&extent_heaps[pind], extent);
}

static chunk_hooks_t
chunk_hooks_get_locked(arena_t *arena)
{

	return (arena->chunk_hooks);
}

chunk_hooks_t
chunk_hooks_get(tsdn_t *tsdn, arena_t *arena)
{
	chunk_hooks_t chunk_hooks;

	malloc_mutex_lock(tsdn, &arena->chunks_mtx);
	chunk_hooks = chunk_hooks_get_locked(arena);
	malloc_mutex_unlock(tsdn, &arena->chunks_mtx);

	return (chunk_hooks);
}

chunk_hooks_t
chunk_hooks_set(tsdn_t *tsdn, arena_t *arena, const chunk_hooks_t *chunk_hooks)
{
	chunk_hooks_t old_chunk_hooks;

	malloc_mutex_lock(tsdn, &arena->chunks_mtx);
	old_chunk_hooks = arena->chunk_hooks;
	/*
	 * Copy each field atomically so that it is impossible for readers to
	 * see partially updated pointers.  There are places where readers only
	 * need one hook function pointer (therefore no need to copy the
	 * entirety of arena->chunk_hooks), and stale reads do not affect
	 * correctness, so they perform unlocked reads.
	 */
#define	ATOMIC_COPY_HOOK(n) do {					\
	union {								\
		chunk_##n##_t	**n;					\
		void		**v;					\
	} u;								\
	u.n = &arena->chunk_hooks.n;					\
	atomic_write_p(u.v, chunk_hooks->n);				\
} while (0)
	ATOMIC_COPY_HOOK(alloc);
	ATOMIC_COPY_HOOK(dalloc);
	ATOMIC_COPY_HOOK(commit);
	ATOMIC_COPY_HOOK(decommit);
	ATOMIC_COPY_HOOK(purge);
	ATOMIC_COPY_HOOK(split);
	ATOMIC_COPY_HOOK(merge);
#undef ATOMIC_COPY_HOOK
	malloc_mutex_unlock(tsdn, &arena->chunks_mtx);

	return (old_chunk_hooks);
}

static void
chunk_hooks_assure_initialized_impl(tsdn_t *tsdn, arena_t *arena,
    chunk_hooks_t *chunk_hooks, bool locked)
{
	static const chunk_hooks_t uninitialized_hooks =
	    CHUNK_HOOKS_INITIALIZER;

	if (memcmp(chunk_hooks, &uninitialized_hooks, sizeof(chunk_hooks_t)) ==
	    0) {
		*chunk_hooks = locked ? chunk_hooks_get_locked(arena) :
		    chunk_hooks_get(tsdn, arena);
	}
}

static void
chunk_hooks_assure_initialized_locked(tsdn_t *tsdn, arena_t *arena,
    chunk_hooks_t *chunk_hooks)
{

	chunk_hooks_assure_initialized_impl(tsdn, arena, chunk_hooks, true);
}

static void
chunk_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena,
    chunk_hooks_t *chunk_hooks)
{

	chunk_hooks_assure_initialized_impl(tsdn, arena, chunk_hooks, false);
}

static bool
extent_rtree_acquire(tsdn_t *tsdn, const extent_t *extent, bool dependent,
    bool init_missing, rtree_elm_t **r_elm_a, rtree_elm_t **r_elm_b)
{

	*r_elm_a = rtree_elm_acquire(tsdn, &chunks_rtree,
	    (uintptr_t)extent_addr_get(extent), dependent, init_missing);
	if (!dependent && *r_elm_a == NULL)
		return (true);
	assert(*r_elm_a != NULL);

	if (extent_size_get(extent) > chunksize) {
		uintptr_t last =
		    (CHUNK_CEILING((uintptr_t)extent_past_get(extent) -
		    chunksize));

		*r_elm_b = rtree_elm_acquire(tsdn, &chunks_rtree, last,
		    dependent, init_missing);
		if (!dependent && *r_elm_b == NULL)
			return (true);
		assert(*r_elm_b != NULL);
	} else
		*r_elm_b = NULL;

	return (false);
}

static void
extent_rtree_write_acquired(tsdn_t *tsdn, rtree_elm_t *elm_a,
    rtree_elm_t *elm_b, const extent_t *extent)
{

	rtree_elm_write_acquired(tsdn, &chunks_rtree, elm_a, extent);
	if (elm_b != NULL)
		rtree_elm_write_acquired(tsdn, &chunks_rtree, elm_b, extent);
}

static void
extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b)
{

	rtree_elm_release(tsdn, &chunks_rtree, elm_a);
	if (elm_b != NULL)
		rtree_elm_release(tsdn, &chunks_rtree, elm_b);
}

bool
chunk_register(tsdn_t *tsdn, const extent_t *extent)
{
	rtree_elm_t *elm_a, *elm_b;

	if (extent_rtree_acquire(tsdn, extent, false, true, &elm_a, &elm_b))
		return (true);
	extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent);
	extent_rtree_release(tsdn, elm_a, elm_b);

	if (config_prof && opt_prof) {
		size_t nadd = (extent_size_get(extent) == 0) ? 1 :
		    extent_size_get(extent) / chunksize;
		size_t cur = atomic_add_z(&curchunks, nadd);
		size_t high = atomic_read_z(&highchunks);
		while (cur > high && atomic_cas_z(&highchunks, high, cur)) {
			/*
			 * Don't refresh cur, because it may have decreased
			 * since this thread lost the highchunks update race.
			 */
			high = atomic_read_z(&highchunks);
		}
		if (cur > high && prof_gdump_get_unlocked())
			prof_gdump(tsdn);
	}

	return (false);
}

void
chunk_deregister(tsdn_t *tsdn, const extent_t *extent)
{
	rtree_elm_t *elm_a, *elm_b;

	extent_rtree_acquire(tsdn, extent, true, false, &elm_a, &elm_b);
	extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL);
	extent_rtree_release(tsdn, elm_a, elm_b);

	if (config_prof && opt_prof) {
		size_t nsub = (extent_size_get(extent) == 0) ? 1 :
		    extent_size_get(extent) / chunksize;
		assert(atomic_read_z(&curchunks) >= nsub);
		atomic_sub_z(&curchunks, nsub);
	}
}

void
chunk_reregister(tsdn_t *tsdn, const extent_t *extent)
{
	bool err;

	err = chunk_register(tsdn, extent);
	assert(!err);
}

/*
 * Do first-best-fit chunk selection, i.e. select the lowest chunk that best
 * fits.
 */
static extent_t *
chunk_first_best_fit(arena_t *arena, extent_heap_t extent_heaps[NPSIZES],
    size_t size)
{
	pszind_t pind, i;

	assert(size == CHUNK_CEILING(size));

	pind = psz2ind(extent_size_quantize_ceil(size));
	for (i = pind; i < NPSIZES; i++) {
		extent_t *extent = extent_heap_first(&extent_heaps[i]);
		if (extent != NULL)
			return (extent);
	}

	return (NULL);
}

static void
chunk_leak(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, bool cache,
    void *addr, size_t size)
{

	/*
	 * Leak chunk after making sure its pages have already been purged, so
	 * that this is only a virtual memory leak.
	 */
	if (cache) {
		chunk_purge_wrapper(tsdn, arena, chunk_hooks, addr, size, 0,
		    size);
	}
}

static void *
chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr,
    size_t size, size_t alignment, bool *zero, bool *commit, bool dalloc_extent)
{
	void *ret;
	extent_t *extent;
	size_t alloc_size, leadsize, trailsize;
	bool zeroed, committed;

	assert(new_addr == NULL || alignment == chunksize);
	/*
	 * Cached chunks use the extent linkage embedded in their headers, in
	 * which case dalloc_extent is true, and new_addr is non-NULL because
	 * we're operating on a specific chunk.
	 */
	assert(dalloc_extent || new_addr != NULL);

	alloc_size = CHUNK_CEILING(s2u(size + alignment - chunksize));
	/* Beware size_t wrap-around. */
	if (alloc_size < size)
		return (NULL);
	malloc_mutex_lock(tsdn, &arena->chunks_mtx);
	chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks);
	if (new_addr != NULL) {
		rtree_elm_t *elm;

		elm = rtree_elm_acquire(tsdn, &chunks_rtree,
		    (uintptr_t)new_addr, false, false);
		if (elm != NULL) {
			extent = rtree_elm_read_acquired(tsdn, &chunks_rtree,
			    elm);
			if (extent != NULL && (extent_active_get(extent) ||
			    extent_retained_get(extent) == cache))
				extent = NULL;
			rtree_elm_release(tsdn, &chunks_rtree, elm);
		} else
			extent = NULL;
	} else
		extent = chunk_first_best_fit(arena, extent_heaps, alloc_size);
	if (extent == NULL || (new_addr != NULL && extent_size_get(extent) <
	    size)) {
		malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
		return (NULL);
	}
	leadsize = ALIGNMENT_CEILING((uintptr_t)extent_addr_get(extent),
	    alignment) - (uintptr_t)extent_addr_get(extent);
	assert(new_addr == NULL || leadsize == 0);
	assert(extent_size_get(extent) >= leadsize + size);
	trailsize = extent_size_get(extent) - leadsize - size;
	ret = (void *)((uintptr_t)extent_addr_get(extent) + leadsize);
	zeroed = extent_zeroed_get(extent);
	if (zeroed)
		*zero = true;
	committed = extent_committed_get(extent);
	if (committed)
		*commit = true;
	/* Split the lead. */
	if (leadsize != 0 &&
	    chunk_hooks->split(extent_addr_get(extent),
	    extent_size_get(extent), leadsize, size, false, arena->ind)) {
		malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
		return (NULL);
	}
	/* Remove extent from the heap. */
	chunk_deregister(tsdn, extent);
	extent_heaps_remove(extent_heaps, extent);
	arena_chunk_cache_maybe_remove(arena, extent, cache);
	if (leadsize != 0) {
		/* Insert the leading space as a smaller chunk. */
		extent_size_set(extent, leadsize);
		if (chunk_register(tsdn, extent)) {
			chunk_leak(tsdn, arena, chunk_hooks, cache,
			    extent_addr_get(extent), extent_size_get(extent));
			arena_extent_dalloc(tsdn, arena, extent);
		} else {
			extent_heaps_insert(extent_heaps, extent);
			arena_chunk_cache_maybe_insert(arena, extent, cache);
		}
		extent = NULL;
	}
	if (trailsize != 0) {
		/* Split the trail. */
		if (chunk_hooks->split(ret, size + trailsize, size,
		    trailsize, false, arena->ind)) {
			if (dalloc_extent && extent != NULL)
				arena_extent_dalloc(tsdn, arena, extent);
			malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
			chunk_record(tsdn, arena, chunk_hooks, extent_heaps,
			    cache, ret, size + trailsize, zeroed, committed);
			return (NULL);
		}
		/* Insert the trailing space as a smaller chunk. */
		if (extent == NULL) {
			extent = arena_extent_alloc(tsdn, arena);
			if (extent == NULL) {
				malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
				chunk_record(tsdn, arena, chunk_hooks,
				    extent_heaps, cache, ret, size + trailsize,
				    zeroed, committed);
				return (NULL);
			}
		}
		extent_init(extent, arena, (void *)((uintptr_t)(ret) + size),
		    trailsize, false, zeroed, committed, false);
		if (chunk_register(tsdn, extent)) {
			chunk_leak(tsdn, arena, chunk_hooks, cache,
			    extent_addr_get(extent), extent_size_get(extent));
			arena_extent_dalloc(tsdn, arena, extent);
		} else {
			extent_heaps_insert(extent_heaps, extent);
			arena_chunk_cache_maybe_insert(arena, extent, cache);
		}
		extent = NULL;
	}
	if (!committed && chunk_hooks->commit(ret, size, 0, size, arena->ind)) {
		malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
		chunk_record(tsdn, arena, chunk_hooks, extent_heaps, cache, ret,
		    size, zeroed, committed);
		return (NULL);
	}
	malloc_mutex_unlock(tsdn, &arena->chunks_mtx);

	assert(dalloc_extent || extent != NULL);
	if (dalloc_extent && extent != NULL)
		arena_extent_dalloc(tsdn, arena, extent);
	if (*zero) {
		if (!zeroed)
			memset(ret, 0, size);
		else if (config_debug) {
			size_t i;
			size_t *p = (size_t *)(uintptr_t)ret;

			for (i = 0; i < size / sizeof(size_t); i++)
				assert(p[i] == 0);
		}
	}
	return (ret);
}

/*
 * If the caller specifies (!*zero), it is still possible to receive zeroed
 * memory, in which case *zero is toggled to true.  arena_chunk_alloc() takes
 * advantage of this to avoid demanding zeroed chunks, but taking advantage of
 * them if they are returned.
 */
static void *
chunk_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
    size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec)
{
	void *ret;

	assert(size != 0);
	assert((size & chunksize_mask) == 0);
	assert(alignment != 0);
	assert((alignment & chunksize_mask) == 0);

	/* "primary" dss. */
	if (have_dss && dss_prec == dss_prec_primary && (ret =
	    chunk_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
	    commit)) != NULL)
		return (ret);
	/* mmap. */
	if ((ret = chunk_alloc_mmap(new_addr, size, alignment, zero, commit)) !=
	    NULL)
		return (ret);
	/* "secondary" dss. */
	if (have_dss && dss_prec == dss_prec_secondary && (ret =
	    chunk_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
	    commit)) != NULL)
		return (ret);

	/* All strategies for allocation failed. */
	return (NULL);
}

void *
chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *new_addr, size_t size, size_t alignment, bool *zero,
    bool dalloc_extent)
{
	void *ret;
	bool commit;

	assert(size != 0);
	assert((size & chunksize_mask) == 0);
	assert(alignment != 0);
	assert((alignment & chunksize_mask) == 0);

	commit = true;
	ret = chunk_recycle(tsdn, arena, chunk_hooks, arena->chunks_cached,
	    true, new_addr, size, alignment, zero, &commit, dalloc_extent);
	if (ret == NULL)
		return (NULL);
	assert(commit);
	return (ret);
}

static arena_t *
chunk_arena_get(tsdn_t *tsdn, unsigned arena_ind)
{
	arena_t *arena;

	arena = arena_get(tsdn, arena_ind, false);
	/*
	 * The arena we're allocating on behalf of must have been initialized
	 * already.
	 */
	assert(arena != NULL);
	return (arena);
}

static void *
chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
    bool *commit, unsigned arena_ind)
{
	void *ret;
	tsdn_t *tsdn;
	arena_t *arena;

	tsdn = tsdn_fetch();
	arena = chunk_arena_get(tsdn, arena_ind);
	ret = chunk_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
	    commit, arena->dss_prec);
	if (ret == NULL)
		return (NULL);

	return (ret);
}

static void *
chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit)
{
	void *ret;

	assert(size != 0);
	assert((size & chunksize_mask) == 0);
	assert(alignment != 0);
	assert((alignment & chunksize_mask) == 0);

	ret = chunk_recycle(tsdn, arena, chunk_hooks, arena->chunks_retained,
	    false, new_addr, size, alignment, zero, commit, true);

	if (config_stats && ret != NULL)
		arena->stats.retained -= size;

	return (ret);
}

void *
chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit)
{
	void *ret;

	chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);

	ret = chunk_alloc_retained(tsdn, arena, chunk_hooks, new_addr, size,
	    alignment, zero, commit);
	if (ret == NULL) {
		ret = chunk_hooks->alloc(new_addr, size, alignment, zero,
		    commit, arena->ind);
		if (ret == NULL)
			return (NULL);
	}

	return (ret);
}

static bool
chunk_can_coalesce(const extent_t *a, const extent_t *b)
{

	assert((void *)CHUNK_CEILING((uintptr_t)extent_past_get(a)) ==
	    extent_addr_get(b));

	if (extent_arena_get(a) != extent_arena_get(b))
		return (false);
	if (extent_active_get(a) != extent_active_get(b))
		return (false);
	if (extent_committed_get(a) != extent_committed_get(b))
		return (false);
	if (extent_retained_get(a) != extent_retained_get(b))
		return (false);

	return (true);
}

static void
chunk_try_coalesce(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    extent_t *a, extent_t *b, extent_heap_t extent_heaps[NPSIZES], bool cache)
{
	rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;

	if (!chunk_can_coalesce(a, b))
		return;

	if (chunk_hooks->merge(extent_addr_get(a), extent_size_get(a),
	    extent_addr_get(b), extent_size_get(b), extent_committed_get(a),
	    arena->ind))
		return;

	/*
	 * The rtree writes must happen while all the relevant elements are
	 * owned, so the following code uses decomposed helper functions rather
	 * than chunk_{,de}register() to do things in the right order.
	 */
	extent_rtree_acquire(tsdn, a, true, false, &a_elm_a, &a_elm_b);
	extent_rtree_acquire(tsdn, b, true, false, &b_elm_a, &b_elm_b);

	if (a_elm_b != NULL) {
		rtree_elm_write_acquired(tsdn, &chunks_rtree, a_elm_b, NULL);
		rtree_elm_release(tsdn, &chunks_rtree, a_elm_b);
	}
	if (b_elm_b != NULL) {
		rtree_elm_write_acquired(tsdn, &chunks_rtree, b_elm_a, NULL);
		rtree_elm_release(tsdn, &chunks_rtree, b_elm_a);
	} else
		b_elm_b = b_elm_a;

	extent_heaps_remove(extent_heaps, a);
	extent_heaps_remove(extent_heaps, b);

	arena_chunk_cache_maybe_remove(extent_arena_get(a), a, cache);
	arena_chunk_cache_maybe_remove(extent_arena_get(b), b, cache);

	extent_size_set(a, extent_size_get(a) + extent_size_get(b));
	extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b));

	extent_heaps_insert(extent_heaps, a);

	extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a);
	extent_rtree_release(tsdn, a_elm_a, b_elm_b);

	arena_chunk_cache_maybe_insert(extent_arena_get(a), a, cache);

	arena_extent_dalloc(tsdn, extent_arena_get(b), b);
}

static void
chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    extent_heap_t extent_heaps[NPSIZES], bool cache, void *chunk, size_t size,
    bool zeroed, bool committed)
{
	extent_t *extent, *prev, *next;

	assert(!cache || !zeroed);

	malloc_mutex_lock(tsdn, &arena->chunks_mtx);
	chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks);

	/* Create/initialize/insert extent. */
	extent = arena_extent_alloc(tsdn, arena);
	if (extent == NULL) {
		chunk_leak(tsdn, arena, chunk_hooks, cache, chunk, size);
		goto label_return;
	}
	extent_init(extent, arena, chunk, size, false, !cache && zeroed,
	    committed, false);
	if (chunk_register(tsdn, extent)) {
		arena_extent_dalloc(tsdn, arena, extent);
		chunk_leak(tsdn, arena, chunk_hooks, cache, chunk, size);
		goto label_return;
	}
	extent_heaps_insert(extent_heaps, extent);
	arena_chunk_cache_maybe_insert(arena, extent, cache);

	/* Try to coalesce forward. */
	next = rtree_read(tsdn, &chunks_rtree,
	    CHUNK_CEILING((uintptr_t)extent_past_get(extent)), false);
	if (next != NULL) {
		chunk_try_coalesce(tsdn, arena, chunk_hooks, extent, next,
		    extent_heaps, cache);
	}

	/* Try to coalesce backward. */
	prev = rtree_read(tsdn, &chunks_rtree,
	    (uintptr_t)extent_addr_get(extent) - chunksize, false);
	if (prev != NULL) {
		chunk_try_coalesce(tsdn, arena, chunk_hooks, prev, extent,
		    extent_heaps, cache);
	}

label_return:
	malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
}

void
chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *chunk, size_t size, bool committed)
{

	assert(chunk != NULL);
	assert(CHUNK_ADDR2BASE(chunk) == chunk);
	assert(size != 0);
	assert((size & chunksize_mask) == 0);

	chunk_record(tsdn, arena, chunk_hooks, arena->chunks_cached, true,
	    chunk, size, false, committed);
	arena_maybe_purge(tsdn, arena);
}

static bool
chunk_dalloc_default(void *chunk, size_t size, bool committed,
    unsigned arena_ind)
{

	if (!have_dss || !chunk_in_dss(tsdn_fetch(), chunk))
		return (chunk_dalloc_mmap(chunk, size));
	return (true);
}

void
chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *chunk, size_t size, bool zeroed, bool committed)
{

	assert(chunk != NULL);
	assert(CHUNK_ADDR2BASE(chunk) == chunk);
	assert(size != 0);
	assert((size & chunksize_mask) == 0);

	chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);
	/* Try to deallocate. */
	if (!chunk_hooks->dalloc(chunk, size, committed, arena->ind))
		return;
	/* Try to decommit; purge if that fails. */
	if (committed) {
		committed = chunk_hooks->decommit(chunk, size, 0, size,
		    arena->ind);
	}
	zeroed = !committed || !chunk_hooks->purge(chunk, size, 0, size,
	    arena->ind);
	chunk_record(tsdn, arena, chunk_hooks, arena->chunks_retained, false,
	    chunk, size, zeroed, committed);

	if (config_stats)
		arena->stats.retained += size;
}

static bool
chunk_commit_default(void *chunk, size_t size, size_t offset, size_t length,
    unsigned arena_ind)
{

	return (pages_commit((void *)((uintptr_t)chunk + (uintptr_t)offset),
	    length));
}

bool
chunk_commit_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *chunk, size_t size, size_t offset, size_t length)
{

	chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);
	return (chunk_hooks->commit(chunk, size, offset, length, arena->ind));
}

static bool
chunk_decommit_default(void *chunk, size_t size, size_t offset, size_t length,
    unsigned arena_ind)
{

	return (pages_decommit((void *)((uintptr_t)chunk + (uintptr_t)offset),
	    length));
}

bool
chunk_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *chunk, size_t size, size_t offset, size_t length)
{

	chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);
	return (chunk_hooks->decommit(chunk, size, offset, length, arena->ind));
}

static bool
chunk_purge_default(void *chunk, size_t size, size_t offset, size_t length,
    unsigned arena_ind)
{

	assert(chunk != NULL);
	assert(CHUNK_ADDR2BASE(chunk) == chunk);
	assert((offset & PAGE_MASK) == 0);
	assert(length != 0);
	assert((length & PAGE_MASK) == 0);

	return (pages_purge((void *)((uintptr_t)chunk + (uintptr_t)offset),
	    length));
}

bool
chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *chunk, size_t size, size_t offset, size_t length)
{

	chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);
	return (chunk_hooks->purge(chunk, size, offset, length, arena->ind));
}

static bool
chunk_split_default(void *chunk, size_t size, size_t size_a, size_t size_b,
    bool committed, unsigned arena_ind)
{

	if (!maps_coalesce)
		return (true);
	return (false);
}

static bool
chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
    bool committed, unsigned arena_ind)
{

	if (!maps_coalesce)
		return (true);
	if (have_dss) {
		tsdn_t *tsdn = tsdn_fetch();
		if (chunk_in_dss(tsdn, chunk_a) != chunk_in_dss(tsdn, chunk_b))
			return (true);
	}

	return (false);
}

bool
chunk_boot(void)
{
#ifdef _WIN32
	SYSTEM_INFO info;
	GetSystemInfo(&info);

	/*
	 * Verify actual page size is equal to or an integral multiple of
	 * configured page size.
	 */
	if (info.dwPageSize & ((1U << LG_PAGE) - 1))
		return (true);

	/*
	 * Configure chunksize (if not set) to match granularity (usually 64K),
	 * so pages_map will always take fast path.
	 */
	if (!opt_lg_chunk) {
		opt_lg_chunk = ffs_u((unsigned)info.dwAllocationGranularity)
		    - 1;
	}
#else
	if (!opt_lg_chunk)
		opt_lg_chunk = LG_CHUNK_DEFAULT;
#endif

	/* Set variables according to the value of opt_lg_chunk. */
	chunksize = (ZU(1) << opt_lg_chunk);
	assert(chunksize >= PAGE);
	chunksize_mask = chunksize - 1;
	chunk_npages = (chunksize >> LG_PAGE);

	if (have_dss && chunk_dss_boot())
		return (true);
	if (rtree_new(&chunks_rtree, (unsigned)((ZU(1) << (LG_SIZEOF_PTR+3)) -
	    opt_lg_chunk)))
		return (true);

	return (false);
}

void
chunk_prefork(tsdn_t *tsdn)
{

	chunk_dss_prefork(tsdn);
}

void
chunk_postfork_parent(tsdn_t *tsdn)
{

	chunk_dss_postfork_parent(tsdn);
}

void
chunk_postfork_child(tsdn_t *tsdn)
{

	chunk_dss_postfork_child(tsdn);
}