Rename huge to large.
This commit is contained in:
parent
714d1640f3
commit
7d63fed0fd
@ -88,7 +88,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
|
|||||||
$(srcroot)src/ctl.c \
|
$(srcroot)src/ctl.c \
|
||||||
$(srcroot)src/extent.c \
|
$(srcroot)src/extent.c \
|
||||||
$(srcroot)src/hash.c \
|
$(srcroot)src/hash.c \
|
||||||
$(srcroot)src/huge.c \
|
$(srcroot)src/large.c \
|
||||||
$(srcroot)src/mb.c \
|
$(srcroot)src/mb.c \
|
||||||
$(srcroot)src/mutex.c \
|
$(srcroot)src/mutex.c \
|
||||||
$(srcroot)src/nstime.c \
|
$(srcroot)src/nstime.c \
|
||||||
|
@ -1872,22 +1872,22 @@ typedef struct {
|
|||||||
<listitem><para>Number of bytes per slab.</para></listitem>
|
<listitem><para>Number of bytes per slab.</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry id="arenas.nhchunks">
|
<varlistentry id="arenas.nlextents">
|
||||||
<term>
|
<term>
|
||||||
<mallctl>arenas.nhchunks</mallctl>
|
<mallctl>arenas.nlextents</mallctl>
|
||||||
(<type>unsigned</type>)
|
(<type>unsigned</type>)
|
||||||
<literal>r-</literal>
|
<literal>r-</literal>
|
||||||
</term>
|
</term>
|
||||||
<listitem><para>Total number of huge size classes.</para></listitem>
|
<listitem><para>Total number of large size classes.</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry id="arenas.hchunk.i.size">
|
<varlistentry id="arenas.lextent.i.size">
|
||||||
<term>
|
<term>
|
||||||
<mallctl>arenas.hchunk.<i>.size</mallctl>
|
<mallctl>arenas.lextent.<i>.size</mallctl>
|
||||||
(<type>size_t</type>)
|
(<type>size_t</type>)
|
||||||
<literal>r-</literal>
|
<literal>r-</literal>
|
||||||
</term>
|
</term>
|
||||||
<listitem><para>Maximum size supported by this huge size
|
<listitem><para>Maximum size supported by this large size
|
||||||
class.</para></listitem>
|
class.</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
@ -2361,50 +2361,6 @@ typedef struct {
|
|||||||
</para></listitem>
|
</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.huge.allocated">
|
|
||||||
<term>
|
|
||||||
<mallctl>stats.arenas.<i>.huge.allocated</mallctl>
|
|
||||||
(<type>size_t</type>)
|
|
||||||
<literal>r-</literal>
|
|
||||||
[<option>--enable-stats</option>]
|
|
||||||
</term>
|
|
||||||
<listitem><para>Number of bytes currently allocated by huge objects.
|
|
||||||
</para></listitem>
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.huge.nmalloc">
|
|
||||||
<term>
|
|
||||||
<mallctl>stats.arenas.<i>.huge.nmalloc</mallctl>
|
|
||||||
(<type>uint64_t</type>)
|
|
||||||
<literal>r-</literal>
|
|
||||||
[<option>--enable-stats</option>]
|
|
||||||
</term>
|
|
||||||
<listitem><para>Cumulative number of huge allocation requests served
|
|
||||||
directly by the arena.</para></listitem>
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.huge.ndalloc">
|
|
||||||
<term>
|
|
||||||
<mallctl>stats.arenas.<i>.huge.ndalloc</mallctl>
|
|
||||||
(<type>uint64_t</type>)
|
|
||||||
<literal>r-</literal>
|
|
||||||
[<option>--enable-stats</option>]
|
|
||||||
</term>
|
|
||||||
<listitem><para>Cumulative number of huge deallocation requests served
|
|
||||||
directly by the arena.</para></listitem>
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.huge.nrequests">
|
|
||||||
<term>
|
|
||||||
<mallctl>stats.arenas.<i>.huge.nrequests</mallctl>
|
|
||||||
(<type>uint64_t</type>)
|
|
||||||
<literal>r-</literal>
|
|
||||||
[<option>--enable-stats</option>]
|
|
||||||
</term>
|
|
||||||
<listitem><para>Cumulative number of huge allocation requests.
|
|
||||||
</para></listitem>
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.bins.j.nmalloc">
|
<varlistentry id="stats.arenas.i.bins.j.nmalloc">
|
||||||
<term>
|
<term>
|
||||||
<mallctl>stats.arenas.<i>.bins.<j>.nmalloc</mallctl>
|
<mallctl>stats.arenas.<i>.bins.<j>.nmalloc</mallctl>
|
||||||
@ -2500,9 +2456,9 @@ typedef struct {
|
|||||||
<listitem><para>Current number of slabs.</para></listitem>
|
<listitem><para>Current number of slabs.</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.hchunks.j.nmalloc">
|
<varlistentry id="stats.arenas.i.lextents.j.nmalloc">
|
||||||
<term>
|
<term>
|
||||||
<mallctl>stats.arenas.<i>.hchunks.<j>.nmalloc</mallctl>
|
<mallctl>stats.arenas.<i>.lextents.<j>.nmalloc</mallctl>
|
||||||
(<type>uint64_t</type>)
|
(<type>uint64_t</type>)
|
||||||
<literal>r-</literal>
|
<literal>r-</literal>
|
||||||
[<option>--enable-stats</option>]
|
[<option>--enable-stats</option>]
|
||||||
@ -2511,9 +2467,9 @@ typedef struct {
|
|||||||
class served directly by the arena.</para></listitem>
|
class served directly by the arena.</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.hchunks.j.ndalloc">
|
<varlistentry id="stats.arenas.i.lextents.j.ndalloc">
|
||||||
<term>
|
<term>
|
||||||
<mallctl>stats.arenas.<i>.hchunks.<j>.ndalloc</mallctl>
|
<mallctl>stats.arenas.<i>.lextents.<j>.ndalloc</mallctl>
|
||||||
(<type>uint64_t</type>)
|
(<type>uint64_t</type>)
|
||||||
<literal>r-</literal>
|
<literal>r-</literal>
|
||||||
[<option>--enable-stats</option>]
|
[<option>--enable-stats</option>]
|
||||||
@ -2522,9 +2478,9 @@ typedef struct {
|
|||||||
size class served directly by the arena.</para></listitem>
|
size class served directly by the arena.</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.hchunks.j.nrequests">
|
<varlistentry id="stats.arenas.i.lextents.j.nrequests">
|
||||||
<term>
|
<term>
|
||||||
<mallctl>stats.arenas.<i>.hchunks.<j>.nrequests</mallctl>
|
<mallctl>stats.arenas.<i>.lextents.<j>.nrequests</mallctl>
|
||||||
(<type>uint64_t</type>)
|
(<type>uint64_t</type>)
|
||||||
<literal>r-</literal>
|
<literal>r-</literal>
|
||||||
[<option>--enable-stats</option>]
|
[<option>--enable-stats</option>]
|
||||||
@ -2533,14 +2489,14 @@ typedef struct {
|
|||||||
class.</para></listitem>
|
class.</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry id="stats.arenas.i.hchunks.j.curhchunks">
|
<varlistentry id="stats.arenas.i.lextents.j.curlextents">
|
||||||
<term>
|
<term>
|
||||||
<mallctl>stats.arenas.<i>.hchunks.<j>.curhchunks</mallctl>
|
<mallctl>stats.arenas.<i>.lextents.<j>.curlextents</mallctl>
|
||||||
(<type>size_t</type>)
|
(<type>size_t</type>)
|
||||||
<literal>r-</literal>
|
<literal>r-</literal>
|
||||||
[<option>--enable-stats</option>]
|
[<option>--enable-stats</option>]
|
||||||
</term>
|
</term>
|
||||||
<listitem><para>Current number of huge allocations for this size class.
|
<listitem><para>Current number of large allocations for this size class.
|
||||||
</para></listitem>
|
</para></listitem>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
</variablelist>
|
</variablelist>
|
||||||
|
@ -229,10 +229,10 @@ struct arena_s {
|
|||||||
*/
|
*/
|
||||||
size_t decay_backlog[SMOOTHSTEP_NSTEPS];
|
size_t decay_backlog[SMOOTHSTEP_NSTEPS];
|
||||||
|
|
||||||
/* Extant huge allocations. */
|
/* Extant large allocations. */
|
||||||
ql_head(extent_t) huge;
|
ql_head(extent_t) large;
|
||||||
/* Synchronizes all huge allocation/update/deallocation. */
|
/* Synchronizes all large allocation/update/deallocation. */
|
||||||
malloc_mutex_t huge_mtx;
|
malloc_mutex_t large_mtx;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Heaps of chunks that were previously allocated. These are used when
|
* Heaps of chunks that were previously allocated. These are used when
|
||||||
@ -287,13 +287,13 @@ void arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent,
|
|||||||
bool cache);
|
bool cache);
|
||||||
void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent,
|
void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent,
|
||||||
bool cache);
|
bool cache);
|
||||||
extent_t *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena,
|
extent_t *arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena,
|
||||||
size_t usize, size_t alignment, bool *zero);
|
size_t usize, size_t alignment, bool *zero);
|
||||||
void arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
void arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||||
bool locked);
|
bool locked);
|
||||||
void arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena,
|
void arena_chunk_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
|
||||||
extent_t *extent, size_t oldsize);
|
extent_t *extent, size_t oldsize);
|
||||||
void arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena,
|
void arena_chunk_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena,
|
||||||
extent_t *extent, size_t oldsize);
|
extent_t *extent, size_t oldsize);
|
||||||
ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena);
|
ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena);
|
||||||
bool arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena,
|
bool arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena,
|
||||||
@ -341,7 +341,7 @@ void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena,
|
|||||||
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
|
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
|
||||||
const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time,
|
const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time,
|
||||||
size_t *nactive, size_t *ndirty, arena_stats_t *astats,
|
size_t *nactive, size_t *ndirty, arena_stats_t *astats,
|
||||||
malloc_bin_stats_t *bstats, malloc_huge_stats_t *hstats);
|
malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats);
|
||||||
unsigned arena_nthreads_get(arena_t *arena, bool internal);
|
unsigned arena_nthreads_get(arena_t *arena, bool internal);
|
||||||
void arena_nthreads_inc(arena_t *arena, bool internal);
|
void arena_nthreads_inc(arena_t *arena, bool internal);
|
||||||
void arena_nthreads_dec(arena_t *arena, bool internal);
|
void arena_nthreads_dec(arena_t *arena, bool internal);
|
||||||
@ -470,7 +470,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
|
|||||||
assert(ptr != NULL);
|
assert(ptr != NULL);
|
||||||
|
|
||||||
if (unlikely(!extent_slab_get(extent)))
|
if (unlikely(!extent_slab_get(extent)))
|
||||||
return (huge_prof_tctx_get(tsdn, extent));
|
return (large_prof_tctx_get(tsdn, extent));
|
||||||
return ((prof_tctx_t *)(uintptr_t)1U);
|
return ((prof_tctx_t *)(uintptr_t)1U);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -483,7 +483,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
|
|||||||
assert(ptr != NULL);
|
assert(ptr != NULL);
|
||||||
|
|
||||||
if (unlikely(!extent_slab_get(extent)))
|
if (unlikely(!extent_slab_get(extent)))
|
||||||
huge_prof_tctx_set(tsdn, extent, tctx);
|
large_prof_tctx_set(tsdn, extent, tctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
JEMALLOC_INLINE void
|
JEMALLOC_INLINE void
|
||||||
@ -495,7 +495,7 @@ arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
|
|||||||
assert(ptr != NULL);
|
assert(ptr != NULL);
|
||||||
assert(!extent_slab_get(extent));
|
assert(!extent_slab_get(extent));
|
||||||
|
|
||||||
huge_prof_tctx_reset(tsdn, extent);
|
large_prof_tctx_reset(tsdn, extent);
|
||||||
}
|
}
|
||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE void
|
JEMALLOC_ALWAYS_INLINE void
|
||||||
@ -535,7 +535,7 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
|
|||||||
tcache, size, ind, zero, slow_path));
|
tcache, size, ind, zero, slow_path));
|
||||||
}
|
}
|
||||||
if (likely(size <= tcache_maxclass)) {
|
if (likely(size <= tcache_maxclass)) {
|
||||||
return (tcache_alloc_huge(tsdn_tsd(tsdn), arena,
|
return (tcache_alloc_large(tsdn_tsd(tsdn), arena,
|
||||||
tcache, size, ind, zero, slow_path));
|
tcache, size, ind, zero, slow_path));
|
||||||
}
|
}
|
||||||
/* (size > tcache_maxclass) case falls through. */
|
/* (size > tcache_maxclass) case falls through. */
|
||||||
@ -563,7 +563,7 @@ arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
|
|||||||
if (likely(extent_slab_get(extent)))
|
if (likely(extent_slab_get(extent)))
|
||||||
ret = index2size(extent_slab_data_get_const(extent)->binind);
|
ret = index2size(extent_slab_data_get_const(extent)->binind);
|
||||||
else
|
else
|
||||||
ret = huge_salloc(tsdn, extent);
|
ret = large_salloc(tsdn, extent);
|
||||||
|
|
||||||
return (ret);
|
return (ret);
|
||||||
}
|
}
|
||||||
@ -594,11 +594,11 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
|
|||||||
arena_dalloc_promoted(tsdn, extent, ptr,
|
arena_dalloc_promoted(tsdn, extent, ptr,
|
||||||
tcache, slow_path);
|
tcache, slow_path);
|
||||||
} else {
|
} else {
|
||||||
tcache_dalloc_huge(tsdn_tsd(tsdn), tcache,
|
tcache_dalloc_large(tsdn_tsd(tsdn), tcache,
|
||||||
ptr, usize, slow_path);
|
ptr, usize, slow_path);
|
||||||
}
|
}
|
||||||
} else
|
} else
|
||||||
huge_dalloc(tsdn, extent);
|
large_dalloc(tsdn, extent);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -627,11 +627,11 @@ arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
|
|||||||
arena_dalloc_promoted(tsdn, extent, ptr,
|
arena_dalloc_promoted(tsdn, extent, ptr,
|
||||||
tcache, slow_path);
|
tcache, slow_path);
|
||||||
} else {
|
} else {
|
||||||
tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, ptr,
|
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
|
||||||
size, slow_path);
|
size, slow_path);
|
||||||
}
|
}
|
||||||
} else
|
} else
|
||||||
huge_dalloc(tsdn, extent);
|
large_dalloc(tsdn, extent);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
# endif /* JEMALLOC_ARENA_INLINE_B */
|
# endif /* JEMALLOC_ARENA_INLINE_B */
|
||||||
|
@ -51,7 +51,7 @@ struct ctl_arena_stats_s {
|
|||||||
uint64_t nrequests_small;
|
uint64_t nrequests_small;
|
||||||
|
|
||||||
malloc_bin_stats_t bstats[NBINS];
|
malloc_bin_stats_t bstats[NBINS];
|
||||||
malloc_huge_stats_t hstats[NSIZES - NBINS];
|
malloc_large_stats_t lstats[NSIZES - NBINS];
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ctl_stats_s {
|
struct ctl_stats_s {
|
||||||
|
@ -51,7 +51,7 @@ struct extent_s {
|
|||||||
/* Small region slab metadata. */
|
/* Small region slab metadata. */
|
||||||
arena_slab_data_t e_slab_data;
|
arena_slab_data_t e_slab_data;
|
||||||
|
|
||||||
/* Profile counters, used for huge objects. */
|
/* Profile counters, used for large objects. */
|
||||||
union {
|
union {
|
||||||
void *e_prof_tctx_pun;
|
void *e_prof_tctx_pun;
|
||||||
prof_tctx_t *e_prof_tctx;
|
prof_tctx_t *e_prof_tctx;
|
||||||
@ -67,7 +67,7 @@ struct extent_s {
|
|||||||
/* Linkage for per size class address-ordered heaps. */
|
/* Linkage for per size class address-ordered heaps. */
|
||||||
phn(extent_t) ph_link;
|
phn(extent_t) ph_link;
|
||||||
|
|
||||||
/* Linkage for arena's huge and extent_cache lists. */
|
/* Linkage for arena's large and extent_cache lists. */
|
||||||
ql_elm(extent_t) ql_link;
|
ql_elm(extent_t) ql_link;
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
@ -1,37 +0,0 @@
|
|||||||
/******************************************************************************/
|
|
||||||
#ifdef JEMALLOC_H_TYPES
|
|
||||||
|
|
||||||
#endif /* JEMALLOC_H_TYPES */
|
|
||||||
/******************************************************************************/
|
|
||||||
#ifdef JEMALLOC_H_STRUCTS
|
|
||||||
|
|
||||||
#endif /* JEMALLOC_H_STRUCTS */
|
|
||||||
/******************************************************************************/
|
|
||||||
#ifdef JEMALLOC_H_EXTERNS
|
|
||||||
|
|
||||||
void *huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
|
|
||||||
void *huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
|
||||||
size_t alignment, bool zero);
|
|
||||||
bool huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
|
|
||||||
size_t usize_max, bool zero);
|
|
||||||
void *huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
|
||||||
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
|
|
||||||
#ifdef JEMALLOC_JET
|
|
||||||
typedef void (huge_dalloc_junk_t)(void *, size_t);
|
|
||||||
extern huge_dalloc_junk_t *huge_dalloc_junk;
|
|
||||||
#else
|
|
||||||
void huge_dalloc_junk(void *ptr, size_t usize);
|
|
||||||
#endif
|
|
||||||
void huge_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent);
|
|
||||||
void huge_dalloc(tsdn_t *tsdn, extent_t *extent);
|
|
||||||
size_t huge_salloc(tsdn_t *tsdn, const extent_t *extent);
|
|
||||||
prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent);
|
|
||||||
void huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx);
|
|
||||||
void huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent);
|
|
||||||
|
|
||||||
#endif /* JEMALLOC_H_EXTERNS */
|
|
||||||
/******************************************************************************/
|
|
||||||
#ifdef JEMALLOC_H_INLINES
|
|
||||||
|
|
||||||
#endif /* JEMALLOC_H_INLINES */
|
|
||||||
/******************************************************************************/
|
|
@ -362,7 +362,7 @@ typedef unsigned szind_t;
|
|||||||
#include "jemalloc/internal/rtree.h"
|
#include "jemalloc/internal/rtree.h"
|
||||||
#include "jemalloc/internal/pages.h"
|
#include "jemalloc/internal/pages.h"
|
||||||
#include "jemalloc/internal/chunk.h"
|
#include "jemalloc/internal/chunk.h"
|
||||||
#include "jemalloc/internal/huge.h"
|
#include "jemalloc/internal/large.h"
|
||||||
#include "jemalloc/internal/tcache.h"
|
#include "jemalloc/internal/tcache.h"
|
||||||
#include "jemalloc/internal/hash.h"
|
#include "jemalloc/internal/hash.h"
|
||||||
#include "jemalloc/internal/prof.h"
|
#include "jemalloc/internal/prof.h"
|
||||||
@ -396,7 +396,7 @@ typedef unsigned szind_t;
|
|||||||
#include "jemalloc/internal/rtree.h"
|
#include "jemalloc/internal/rtree.h"
|
||||||
#include "jemalloc/internal/pages.h"
|
#include "jemalloc/internal/pages.h"
|
||||||
#include "jemalloc/internal/chunk.h"
|
#include "jemalloc/internal/chunk.h"
|
||||||
#include "jemalloc/internal/huge.h"
|
#include "jemalloc/internal/large.h"
|
||||||
#include "jemalloc/internal/tcache.h"
|
#include "jemalloc/internal/tcache.h"
|
||||||
#include "jemalloc/internal/hash.h"
|
#include "jemalloc/internal/hash.h"
|
||||||
#include "jemalloc/internal/prof.h"
|
#include "jemalloc/internal/prof.h"
|
||||||
@ -486,7 +486,7 @@ void jemalloc_postfork_child(void);
|
|||||||
#include "jemalloc/internal/rtree.h"
|
#include "jemalloc/internal/rtree.h"
|
||||||
#include "jemalloc/internal/pages.h"
|
#include "jemalloc/internal/pages.h"
|
||||||
#include "jemalloc/internal/chunk.h"
|
#include "jemalloc/internal/chunk.h"
|
||||||
#include "jemalloc/internal/huge.h"
|
#include "jemalloc/internal/large.h"
|
||||||
#include "jemalloc/internal/tcache.h"
|
#include "jemalloc/internal/tcache.h"
|
||||||
#include "jemalloc/internal/hash.h"
|
#include "jemalloc/internal/hash.h"
|
||||||
#include "jemalloc/internal/prof.h"
|
#include "jemalloc/internal/prof.h"
|
||||||
@ -515,7 +515,7 @@ void jemalloc_postfork_child(void);
|
|||||||
#include "jemalloc/internal/rtree.h"
|
#include "jemalloc/internal/rtree.h"
|
||||||
#include "jemalloc/internal/pages.h"
|
#include "jemalloc/internal/pages.h"
|
||||||
#include "jemalloc/internal/chunk.h"
|
#include "jemalloc/internal/chunk.h"
|
||||||
#include "jemalloc/internal/huge.h"
|
#include "jemalloc/internal/large.h"
|
||||||
|
|
||||||
#ifndef JEMALLOC_ENABLE_INLINE
|
#ifndef JEMALLOC_ENABLE_INLINE
|
||||||
pszind_t psz2ind(size_t psz);
|
pszind_t psz2ind(size_t psz);
|
||||||
@ -547,7 +547,7 @@ JEMALLOC_INLINE pszind_t
|
|||||||
psz2ind(size_t psz)
|
psz2ind(size_t psz)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (unlikely(psz > HUGE_MAXCLASS))
|
if (unlikely(psz > LARGE_MAXCLASS))
|
||||||
return (NPSIZES);
|
return (NPSIZES);
|
||||||
{
|
{
|
||||||
pszind_t x = lg_floor((psz<<1)-1);
|
pszind_t x = lg_floor((psz<<1)-1);
|
||||||
@ -608,7 +608,7 @@ JEMALLOC_INLINE size_t
|
|||||||
psz2u(size_t psz)
|
psz2u(size_t psz)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (unlikely(psz > HUGE_MAXCLASS))
|
if (unlikely(psz > LARGE_MAXCLASS))
|
||||||
return (0);
|
return (0);
|
||||||
{
|
{
|
||||||
size_t x = lg_floor((psz<<1)-1);
|
size_t x = lg_floor((psz<<1)-1);
|
||||||
@ -625,7 +625,7 @@ JEMALLOC_INLINE szind_t
|
|||||||
size2index_compute(size_t size)
|
size2index_compute(size_t size)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (unlikely(size > HUGE_MAXCLASS))
|
if (unlikely(size > LARGE_MAXCLASS))
|
||||||
return (NSIZES);
|
return (NSIZES);
|
||||||
#if (NTBINS != 0)
|
#if (NTBINS != 0)
|
||||||
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
|
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
|
||||||
@ -721,7 +721,7 @@ JEMALLOC_ALWAYS_INLINE size_t
|
|||||||
s2u_compute(size_t size)
|
s2u_compute(size_t size)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (unlikely(size > HUGE_MAXCLASS))
|
if (unlikely(size > LARGE_MAXCLASS))
|
||||||
return (0);
|
return (0);
|
||||||
#if (NTBINS > 0)
|
#if (NTBINS > 0)
|
||||||
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
|
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
|
||||||
@ -797,9 +797,9 @@ sa2u(size_t size, size_t alignment)
|
|||||||
return (usize);
|
return (usize);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Huge size class. Beware of overflow. */
|
/* Large size class. Beware of overflow. */
|
||||||
|
|
||||||
if (unlikely(alignment > HUGE_MAXCLASS))
|
if (unlikely(alignment > LARGE_MAXCLASS))
|
||||||
return (0);
|
return (0);
|
||||||
|
|
||||||
/* Make sure result is a large size class. */
|
/* Make sure result is a large size class. */
|
||||||
@ -814,7 +814,7 @@ sa2u(size_t size, size_t alignment)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Calculate the multi-page mapping that huge_palloc() would need in
|
* Calculate the multi-page mapping that large_palloc() would need in
|
||||||
* order to guarantee the alignment.
|
* order to guarantee the alignment.
|
||||||
*/
|
*/
|
||||||
if (usize + large_pad + PAGE_CEILING(alignment) < usize) {
|
if (usize + large_pad + PAGE_CEILING(alignment) < usize) {
|
||||||
@ -1113,7 +1113,7 @@ iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
|
|||||||
size_t usize, copysize;
|
size_t usize, copysize;
|
||||||
|
|
||||||
usize = sa2u(size + extra, alignment);
|
usize = sa2u(size + extra, alignment);
|
||||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
|
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
|
||||||
return (NULL);
|
return (NULL);
|
||||||
p = ipalloct(tsdn, usize, alignment, zero, tcache, arena);
|
p = ipalloct(tsdn, usize, alignment, zero, tcache, arena);
|
||||||
if (p == NULL) {
|
if (p == NULL) {
|
||||||
@ -1121,7 +1121,7 @@ iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
|
|||||||
return (NULL);
|
return (NULL);
|
||||||
/* Try again, without extra this time. */
|
/* Try again, without extra this time. */
|
||||||
usize = sa2u(size, alignment);
|
usize = sa2u(size, alignment);
|
||||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
|
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
|
||||||
return (NULL);
|
return (NULL);
|
||||||
p = ipalloct(tsdn, usize, alignment, zero, tcache, arena);
|
p = ipalloct(tsdn, usize, alignment, zero, tcache, arena);
|
||||||
if (p == NULL)
|
if (p == NULL)
|
||||||
|
37
include/jemalloc/internal/large.h
Normal file
37
include/jemalloc/internal/large.h
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
/******************************************************************************/
|
||||||
|
#ifdef JEMALLOC_H_TYPES
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_H_TYPES */
|
||||||
|
/******************************************************************************/
|
||||||
|
#ifdef JEMALLOC_H_STRUCTS
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_H_STRUCTS */
|
||||||
|
/******************************************************************************/
|
||||||
|
#ifdef JEMALLOC_H_EXTERNS
|
||||||
|
|
||||||
|
void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
|
||||||
|
void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||||
|
size_t alignment, bool zero);
|
||||||
|
bool large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
|
||||||
|
size_t usize_max, bool zero);
|
||||||
|
void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||||
|
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
|
||||||
|
#ifdef JEMALLOC_JET
|
||||||
|
typedef void (large_dalloc_junk_t)(void *, size_t);
|
||||||
|
extern large_dalloc_junk_t *large_dalloc_junk;
|
||||||
|
#else
|
||||||
|
void large_dalloc_junk(void *ptr, size_t usize);
|
||||||
|
#endif
|
||||||
|
void large_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent);
|
||||||
|
void large_dalloc(tsdn_t *tsdn, extent_t *extent);
|
||||||
|
size_t large_salloc(tsdn_t *tsdn, const extent_t *extent);
|
||||||
|
prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent);
|
||||||
|
void large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx);
|
||||||
|
void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent);
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_H_EXTERNS */
|
||||||
|
/******************************************************************************/
|
||||||
|
#ifdef JEMALLOC_H_INLINES
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_H_INLINES */
|
||||||
|
/******************************************************************************/
|
@ -9,14 +9,14 @@ arena_boot
|
|||||||
arena_choose
|
arena_choose
|
||||||
arena_choose_hard
|
arena_choose_hard
|
||||||
arena_choose_impl
|
arena_choose_impl
|
||||||
arena_chunk_alloc_huge
|
arena_chunk_alloc_large
|
||||||
arena_chunk_cache_alloc
|
arena_chunk_cache_alloc
|
||||||
arena_chunk_cache_dalloc
|
arena_chunk_cache_dalloc
|
||||||
arena_chunk_cache_maybe_insert
|
arena_chunk_cache_maybe_insert
|
||||||
arena_chunk_cache_maybe_remove
|
arena_chunk_cache_maybe_remove
|
||||||
arena_chunk_dalloc_huge
|
arena_chunk_dalloc_large
|
||||||
arena_chunk_ralloc_huge_expand
|
arena_chunk_ralloc_large_expand
|
||||||
arena_chunk_ralloc_huge_shrink
|
arena_chunk_ralloc_large_shrink
|
||||||
arena_cleanup
|
arena_cleanup
|
||||||
arena_dalloc
|
arena_dalloc
|
||||||
arena_dalloc_bin_junked_locked
|
arena_dalloc_bin_junked_locked
|
||||||
@ -222,17 +222,6 @@ hash_rotl_64
|
|||||||
hash_x64_128
|
hash_x64_128
|
||||||
hash_x86_128
|
hash_x86_128
|
||||||
hash_x86_32
|
hash_x86_32
|
||||||
huge_dalloc
|
|
||||||
huge_dalloc_junk
|
|
||||||
huge_dalloc_junked_locked
|
|
||||||
huge_malloc
|
|
||||||
huge_palloc
|
|
||||||
huge_prof_tctx_get
|
|
||||||
huge_prof_tctx_reset
|
|
||||||
huge_prof_tctx_set
|
|
||||||
huge_ralloc
|
|
||||||
huge_ralloc_no_move
|
|
||||||
huge_salloc
|
|
||||||
iaalloc
|
iaalloc
|
||||||
ialloc
|
ialloc
|
||||||
iallocztm
|
iallocztm
|
||||||
@ -258,6 +247,17 @@ ixalloc
|
|||||||
jemalloc_postfork_child
|
jemalloc_postfork_child
|
||||||
jemalloc_postfork_parent
|
jemalloc_postfork_parent
|
||||||
jemalloc_prefork
|
jemalloc_prefork
|
||||||
|
large_dalloc
|
||||||
|
large_dalloc_junk
|
||||||
|
large_dalloc_junked_locked
|
||||||
|
large_malloc
|
||||||
|
large_palloc
|
||||||
|
large_prof_tctx_get
|
||||||
|
large_prof_tctx_reset
|
||||||
|
large_prof_tctx_set
|
||||||
|
large_ralloc
|
||||||
|
large_ralloc_no_move
|
||||||
|
large_salloc
|
||||||
lg_floor
|
lg_floor
|
||||||
lg_prof_sample
|
lg_prof_sample
|
||||||
malloc_cprintf
|
malloc_cprintf
|
||||||
@ -438,17 +438,17 @@ stats_cactive_get
|
|||||||
stats_cactive_sub
|
stats_cactive_sub
|
||||||
stats_print
|
stats_print
|
||||||
tcache_alloc_easy
|
tcache_alloc_easy
|
||||||
tcache_alloc_huge
|
tcache_alloc_large
|
||||||
tcache_alloc_small
|
tcache_alloc_small
|
||||||
tcache_alloc_small_hard
|
tcache_alloc_small_hard
|
||||||
tcache_arena_reassociate
|
tcache_arena_reassociate
|
||||||
tcache_bin_flush_huge
|
tcache_bin_flush_large
|
||||||
tcache_bin_flush_small
|
tcache_bin_flush_small
|
||||||
tcache_bin_info
|
tcache_bin_info
|
||||||
tcache_boot
|
tcache_boot
|
||||||
tcache_cleanup
|
tcache_cleanup
|
||||||
tcache_create
|
tcache_create
|
||||||
tcache_dalloc_huge
|
tcache_dalloc_large
|
||||||
tcache_dalloc_small
|
tcache_dalloc_small
|
||||||
tcache_enabled_cleanup
|
tcache_enabled_cleanup
|
||||||
tcache_enabled_get
|
tcache_enabled_get
|
||||||
|
@ -237,7 +237,7 @@ size_classes() {
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
# Final written value is correct:
|
# Final written value is correct:
|
||||||
huge_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))"
|
large_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))"
|
||||||
index=$((${index} + 1))
|
index=$((${index} + 1))
|
||||||
ndelta=$((${ndelta} + 1))
|
ndelta=$((${ndelta} + 1))
|
||||||
done
|
done
|
||||||
@ -257,7 +257,7 @@ size_classes() {
|
|||||||
# - lookup_maxclass
|
# - lookup_maxclass
|
||||||
# - small_maxclass
|
# - small_maxclass
|
||||||
# - lg_large_minclass
|
# - lg_large_minclass
|
||||||
# - huge_maxclass
|
# - large_maxclass
|
||||||
}
|
}
|
||||||
|
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
@ -290,7 +290,7 @@ cat <<EOF
|
|||||||
* LOOKUP_MAXCLASS: Maximum size class included in lookup table.
|
* LOOKUP_MAXCLASS: Maximum size class included in lookup table.
|
||||||
* SMALL_MAXCLASS: Maximum small size class.
|
* SMALL_MAXCLASS: Maximum small size class.
|
||||||
* LG_LARGE_MINCLASS: Lg of minimum large size class.
|
* LG_LARGE_MINCLASS: Lg of minimum large size class.
|
||||||
* HUGE_MAXCLASS: Maximum (huge) size class.
|
* LARGE_MAXCLASS: Maximum (large) size class.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#define LG_SIZE_CLASS_GROUP ${lg_g}
|
#define LG_SIZE_CLASS_GROUP ${lg_g}
|
||||||
@ -315,7 +315,7 @@ for lg_z in ${lg_zarr} ; do
|
|||||||
echo "#define LOOKUP_MAXCLASS ${lookup_maxclass}"
|
echo "#define LOOKUP_MAXCLASS ${lookup_maxclass}"
|
||||||
echo "#define SMALL_MAXCLASS ${small_maxclass}"
|
echo "#define SMALL_MAXCLASS ${small_maxclass}"
|
||||||
echo "#define LG_LARGE_MINCLASS ${lg_large_minclass}"
|
echo "#define LG_LARGE_MINCLASS ${lg_large_minclass}"
|
||||||
echo "#define HUGE_MAXCLASS ${huge_maxclass}"
|
echo "#define LARGE_MAXCLASS ${large_maxclass}"
|
||||||
echo "#endif"
|
echo "#endif"
|
||||||
echo
|
echo
|
||||||
done
|
done
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
typedef struct tcache_bin_stats_s tcache_bin_stats_t;
|
typedef struct tcache_bin_stats_s tcache_bin_stats_t;
|
||||||
typedef struct malloc_bin_stats_s malloc_bin_stats_t;
|
typedef struct malloc_bin_stats_s malloc_bin_stats_t;
|
||||||
typedef struct malloc_huge_stats_s malloc_huge_stats_t;
|
typedef struct malloc_large_stats_s malloc_large_stats_t;
|
||||||
typedef struct arena_stats_s arena_stats_t;
|
typedef struct arena_stats_s arena_stats_t;
|
||||||
typedef struct chunk_stats_s chunk_stats_t;
|
typedef struct chunk_stats_s chunk_stats_t;
|
||||||
|
|
||||||
@ -61,7 +61,7 @@ struct malloc_bin_stats_s {
|
|||||||
size_t curslabs;
|
size_t curslabs;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct malloc_huge_stats_s {
|
struct malloc_large_stats_s {
|
||||||
/*
|
/*
|
||||||
* Total number of allocation/deallocation requests served directly by
|
* Total number of allocation/deallocation requests served directly by
|
||||||
* the arena.
|
* the arena.
|
||||||
@ -77,7 +77,7 @@ struct malloc_huge_stats_s {
|
|||||||
uint64_t nrequests;
|
uint64_t nrequests;
|
||||||
|
|
||||||
/* Current number of (multi-)chunk allocations of this size class. */
|
/* Current number of (multi-)chunk allocations of this size class. */
|
||||||
size_t curhchunks;
|
size_t curlextents;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct arena_stats_s {
|
struct arena_stats_s {
|
||||||
@ -108,13 +108,13 @@ struct arena_stats_s {
|
|||||||
size_t metadata_mapped;
|
size_t metadata_mapped;
|
||||||
size_t metadata_allocated; /* Protected via atomic_*_z(). */
|
size_t metadata_allocated; /* Protected via atomic_*_z(). */
|
||||||
|
|
||||||
size_t allocated_huge;
|
size_t allocated_large;
|
||||||
uint64_t nmalloc_huge;
|
uint64_t nmalloc_large;
|
||||||
uint64_t ndalloc_huge;
|
uint64_t ndalloc_large;
|
||||||
uint64_t nrequests_huge;
|
uint64_t nrequests_large;
|
||||||
|
|
||||||
/* One element for each huge size class. */
|
/* One element for each large size class. */
|
||||||
malloc_huge_stats_t hstats[NSIZES - NBINS];
|
malloc_large_stats_t lstats[NSIZES - NBINS];
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif /* JEMALLOC_H_STRUCTS */
|
#endif /* JEMALLOC_H_STRUCTS */
|
||||||
|
@ -30,8 +30,8 @@ typedef struct tcaches_s tcaches_t;
|
|||||||
*/
|
*/
|
||||||
#define TCACHE_NSLOTS_SMALL_MAX 200
|
#define TCACHE_NSLOTS_SMALL_MAX 200
|
||||||
|
|
||||||
/* Number of cache slots for huge size classes. */
|
/* Number of cache slots for large size classes. */
|
||||||
#define TCACHE_NSLOTS_HUGE 20
|
#define TCACHE_NSLOTS_LARGE 20
|
||||||
|
|
||||||
/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
|
/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
|
||||||
#define LG_TCACHE_MAXCLASS_DEFAULT 15
|
#define LG_TCACHE_MAXCLASS_DEFAULT 15
|
||||||
@ -113,7 +113,7 @@ extern tcache_bin_info_t *tcache_bin_info;
|
|||||||
|
|
||||||
/*
|
/*
|
||||||
* Number of tcache bins. There are NBINS small-object bins, plus 0 or more
|
* Number of tcache bins. There are NBINS small-object bins, plus 0 or more
|
||||||
* huge-object bins.
|
* large-object bins.
|
||||||
*/
|
*/
|
||||||
extern unsigned nhbins;
|
extern unsigned nhbins;
|
||||||
|
|
||||||
@ -136,7 +136,7 @@ void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
|
|||||||
tcache_bin_t *tbin, szind_t binind, bool *tcache_success);
|
tcache_bin_t *tbin, szind_t binind, bool *tcache_success);
|
||||||
void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
|
void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
|
||||||
szind_t binind, unsigned rem);
|
szind_t binind, unsigned rem);
|
||||||
void tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
||||||
unsigned rem, tcache_t *tcache);
|
unsigned rem, tcache_t *tcache);
|
||||||
void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
|
void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
|
||||||
arena_t *oldarena, arena_t *newarena);
|
arena_t *oldarena, arena_t *newarena);
|
||||||
@ -163,11 +163,11 @@ void tcache_enabled_set(bool enabled);
|
|||||||
void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
|
void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
|
||||||
void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
|
void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
|
||||||
size_t size, szind_t ind, bool zero, bool slow_path);
|
size_t size, szind_t ind, bool zero, bool slow_path);
|
||||||
void *tcache_alloc_huge(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
|
void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
|
||||||
size_t size, szind_t ind, bool zero, bool slow_path);
|
size_t size, szind_t ind, bool zero, bool slow_path);
|
||||||
void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
|
void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
|
||||||
szind_t binind, bool slow_path);
|
szind_t binind, bool slow_path);
|
||||||
void tcache_dalloc_huge(tsd_t *tsd, tcache_t *tcache, void *ptr,
|
void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
|
||||||
size_t size, bool slow_path);
|
size_t size, bool slow_path);
|
||||||
tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
|
tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
|
||||||
#endif
|
#endif
|
||||||
@ -336,7 +336,7 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
|||||||
}
|
}
|
||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE void *
|
JEMALLOC_ALWAYS_INLINE void *
|
||||||
tcache_alloc_huge(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
||||||
szind_t binind, bool zero, bool slow_path)
|
szind_t binind, bool zero, bool slow_path)
|
||||||
{
|
{
|
||||||
void *ret;
|
void *ret;
|
||||||
@ -349,14 +349,14 @@ tcache_alloc_huge(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
|||||||
assert(tcache_success == (ret != NULL));
|
assert(tcache_success == (ret != NULL));
|
||||||
if (unlikely(!tcache_success)) {
|
if (unlikely(!tcache_success)) {
|
||||||
/*
|
/*
|
||||||
* Only allocate one huge object at a time, because it's quite
|
* Only allocate one large object at a time, because it's quite
|
||||||
* expensive to create one and not use it.
|
* expensive to create one and not use it.
|
||||||
*/
|
*/
|
||||||
arena = arena_choose(tsd, arena);
|
arena = arena_choose(tsd, arena);
|
||||||
if (unlikely(arena == NULL))
|
if (unlikely(arena == NULL))
|
||||||
return (NULL);
|
return (NULL);
|
||||||
|
|
||||||
ret = huge_malloc(tsd_tsdn(tsd), arena, s2u(size), zero);
|
ret = large_malloc(tsd_tsdn(tsd), arena, s2u(size), zero);
|
||||||
if (ret == NULL)
|
if (ret == NULL)
|
||||||
return (NULL);
|
return (NULL);
|
||||||
} else {
|
} else {
|
||||||
@ -416,7 +416,7 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
|
|||||||
}
|
}
|
||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE void
|
JEMALLOC_ALWAYS_INLINE void
|
||||||
tcache_dalloc_huge(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
|
tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
|
||||||
bool slow_path)
|
bool slow_path)
|
||||||
{
|
{
|
||||||
szind_t binind;
|
szind_t binind;
|
||||||
@ -429,12 +429,12 @@ tcache_dalloc_huge(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
|
|||||||
binind = size2index(size);
|
binind = size2index(size);
|
||||||
|
|
||||||
if (slow_path && config_fill && unlikely(opt_junk_free))
|
if (slow_path && config_fill && unlikely(opt_junk_free))
|
||||||
huge_dalloc_junk(ptr, size);
|
large_dalloc_junk(ptr, size);
|
||||||
|
|
||||||
tbin = &tcache->tbins[binind];
|
tbin = &tcache->tbins[binind];
|
||||||
tbin_info = &tcache_bin_info[binind];
|
tbin_info = &tcache_bin_info[binind];
|
||||||
if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
|
if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
|
||||||
tcache_bin_flush_huge(tsd, tbin, binind,
|
tcache_bin_flush_large(tsd, tbin, binind,
|
||||||
(tbin_info->ncached_max >> 1), tcache);
|
(tbin_info->ncached_max >> 1), tcache);
|
||||||
}
|
}
|
||||||
assert(tbin->ncached < tbin_info->ncached_max);
|
assert(tbin->ncached < tbin_info->ncached_max);
|
||||||
|
@ -32,7 +32,7 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
|
|||||||
|
|
||||||
#define WITNESS_RANK_LEAF 0xffffffffU
|
#define WITNESS_RANK_LEAF 0xffffffffU
|
||||||
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF
|
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF
|
||||||
#define WITNESS_RANK_ARENA_HUGE WITNESS_RANK_LEAF
|
#define WITNESS_RANK_ARENA_LARGE WITNESS_RANK_LEAF
|
||||||
#define WITNESS_RANK_DSS WITNESS_RANK_LEAF
|
#define WITNESS_RANK_DSS WITNESS_RANK_LEAF
|
||||||
#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
|
#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
|
||||||
#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF
|
#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF
|
||||||
|
@ -47,11 +47,11 @@
|
|||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ctl.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ctl.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\huge.h" />
|
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_decls.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_decls.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_defs.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_defs.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h" />
|
||||||
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\large.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mutex.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mutex.h" />
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\nstime.h" />
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\nstime.h" />
|
||||||
@ -98,8 +98,8 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\ctl.c" />
|
<ClCompile Include="..\..\..\..\src\ctl.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\extent.c" />
|
<ClCompile Include="..\..\..\..\src\extent.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\hash.c" />
|
<ClCompile Include="..\..\..\..\src\hash.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\huge.c" />
|
|
||||||
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
|
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
|
||||||
|
<ClCompile Include="..\..\..\..\src\large.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\mb.c" />
|
<ClCompile Include="..\..\..\..\src\mb.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\mutex.c" />
|
<ClCompile Include="..\..\..\..\src\mutex.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\nstime.c" />
|
<ClCompile Include="..\..\..\..\src\nstime.c" />
|
||||||
|
@ -80,9 +80,6 @@
|
|||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h">
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h">
|
||||||
<Filter>Header Files\internal</Filter>
|
<Filter>Header Files\internal</Filter>
|
||||||
</ClInclude>
|
</ClInclude>
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\huge.h">
|
|
||||||
<Filter>Header Files\internal</Filter>
|
|
||||||
</ClInclude>
|
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h">
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h">
|
||||||
<Filter>Header Files\internal</Filter>
|
<Filter>Header Files\internal</Filter>
|
||||||
</ClInclude>
|
</ClInclude>
|
||||||
@ -95,6 +92,9 @@
|
|||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h">
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h">
|
||||||
<Filter>Header Files\internal</Filter>
|
<Filter>Header Files\internal</Filter>
|
||||||
</ClInclude>
|
</ClInclude>
|
||||||
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\large.h">
|
||||||
|
<Filter>Header Files\internal</Filter>
|
||||||
|
</ClInclude>
|
||||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h">
|
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h">
|
||||||
<Filter>Header Files\internal</Filter>
|
<Filter>Header Files\internal</Filter>
|
||||||
</ClInclude>
|
</ClInclude>
|
||||||
@ -205,10 +205,10 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\hash.c">
|
<ClCompile Include="..\..\..\..\src\hash.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\huge.c">
|
<ClCompile Include="..\..\..\..\src\jemalloc.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\jemalloc.c">
|
<ClCompile Include="..\..\..\..\src\large.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\mb.c">
|
<ClCompile Include="..\..\..\..\src\mb.c">
|
||||||
|
143
src/arena.c
143
src/arena.c
@ -256,71 +256,71 @@ arena_nactive_sub(arena_t *arena, size_t sub_pages)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
arena_huge_malloc_stats_update(arena_t *arena, size_t usize)
|
arena_large_malloc_stats_update(arena_t *arena, size_t usize)
|
||||||
{
|
{
|
||||||
szind_t index = size2index(usize);
|
szind_t index = size2index(usize);
|
||||||
szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
|
szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
|
||||||
|
|
||||||
cassert(config_stats);
|
cassert(config_stats);
|
||||||
|
|
||||||
arena->stats.nmalloc_huge++;
|
arena->stats.nmalloc_large++;
|
||||||
arena->stats.allocated_huge += usize;
|
arena->stats.allocated_large += usize;
|
||||||
arena->stats.hstats[hindex].nmalloc++;
|
arena->stats.lstats[hindex].nmalloc++;
|
||||||
arena->stats.hstats[hindex].nrequests++;
|
arena->stats.lstats[hindex].nrequests++;
|
||||||
arena->stats.hstats[hindex].curhchunks++;
|
arena->stats.lstats[hindex].curlextents++;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize)
|
arena_large_malloc_stats_update_undo(arena_t *arena, size_t usize)
|
||||||
{
|
{
|
||||||
szind_t index = size2index(usize);
|
szind_t index = size2index(usize);
|
||||||
szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
|
szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
|
||||||
|
|
||||||
cassert(config_stats);
|
cassert(config_stats);
|
||||||
|
|
||||||
arena->stats.nmalloc_huge--;
|
arena->stats.nmalloc_large--;
|
||||||
arena->stats.allocated_huge -= usize;
|
arena->stats.allocated_large -= usize;
|
||||||
arena->stats.hstats[hindex].nmalloc--;
|
arena->stats.lstats[hindex].nmalloc--;
|
||||||
arena->stats.hstats[hindex].nrequests--;
|
arena->stats.lstats[hindex].nrequests--;
|
||||||
arena->stats.hstats[hindex].curhchunks--;
|
arena->stats.lstats[hindex].curlextents--;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
arena_huge_dalloc_stats_update(arena_t *arena, size_t usize)
|
arena_large_dalloc_stats_update(arena_t *arena, size_t usize)
|
||||||
{
|
{
|
||||||
szind_t index = size2index(usize);
|
szind_t index = size2index(usize);
|
||||||
szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
|
szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
|
||||||
|
|
||||||
cassert(config_stats);
|
cassert(config_stats);
|
||||||
|
|
||||||
arena->stats.ndalloc_huge++;
|
arena->stats.ndalloc_large++;
|
||||||
arena->stats.allocated_huge -= usize;
|
arena->stats.allocated_large -= usize;
|
||||||
arena->stats.hstats[hindex].ndalloc++;
|
arena->stats.lstats[hindex].ndalloc++;
|
||||||
arena->stats.hstats[hindex].curhchunks--;
|
arena->stats.lstats[hindex].curlextents--;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
arena_huge_reset_stats_cancel(arena_t *arena, size_t usize)
|
arena_large_reset_stats_cancel(arena_t *arena, size_t usize)
|
||||||
{
|
{
|
||||||
szind_t index = size2index(usize);
|
szind_t index = size2index(usize);
|
||||||
szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
|
szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
|
||||||
|
|
||||||
cassert(config_stats);
|
cassert(config_stats);
|
||||||
|
|
||||||
arena->stats.ndalloc_huge++;
|
arena->stats.ndalloc_large++;
|
||||||
arena->stats.hstats[hindex].ndalloc--;
|
arena->stats.lstats[hindex].ndalloc--;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
arena_huge_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize)
|
arena_large_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize)
|
||||||
{
|
{
|
||||||
|
|
||||||
arena_huge_dalloc_stats_update(arena, oldusize);
|
arena_large_dalloc_stats_update(arena, oldusize);
|
||||||
arena_huge_malloc_stats_update(arena, usize);
|
arena_large_malloc_stats_update(arena, usize);
|
||||||
}
|
}
|
||||||
|
|
||||||
static extent_t *
|
static extent_t *
|
||||||
arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
|
arena_chunk_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,
|
||||||
chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero)
|
chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero)
|
||||||
{
|
{
|
||||||
extent_t *extent;
|
extent_t *extent;
|
||||||
@ -332,7 +332,7 @@ arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
|
|||||||
/* Revert optimistic stats updates. */
|
/* Revert optimistic stats updates. */
|
||||||
malloc_mutex_lock(tsdn, &arena->lock);
|
malloc_mutex_lock(tsdn, &arena->lock);
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
arena_huge_malloc_stats_update_undo(arena, usize);
|
arena_large_malloc_stats_update_undo(arena, usize);
|
||||||
arena->stats.mapped -= usize;
|
arena->stats.mapped -= usize;
|
||||||
}
|
}
|
||||||
arena_nactive_sub(arena, (usize + large_pad) >> LG_PAGE);
|
arena_nactive_sub(arena, (usize + large_pad) >> LG_PAGE);
|
||||||
@ -343,7 +343,7 @@ arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
|
|||||||
}
|
}
|
||||||
|
|
||||||
extent_t *
|
extent_t *
|
||||||
arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||||
size_t alignment, bool *zero)
|
size_t alignment, bool *zero)
|
||||||
{
|
{
|
||||||
extent_t *extent;
|
extent_t *extent;
|
||||||
@ -353,7 +353,7 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
|||||||
|
|
||||||
/* Optimistically update stats. */
|
/* Optimistically update stats. */
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
arena_huge_malloc_stats_update(arena, usize);
|
arena_large_malloc_stats_update(arena, usize);
|
||||||
arena->stats.mapped += usize;
|
arena->stats.mapped += usize;
|
||||||
}
|
}
|
||||||
arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE);
|
arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE);
|
||||||
@ -362,7 +362,7 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
|||||||
usize, large_pad, alignment, zero, false);
|
usize, large_pad, alignment, zero, false);
|
||||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||||
if (extent == NULL) {
|
if (extent == NULL) {
|
||||||
extent = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks,
|
extent = arena_chunk_alloc_large_hard(tsdn, arena, &chunk_hooks,
|
||||||
usize, alignment, zero);
|
usize, alignment, zero);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -370,7 +370,7 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
|||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||||
bool locked)
|
bool locked)
|
||||||
{
|
{
|
||||||
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
|
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
|
||||||
@ -378,7 +378,8 @@ arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
|||||||
if (!locked)
|
if (!locked)
|
||||||
malloc_mutex_lock(tsdn, &arena->lock);
|
malloc_mutex_lock(tsdn, &arena->lock);
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
arena_huge_dalloc_stats_update(arena, extent_usize_get(extent));
|
arena_large_dalloc_stats_update(arena,
|
||||||
|
extent_usize_get(extent));
|
||||||
arena->stats.mapped -= extent_size_get(extent);
|
arena->stats.mapped -= extent_size_get(extent);
|
||||||
}
|
}
|
||||||
arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE);
|
arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE);
|
||||||
@ -389,7 +390,7 @@ arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
|||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
arena_chunk_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||||
size_t oldusize)
|
size_t oldusize)
|
||||||
{
|
{
|
||||||
size_t usize = extent_usize_get(extent);
|
size_t usize = extent_usize_get(extent);
|
||||||
@ -397,7 +398,7 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
|||||||
|
|
||||||
malloc_mutex_lock(tsdn, &arena->lock);
|
malloc_mutex_lock(tsdn, &arena->lock);
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
arena_huge_ralloc_stats_update(arena, oldusize, usize);
|
arena_large_ralloc_stats_update(arena, oldusize, usize);
|
||||||
arena->stats.mapped -= udiff;
|
arena->stats.mapped -= udiff;
|
||||||
}
|
}
|
||||||
arena_nactive_sub(arena, udiff >> LG_PAGE);
|
arena_nactive_sub(arena, udiff >> LG_PAGE);
|
||||||
@ -405,7 +406,7 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
|||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
arena_chunk_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||||
size_t oldusize)
|
size_t oldusize)
|
||||||
{
|
{
|
||||||
size_t usize = extent_usize_get(extent);
|
size_t usize = extent_usize_get(extent);
|
||||||
@ -413,7 +414,7 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
|||||||
|
|
||||||
malloc_mutex_lock(tsdn, &arena->lock);
|
malloc_mutex_lock(tsdn, &arena->lock);
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
arena_huge_ralloc_stats_update(arena, oldusize, usize);
|
arena_large_ralloc_stats_update(arena, oldusize, usize);
|
||||||
arena->stats.mapped += udiff;
|
arena->stats.mapped += udiff;
|
||||||
}
|
}
|
||||||
arena_nactive_add(arena, udiff >> LG_PAGE);
|
arena_nactive_add(arena, udiff >> LG_PAGE);
|
||||||
@ -891,26 +892,26 @@ arena_reset(tsd_t *tsd, arena_t *arena)
|
|||||||
* stats refreshes would impose an inconvenient burden.
|
* stats refreshes would impose an inconvenient burden.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Huge allocations. */
|
/* Large allocations. */
|
||||||
malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx);
|
malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
|
||||||
for (extent = ql_last(&arena->huge, ql_link); extent != NULL; extent =
|
for (extent = ql_last(&arena->large, ql_link); extent != NULL; extent =
|
||||||
ql_last(&arena->huge, ql_link)) {
|
ql_last(&arena->large, ql_link)) {
|
||||||
void *ptr = extent_base_get(extent);
|
void *ptr = extent_base_get(extent);
|
||||||
size_t usize;
|
size_t usize;
|
||||||
|
|
||||||
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx);
|
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
|
||||||
if (config_stats || (config_prof && opt_prof))
|
if (config_stats || (config_prof && opt_prof))
|
||||||
usize = isalloc(tsd_tsdn(tsd), extent, ptr);
|
usize = isalloc(tsd_tsdn(tsd), extent, ptr);
|
||||||
/* Remove huge allocation from prof sample set. */
|
/* Remove large allocation from prof sample set. */
|
||||||
if (config_prof && opt_prof)
|
if (config_prof && opt_prof)
|
||||||
prof_free(tsd, extent, ptr, usize);
|
prof_free(tsd, extent, ptr, usize);
|
||||||
huge_dalloc(tsd_tsdn(tsd), extent);
|
large_dalloc(tsd_tsdn(tsd), extent);
|
||||||
malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx);
|
malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
|
||||||
/* Cancel out unwanted effects on stats. */
|
/* Cancel out unwanted effects on stats. */
|
||||||
if (config_stats)
|
if (config_stats)
|
||||||
arena_huge_reset_stats_cancel(arena, usize);
|
arena_large_reset_stats_cancel(arena, usize);
|
||||||
}
|
}
|
||||||
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx);
|
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
|
||||||
|
|
||||||
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
|
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
|
||||||
|
|
||||||
@ -1283,7 +1284,7 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
|
|||||||
|
|
||||||
if (likely(size <= SMALL_MAXCLASS))
|
if (likely(size <= SMALL_MAXCLASS))
|
||||||
return (arena_malloc_small(tsdn, arena, ind, zero));
|
return (arena_malloc_small(tsdn, arena, ind, zero));
|
||||||
return (huge_malloc(tsdn, arena, index2size(ind), zero));
|
return (large_malloc(tsdn, arena, index2size(ind), zero));
|
||||||
}
|
}
|
||||||
|
|
||||||
void *
|
void *
|
||||||
@ -1299,9 +1300,9 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||||||
tcache, true);
|
tcache, true);
|
||||||
} else {
|
} else {
|
||||||
if (likely(alignment <= CACHELINE))
|
if (likely(alignment <= CACHELINE))
|
||||||
ret = huge_malloc(tsdn, arena, usize, zero);
|
ret = large_malloc(tsdn, arena, usize, zero);
|
||||||
else
|
else
|
||||||
ret = huge_palloc(tsdn, arena, usize, alignment, zero);
|
ret = large_palloc(tsdn, arena, usize, alignment, zero);
|
||||||
}
|
}
|
||||||
return (ret);
|
return (ret);
|
||||||
}
|
}
|
||||||
@ -1360,10 +1361,10 @@ arena_dalloc_promoted(tsdn_t *tsdn, extent_t *extent, void *ptr,
|
|||||||
|
|
||||||
usize = arena_prof_demote(tsdn, extent, ptr);
|
usize = arena_prof_demote(tsdn, extent, ptr);
|
||||||
if (usize <= tcache_maxclass) {
|
if (usize <= tcache_maxclass) {
|
||||||
tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, ptr, usize,
|
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, usize,
|
||||||
slow_path);
|
slow_path);
|
||||||
} else
|
} else
|
||||||
huge_dalloc(tsdn, extent);
|
large_dalloc(tsdn, extent);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
@ -1493,9 +1494,9 @@ arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
|
|||||||
size_t usize_min, usize_max;
|
size_t usize_min, usize_max;
|
||||||
|
|
||||||
/* Calls with non-zero extra had to clamp extra. */
|
/* Calls with non-zero extra had to clamp extra. */
|
||||||
assert(extra == 0 || size + extra <= HUGE_MAXCLASS);
|
assert(extra == 0 || size + extra <= LARGE_MAXCLASS);
|
||||||
|
|
||||||
if (unlikely(size > HUGE_MAXCLASS))
|
if (unlikely(size > LARGE_MAXCLASS))
|
||||||
return (true);
|
return (true);
|
||||||
|
|
||||||
usize_min = s2u(size);
|
usize_min = s2u(size);
|
||||||
@ -1515,7 +1516,7 @@ arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
|
|||||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||||
return (false);
|
return (false);
|
||||||
} else if (oldsize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS) {
|
} else if (oldsize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS) {
|
||||||
return (huge_ralloc_no_move(tsdn, extent, usize_min, usize_max,
|
return (large_ralloc_no_move(tsdn, extent, usize_min, usize_max,
|
||||||
zero));
|
zero));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1531,7 +1532,7 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
|||||||
return (arena_malloc(tsdn, arena, usize, size2index(usize),
|
return (arena_malloc(tsdn, arena, usize, size2index(usize),
|
||||||
zero, tcache, true));
|
zero, tcache, true));
|
||||||
usize = sa2u(usize, alignment);
|
usize = sa2u(usize, alignment);
|
||||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
|
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
|
||||||
return (NULL);
|
return (NULL);
|
||||||
return (ipalloct(tsdn, usize, alignment, zero, tcache, arena));
|
return (ipalloct(tsdn, usize, alignment, zero, tcache, arena));
|
||||||
}
|
}
|
||||||
@ -1544,7 +1545,7 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
|
|||||||
size_t usize, copysize;
|
size_t usize, copysize;
|
||||||
|
|
||||||
usize = s2u(size);
|
usize = s2u(size);
|
||||||
if (unlikely(usize == 0 || size > HUGE_MAXCLASS))
|
if (unlikely(usize == 0 || size > LARGE_MAXCLASS))
|
||||||
return (NULL);
|
return (NULL);
|
||||||
|
|
||||||
if (likely(usize <= SMALL_MAXCLASS)) {
|
if (likely(usize <= SMALL_MAXCLASS)) {
|
||||||
@ -1555,8 +1556,8 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (oldsize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS) {
|
if (oldsize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS) {
|
||||||
return (huge_ralloc(tsdn, arena, extent, usize, alignment, zero,
|
return (large_ralloc(tsdn, arena, extent, usize, alignment,
|
||||||
tcache));
|
zero, tcache));
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -1670,7 +1671,7 @@ void
|
|||||||
arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
|
arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
|
||||||
const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time,
|
const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time,
|
||||||
size_t *nactive, size_t *ndirty, arena_stats_t *astats,
|
size_t *nactive, size_t *ndirty, arena_stats_t *astats,
|
||||||
malloc_bin_stats_t *bstats, malloc_huge_stats_t *hstats)
|
malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats)
|
||||||
{
|
{
|
||||||
unsigned i;
|
unsigned i;
|
||||||
|
|
||||||
@ -1687,16 +1688,16 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
|
|||||||
astats->purged += arena->stats.purged;
|
astats->purged += arena->stats.purged;
|
||||||
astats->metadata_mapped += arena->stats.metadata_mapped;
|
astats->metadata_mapped += arena->stats.metadata_mapped;
|
||||||
astats->metadata_allocated += arena_metadata_allocated_get(arena);
|
astats->metadata_allocated += arena_metadata_allocated_get(arena);
|
||||||
astats->allocated_huge += arena->stats.allocated_huge;
|
astats->allocated_large += arena->stats.allocated_large;
|
||||||
astats->nmalloc_huge += arena->stats.nmalloc_huge;
|
astats->nmalloc_large += arena->stats.nmalloc_large;
|
||||||
astats->ndalloc_huge += arena->stats.ndalloc_huge;
|
astats->ndalloc_large += arena->stats.ndalloc_large;
|
||||||
astats->nrequests_huge += arena->stats.nrequests_huge;
|
astats->nrequests_large += arena->stats.nrequests_large;
|
||||||
|
|
||||||
for (i = 0; i < NSIZES - NBINS; i++) {
|
for (i = 0; i < NSIZES - NBINS; i++) {
|
||||||
hstats[i].nmalloc += arena->stats.hstats[i].nmalloc;
|
lstats[i].nmalloc += arena->stats.lstats[i].nmalloc;
|
||||||
hstats[i].ndalloc += arena->stats.hstats[i].ndalloc;
|
lstats[i].ndalloc += arena->stats.lstats[i].ndalloc;
|
||||||
hstats[i].nrequests += arena->stats.hstats[i].nrequests;
|
lstats[i].nrequests += arena->stats.lstats[i].nrequests;
|
||||||
hstats[i].curhchunks += arena->stats.hstats[i].curhchunks;
|
lstats[i].curlextents += arena->stats.lstats[i].curlextents;
|
||||||
}
|
}
|
||||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||||
|
|
||||||
@ -1786,9 +1787,9 @@ arena_new(tsdn_t *tsdn, unsigned ind)
|
|||||||
if (opt_purge == purge_mode_decay)
|
if (opt_purge == purge_mode_decay)
|
||||||
arena_decay_init(arena, arena_decay_time_default_get());
|
arena_decay_init(arena, arena_decay_time_default_get());
|
||||||
|
|
||||||
ql_new(&arena->huge);
|
ql_new(&arena->large);
|
||||||
if (malloc_mutex_init(&arena->huge_mtx, "arena_huge",
|
if (malloc_mutex_init(&arena->large_mtx, "arena_large",
|
||||||
WITNESS_RANK_ARENA_HUGE))
|
WITNESS_RANK_ARENA_LARGE))
|
||||||
return (NULL);
|
return (NULL);
|
||||||
|
|
||||||
for (i = 0; i < NPSIZES; i++) {
|
for (i = 0; i < NPSIZES; i++) {
|
||||||
@ -1859,7 +1860,7 @@ arena_prefork3(tsdn_t *tsdn, arena_t *arena)
|
|||||||
|
|
||||||
for (i = 0; i < NBINS; i++)
|
for (i = 0; i < NBINS; i++)
|
||||||
malloc_mutex_prefork(tsdn, &arena->bins[i].lock);
|
malloc_mutex_prefork(tsdn, &arena->bins[i].lock);
|
||||||
malloc_mutex_prefork(tsdn, &arena->huge_mtx);
|
malloc_mutex_prefork(tsdn, &arena->large_mtx);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
@ -1867,7 +1868,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena)
|
|||||||
{
|
{
|
||||||
unsigned i;
|
unsigned i;
|
||||||
|
|
||||||
malloc_mutex_postfork_parent(tsdn, &arena->huge_mtx);
|
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
|
||||||
for (i = 0; i < NBINS; i++)
|
for (i = 0; i < NBINS; i++)
|
||||||
malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock);
|
malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock);
|
||||||
malloc_mutex_postfork_parent(tsdn, &arena->extent_cache_mtx);
|
malloc_mutex_postfork_parent(tsdn, &arena->extent_cache_mtx);
|
||||||
@ -1880,7 +1881,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena)
|
|||||||
{
|
{
|
||||||
unsigned i;
|
unsigned i;
|
||||||
|
|
||||||
malloc_mutex_postfork_child(tsdn, &arena->huge_mtx);
|
malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
|
||||||
for (i = 0; i < NBINS; i++)
|
for (i = 0; i < NBINS; i++)
|
||||||
malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock);
|
malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock);
|
||||||
malloc_mutex_postfork_child(tsdn, &arena->extent_cache_mtx);
|
malloc_mutex_postfork_child(tsdn, &arena->extent_cache_mtx);
|
||||||
|
@ -78,7 +78,7 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
|||||||
|
|
||||||
/*
|
/*
|
||||||
* sbrk() uses a signed increment argument, so take care not to
|
* sbrk() uses a signed increment argument, so take care not to
|
||||||
* interpret a huge allocation request as a negative increment.
|
* interpret a large allocation request as a negative increment.
|
||||||
*/
|
*/
|
||||||
if ((intptr_t)size < 0)
|
if ((intptr_t)size < 0)
|
||||||
return (NULL);
|
return (NULL);
|
||||||
|
@ -267,7 +267,7 @@ ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
|
|||||||
|
|
||||||
lg_curcells++;
|
lg_curcells++;
|
||||||
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
|
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
|
||||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
|
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
|
||||||
ret = true;
|
ret = true;
|
||||||
goto label_return;
|
goto label_return;
|
||||||
}
|
}
|
||||||
@ -315,7 +315,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
|
|||||||
lg_prevbuckets = ckh->lg_curbuckets;
|
lg_prevbuckets = ckh->lg_curbuckets;
|
||||||
lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1;
|
lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1;
|
||||||
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
|
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
|
||||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
|
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
|
||||||
return;
|
return;
|
||||||
tab = (ckhc_t *)ipallocztm(tsdn, usize, CACHELINE, true, NULL, true,
|
tab = (ckhc_t *)ipallocztm(tsdn, usize, CACHELINE, true, NULL, true,
|
||||||
arena_ichoose(tsdn, NULL));
|
arena_ichoose(tsdn, NULL));
|
||||||
@ -390,7 +390,7 @@ ckh_new(tsdn_t *tsdn, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
|
|||||||
ckh->keycomp = keycomp;
|
ckh->keycomp = keycomp;
|
||||||
|
|
||||||
usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE);
|
usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE);
|
||||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
|
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
|
||||||
ret = true;
|
ret = true;
|
||||||
goto label_return;
|
goto label_return;
|
||||||
}
|
}
|
||||||
|
146
src/ctl.c
146
src/ctl.c
@ -126,8 +126,8 @@ CTL_PROTO(arenas_bin_i_size)
|
|||||||
CTL_PROTO(arenas_bin_i_nregs)
|
CTL_PROTO(arenas_bin_i_nregs)
|
||||||
CTL_PROTO(arenas_bin_i_slab_size)
|
CTL_PROTO(arenas_bin_i_slab_size)
|
||||||
INDEX_PROTO(arenas_bin_i)
|
INDEX_PROTO(arenas_bin_i)
|
||||||
CTL_PROTO(arenas_hchunk_i_size)
|
CTL_PROTO(arenas_lextent_i_size)
|
||||||
INDEX_PROTO(arenas_hchunk_i)
|
INDEX_PROTO(arenas_lextent_i)
|
||||||
CTL_PROTO(arenas_narenas)
|
CTL_PROTO(arenas_narenas)
|
||||||
CTL_PROTO(arenas_initialized)
|
CTL_PROTO(arenas_initialized)
|
||||||
CTL_PROTO(arenas_lg_dirty_mult)
|
CTL_PROTO(arenas_lg_dirty_mult)
|
||||||
@ -137,7 +137,7 @@ CTL_PROTO(arenas_page)
|
|||||||
CTL_PROTO(arenas_tcache_max)
|
CTL_PROTO(arenas_tcache_max)
|
||||||
CTL_PROTO(arenas_nbins)
|
CTL_PROTO(arenas_nbins)
|
||||||
CTL_PROTO(arenas_nhbins)
|
CTL_PROTO(arenas_nhbins)
|
||||||
CTL_PROTO(arenas_nhchunks)
|
CTL_PROTO(arenas_nlextents)
|
||||||
CTL_PROTO(arenas_extend)
|
CTL_PROTO(arenas_extend)
|
||||||
CTL_PROTO(prof_thread_active_init)
|
CTL_PROTO(prof_thread_active_init)
|
||||||
CTL_PROTO(prof_active)
|
CTL_PROTO(prof_active)
|
||||||
@ -150,10 +150,10 @@ CTL_PROTO(stats_arenas_i_small_allocated)
|
|||||||
CTL_PROTO(stats_arenas_i_small_nmalloc)
|
CTL_PROTO(stats_arenas_i_small_nmalloc)
|
||||||
CTL_PROTO(stats_arenas_i_small_ndalloc)
|
CTL_PROTO(stats_arenas_i_small_ndalloc)
|
||||||
CTL_PROTO(stats_arenas_i_small_nrequests)
|
CTL_PROTO(stats_arenas_i_small_nrequests)
|
||||||
CTL_PROTO(stats_arenas_i_huge_allocated)
|
CTL_PROTO(stats_arenas_i_large_allocated)
|
||||||
CTL_PROTO(stats_arenas_i_huge_nmalloc)
|
CTL_PROTO(stats_arenas_i_large_nmalloc)
|
||||||
CTL_PROTO(stats_arenas_i_huge_ndalloc)
|
CTL_PROTO(stats_arenas_i_large_ndalloc)
|
||||||
CTL_PROTO(stats_arenas_i_huge_nrequests)
|
CTL_PROTO(stats_arenas_i_large_nrequests)
|
||||||
CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
|
CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
|
||||||
CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
|
CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
|
||||||
CTL_PROTO(stats_arenas_i_bins_j_nrequests)
|
CTL_PROTO(stats_arenas_i_bins_j_nrequests)
|
||||||
@ -164,11 +164,11 @@ CTL_PROTO(stats_arenas_i_bins_j_nslabs)
|
|||||||
CTL_PROTO(stats_arenas_i_bins_j_nreslabs)
|
CTL_PROTO(stats_arenas_i_bins_j_nreslabs)
|
||||||
CTL_PROTO(stats_arenas_i_bins_j_curslabs)
|
CTL_PROTO(stats_arenas_i_bins_j_curslabs)
|
||||||
INDEX_PROTO(stats_arenas_i_bins_j)
|
INDEX_PROTO(stats_arenas_i_bins_j)
|
||||||
CTL_PROTO(stats_arenas_i_hchunks_j_nmalloc)
|
CTL_PROTO(stats_arenas_i_lextents_j_nmalloc)
|
||||||
CTL_PROTO(stats_arenas_i_hchunks_j_ndalloc)
|
CTL_PROTO(stats_arenas_i_lextents_j_ndalloc)
|
||||||
CTL_PROTO(stats_arenas_i_hchunks_j_nrequests)
|
CTL_PROTO(stats_arenas_i_lextents_j_nrequests)
|
||||||
CTL_PROTO(stats_arenas_i_hchunks_j_curhchunks)
|
CTL_PROTO(stats_arenas_i_lextents_j_curlextents)
|
||||||
INDEX_PROTO(stats_arenas_i_hchunks_j)
|
INDEX_PROTO(stats_arenas_i_lextents_j)
|
||||||
CTL_PROTO(stats_arenas_i_nthreads)
|
CTL_PROTO(stats_arenas_i_nthreads)
|
||||||
CTL_PROTO(stats_arenas_i_dss)
|
CTL_PROTO(stats_arenas_i_dss)
|
||||||
CTL_PROTO(stats_arenas_i_lg_dirty_mult)
|
CTL_PROTO(stats_arenas_i_lg_dirty_mult)
|
||||||
@ -310,15 +310,15 @@ static const ctl_indexed_node_t arenas_bin_node[] = {
|
|||||||
{INDEX(arenas_bin_i)}
|
{INDEX(arenas_bin_i)}
|
||||||
};
|
};
|
||||||
|
|
||||||
static const ctl_named_node_t arenas_hchunk_i_node[] = {
|
static const ctl_named_node_t arenas_lextent_i_node[] = {
|
||||||
{NAME("size"), CTL(arenas_hchunk_i_size)}
|
{NAME("size"), CTL(arenas_lextent_i_size)}
|
||||||
};
|
};
|
||||||
static const ctl_named_node_t super_arenas_hchunk_i_node[] = {
|
static const ctl_named_node_t super_arenas_lextent_i_node[] = {
|
||||||
{NAME(""), CHILD(named, arenas_hchunk_i)}
|
{NAME(""), CHILD(named, arenas_lextent_i)}
|
||||||
};
|
};
|
||||||
|
|
||||||
static const ctl_indexed_node_t arenas_hchunk_node[] = {
|
static const ctl_indexed_node_t arenas_lextent_node[] = {
|
||||||
{INDEX(arenas_hchunk_i)}
|
{INDEX(arenas_lextent_i)}
|
||||||
};
|
};
|
||||||
|
|
||||||
static const ctl_named_node_t arenas_node[] = {
|
static const ctl_named_node_t arenas_node[] = {
|
||||||
@ -332,8 +332,8 @@ static const ctl_named_node_t arenas_node[] = {
|
|||||||
{NAME("nbins"), CTL(arenas_nbins)},
|
{NAME("nbins"), CTL(arenas_nbins)},
|
||||||
{NAME("nhbins"), CTL(arenas_nhbins)},
|
{NAME("nhbins"), CTL(arenas_nhbins)},
|
||||||
{NAME("bin"), CHILD(indexed, arenas_bin)},
|
{NAME("bin"), CHILD(indexed, arenas_bin)},
|
||||||
{NAME("nhchunks"), CTL(arenas_nhchunks)},
|
{NAME("nlextents"), CTL(arenas_nlextents)},
|
||||||
{NAME("hchunk"), CHILD(indexed, arenas_hchunk)},
|
{NAME("lextent"), CHILD(indexed, arenas_lextent)},
|
||||||
{NAME("extend"), CTL(arenas_extend)}
|
{NAME("extend"), CTL(arenas_extend)}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -359,11 +359,11 @@ static const ctl_named_node_t stats_arenas_i_small_node[] = {
|
|||||||
{NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}
|
{NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}
|
||||||
};
|
};
|
||||||
|
|
||||||
static const ctl_named_node_t stats_arenas_i_huge_node[] = {
|
static const ctl_named_node_t stats_arenas_i_large_node[] = {
|
||||||
{NAME("allocated"), CTL(stats_arenas_i_huge_allocated)},
|
{NAME("allocated"), CTL(stats_arenas_i_large_allocated)},
|
||||||
{NAME("nmalloc"), CTL(stats_arenas_i_huge_nmalloc)},
|
{NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)},
|
||||||
{NAME("ndalloc"), CTL(stats_arenas_i_huge_ndalloc)},
|
{NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)},
|
||||||
{NAME("nrequests"), CTL(stats_arenas_i_huge_nrequests)}
|
{NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}
|
||||||
};
|
};
|
||||||
|
|
||||||
static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
|
static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
|
||||||
@ -385,18 +385,18 @@ static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {
|
|||||||
{INDEX(stats_arenas_i_bins_j)}
|
{INDEX(stats_arenas_i_bins_j)}
|
||||||
};
|
};
|
||||||
|
|
||||||
static const ctl_named_node_t stats_arenas_i_hchunks_j_node[] = {
|
static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = {
|
||||||
{NAME("nmalloc"), CTL(stats_arenas_i_hchunks_j_nmalloc)},
|
{NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)},
|
||||||
{NAME("ndalloc"), CTL(stats_arenas_i_hchunks_j_ndalloc)},
|
{NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)},
|
||||||
{NAME("nrequests"), CTL(stats_arenas_i_hchunks_j_nrequests)},
|
{NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)},
|
||||||
{NAME("curhchunks"), CTL(stats_arenas_i_hchunks_j_curhchunks)}
|
{NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)}
|
||||||
};
|
};
|
||||||
static const ctl_named_node_t super_stats_arenas_i_hchunks_j_node[] = {
|
static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = {
|
||||||
{NAME(""), CHILD(named, stats_arenas_i_hchunks_j)}
|
{NAME(""), CHILD(named, stats_arenas_i_lextents_j)}
|
||||||
};
|
};
|
||||||
|
|
||||||
static const ctl_indexed_node_t stats_arenas_i_hchunks_node[] = {
|
static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
|
||||||
{INDEX(stats_arenas_i_hchunks_j)}
|
{INDEX(stats_arenas_i_lextents_j)}
|
||||||
};
|
};
|
||||||
|
|
||||||
static const ctl_named_node_t stats_arenas_i_node[] = {
|
static const ctl_named_node_t stats_arenas_i_node[] = {
|
||||||
@ -413,9 +413,9 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
|
|||||||
{NAME("purged"), CTL(stats_arenas_i_purged)},
|
{NAME("purged"), CTL(stats_arenas_i_purged)},
|
||||||
{NAME("metadata"), CHILD(named, stats_arenas_i_metadata)},
|
{NAME("metadata"), CHILD(named, stats_arenas_i_metadata)},
|
||||||
{NAME("small"), CHILD(named, stats_arenas_i_small)},
|
{NAME("small"), CHILD(named, stats_arenas_i_small)},
|
||||||
{NAME("huge"), CHILD(named, stats_arenas_i_huge)},
|
{NAME("large"), CHILD(named, stats_arenas_i_large)},
|
||||||
{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
|
{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
|
||||||
{NAME("hchunks"), CHILD(indexed, stats_arenas_i_hchunks)}
|
{NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}
|
||||||
};
|
};
|
||||||
static const ctl_named_node_t super_stats_arenas_i_node[] = {
|
static const ctl_named_node_t super_stats_arenas_i_node[] = {
|
||||||
{NAME(""), CHILD(named, stats_arenas_i)}
|
{NAME(""), CHILD(named, stats_arenas_i)}
|
||||||
@ -476,8 +476,8 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
|
|||||||
astats->ndalloc_small = 0;
|
astats->ndalloc_small = 0;
|
||||||
astats->nrequests_small = 0;
|
astats->nrequests_small = 0;
|
||||||
memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t));
|
memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t));
|
||||||
memset(astats->hstats, 0, (NSIZES - NBINS) *
|
memset(astats->lstats, 0, (NSIZES - NBINS) *
|
||||||
sizeof(malloc_huge_stats_t));
|
sizeof(malloc_large_stats_t));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -490,7 +490,7 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_stats_t *cstats, arena_t *arena)
|
|||||||
arena_stats_merge(tsdn, arena, &cstats->nthreads, &cstats->dss,
|
arena_stats_merge(tsdn, arena, &cstats->nthreads, &cstats->dss,
|
||||||
&cstats->lg_dirty_mult, &cstats->decay_time,
|
&cstats->lg_dirty_mult, &cstats->decay_time,
|
||||||
&cstats->pactive, &cstats->pdirty, &cstats->astats,
|
&cstats->pactive, &cstats->pdirty, &cstats->astats,
|
||||||
cstats->bstats, cstats->hstats);
|
cstats->bstats, cstats->lstats);
|
||||||
|
|
||||||
for (i = 0; i < NBINS; i++) {
|
for (i = 0; i < NBINS; i++) {
|
||||||
cstats->allocated_small += cstats->bstats[i].curregs *
|
cstats->allocated_small += cstats->bstats[i].curregs *
|
||||||
@ -532,10 +532,12 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
|
|||||||
sstats->ndalloc_small += astats->ndalloc_small;
|
sstats->ndalloc_small += astats->ndalloc_small;
|
||||||
sstats->nrequests_small += astats->nrequests_small;
|
sstats->nrequests_small += astats->nrequests_small;
|
||||||
|
|
||||||
sstats->astats.allocated_huge += astats->astats.allocated_huge;
|
sstats->astats.allocated_large +=
|
||||||
sstats->astats.nmalloc_huge += astats->astats.nmalloc_huge;
|
astats->astats.allocated_large;
|
||||||
sstats->astats.ndalloc_huge += astats->astats.ndalloc_huge;
|
sstats->astats.nmalloc_large += astats->astats.nmalloc_large;
|
||||||
sstats->astats.nrequests_huge += astats->astats.nrequests_huge;
|
sstats->astats.ndalloc_large += astats->astats.ndalloc_large;
|
||||||
|
sstats->astats.nrequests_large +=
|
||||||
|
astats->astats.nrequests_large;
|
||||||
|
|
||||||
for (i = 0; i < NBINS; i++) {
|
for (i = 0; i < NBINS; i++) {
|
||||||
sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
|
sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
|
||||||
@ -556,12 +558,12 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (i = 0; i < NSIZES - NBINS; i++) {
|
for (i = 0; i < NSIZES - NBINS; i++) {
|
||||||
sstats->hstats[i].nmalloc += astats->hstats[i].nmalloc;
|
sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc;
|
||||||
sstats->hstats[i].ndalloc += astats->hstats[i].ndalloc;
|
sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc;
|
||||||
sstats->hstats[i].nrequests +=
|
sstats->lstats[i].nrequests +=
|
||||||
astats->hstats[i].nrequests;
|
astats->lstats[i].nrequests;
|
||||||
sstats->hstats[i].curhchunks +=
|
sstats->lstats[i].curlextents +=
|
||||||
astats->hstats[i].curhchunks;
|
astats->lstats[i].curlextents;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -643,7 +645,7 @@ ctl_refresh(tsdn_t *tsdn)
|
|||||||
&base_mapped);
|
&base_mapped);
|
||||||
ctl_stats.allocated =
|
ctl_stats.allocated =
|
||||||
ctl_stats.arenas[ctl_stats.narenas].allocated_small +
|
ctl_stats.arenas[ctl_stats.narenas].allocated_small +
|
||||||
ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge;
|
ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large;
|
||||||
ctl_stats.active =
|
ctl_stats.active =
|
||||||
(ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE);
|
(ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE);
|
||||||
ctl_stats.metadata = base_allocated +
|
ctl_stats.metadata = base_allocated +
|
||||||
@ -1812,15 +1814,15 @@ arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
|
|||||||
return (super_arenas_bin_i_node);
|
return (super_arenas_bin_i_node);
|
||||||
}
|
}
|
||||||
|
|
||||||
CTL_RO_NL_GEN(arenas_nhchunks, NSIZES - NBINS, unsigned)
|
CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned)
|
||||||
CTL_RO_NL_GEN(arenas_hchunk_i_size, index2size(NBINS+(szind_t)mib[2]), size_t)
|
CTL_RO_NL_GEN(arenas_lextent_i_size, index2size(NBINS+(szind_t)mib[2]), size_t)
|
||||||
static const ctl_named_node_t *
|
static const ctl_named_node_t *
|
||||||
arenas_hchunk_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
|
arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (i > NSIZES - NBINS)
|
if (i > NSIZES - NBINS)
|
||||||
return (NULL);
|
return (NULL);
|
||||||
return (super_arenas_hchunk_i_node);
|
return (super_arenas_lextent_i_node);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
@ -2012,14 +2014,14 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc,
|
|||||||
ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t)
|
ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests,
|
||||||
ctl_stats.arenas[mib[2]].nrequests_small, uint64_t)
|
ctl_stats.arenas[mib[2]].nrequests_small, uint64_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_huge_allocated,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated,
|
||||||
ctl_stats.arenas[mib[2]].astats.allocated_huge, size_t)
|
ctl_stats.arenas[mib[2]].astats.allocated_large, size_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nmalloc,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc,
|
||||||
ctl_stats.arenas[mib[2]].astats.nmalloc_huge, uint64_t)
|
ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_huge_ndalloc,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc,
|
||||||
ctl_stats.arenas[mib[2]].astats.ndalloc_huge, uint64_t)
|
ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nrequests,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests,
|
||||||
ctl_stats.arenas[mib[2]].astats.nmalloc_huge, uint64_t) /* Intentional. */
|
ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t) /* Intentional. */
|
||||||
|
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc,
|
||||||
ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t)
|
ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t)
|
||||||
@ -2050,23 +2052,23 @@ stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
|
|||||||
return (super_stats_arenas_i_bins_j_node);
|
return (super_stats_arenas_i_bins_j_node);
|
||||||
}
|
}
|
||||||
|
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_nmalloc,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc,
|
||||||
ctl_stats.arenas[mib[2]].hstats[mib[4]].nmalloc, uint64_t)
|
ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_ndalloc,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc,
|
||||||
ctl_stats.arenas[mib[2]].hstats[mib[4]].ndalloc, uint64_t)
|
ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_nrequests,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests,
|
||||||
ctl_stats.arenas[mib[2]].hstats[mib[4]].nrequests, uint64_t)
|
ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_curhchunks,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents,
|
||||||
ctl_stats.arenas[mib[2]].hstats[mib[4]].curhchunks, size_t)
|
ctl_stats.arenas[mib[2]].lstats[mib[4]].curlextents, size_t)
|
||||||
|
|
||||||
static const ctl_named_node_t *
|
static const ctl_named_node_t *
|
||||||
stats_arenas_i_hchunks_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
|
stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
|
||||||
size_t j)
|
size_t j)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (j > NSIZES - NBINS)
|
if (j > NSIZES - NBINS)
|
||||||
return (NULL);
|
return (NULL);
|
||||||
return (super_stats_arenas_i_hchunks_j_node);
|
return (super_stats_arenas_i_lextents_j_node);
|
||||||
}
|
}
|
||||||
|
|
||||||
static const ctl_named_node_t *
|
static const ctl_named_node_t *
|
||||||
|
@ -40,7 +40,7 @@ extent_size_quantize_floor(size_t size)
|
|||||||
pszind_t pind;
|
pszind_t pind;
|
||||||
|
|
||||||
assert(size > 0);
|
assert(size > 0);
|
||||||
assert(size - large_pad <= HUGE_MAXCLASS);
|
assert(size - large_pad <= LARGE_MAXCLASS);
|
||||||
assert((size & PAGE_MASK) == 0);
|
assert((size & PAGE_MASK) == 0);
|
||||||
|
|
||||||
assert(size != 0);
|
assert(size != 0);
|
||||||
@ -77,7 +77,7 @@ extent_size_quantize_ceil(size_t size)
|
|||||||
size_t ret;
|
size_t ret;
|
||||||
|
|
||||||
assert(size > 0);
|
assert(size > 0);
|
||||||
assert(size - large_pad <= HUGE_MAXCLASS);
|
assert(size - large_pad <= LARGE_MAXCLASS);
|
||||||
assert((size & PAGE_MASK) == 0);
|
assert((size & PAGE_MASK) == 0);
|
||||||
|
|
||||||
ret = extent_size_quantize_floor(size);
|
ret = extent_size_quantize_floor(size);
|
||||||
|
@ -1457,7 +1457,7 @@ ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize,
|
|||||||
|
|
||||||
if (config_stats || (config_prof && opt_prof)) {
|
if (config_stats || (config_prof && opt_prof)) {
|
||||||
*usize = index2size(ind);
|
*usize = index2size(ind);
|
||||||
assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
|
assert(*usize > 0 && *usize <= LARGE_MAXCLASS);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config_prof && opt_prof)
|
if (config_prof && opt_prof)
|
||||||
@ -1589,7 +1589,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
|
|||||||
}
|
}
|
||||||
|
|
||||||
usize = sa2u(size, alignment);
|
usize = sa2u(size, alignment);
|
||||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
|
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
|
||||||
result = NULL;
|
result = NULL;
|
||||||
goto label_oom;
|
goto label_oom;
|
||||||
}
|
}
|
||||||
@ -1663,7 +1663,7 @@ je_calloc(size_t num, size_t size)
|
|||||||
if (num == 0 || size == 0)
|
if (num == 0 || size == 0)
|
||||||
num_size = 1;
|
num_size = 1;
|
||||||
else
|
else
|
||||||
num_size = HUGE_MAXCLASS + 1; /* Trigger OOM. */
|
num_size = LARGE_MAXCLASS + 1; /* Trigger OOM. */
|
||||||
/*
|
/*
|
||||||
* Try to avoid division here. We know that it isn't possible to
|
* Try to avoid division here. We know that it isn't possible to
|
||||||
* overflow during multiplication if neither operand uses any of the
|
* overflow during multiplication if neither operand uses any of the
|
||||||
@ -1671,7 +1671,7 @@ je_calloc(size_t num, size_t size)
|
|||||||
*/
|
*/
|
||||||
} else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) <<
|
} else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) <<
|
||||||
2))) && (num_size / size != num)))
|
2))) && (num_size / size != num)))
|
||||||
num_size = HUGE_MAXCLASS + 1; /* size_t overflow. */
|
num_size = LARGE_MAXCLASS + 1; /* size_t overflow. */
|
||||||
|
|
||||||
if (likely(!malloc_slow)) {
|
if (likely(!malloc_slow)) {
|
||||||
ret = ialloc_body(num_size, true, &tsdn, &usize, false);
|
ret = ialloc_body(num_size, true, &tsdn, &usize, false);
|
||||||
@ -1819,7 +1819,7 @@ je_realloc(void *ptr, size_t size)
|
|||||||
old_usize = isalloc(tsd_tsdn(tsd), extent, ptr);
|
old_usize = isalloc(tsd_tsdn(tsd), extent, ptr);
|
||||||
if (config_prof && opt_prof) {
|
if (config_prof && opt_prof) {
|
||||||
usize = s2u(size);
|
usize = s2u(size);
|
||||||
ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ?
|
ret = unlikely(usize == 0 || usize > LARGE_MAXCLASS) ?
|
||||||
NULL : irealloc_prof(tsd, extent, ptr, old_usize,
|
NULL : irealloc_prof(tsd, extent, ptr, old_usize,
|
||||||
usize);
|
usize);
|
||||||
} else {
|
} else {
|
||||||
@ -1956,7 +1956,7 @@ imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize,
|
|||||||
*alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
|
*alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
|
||||||
*usize = sa2u(size, *alignment);
|
*usize = sa2u(size, *alignment);
|
||||||
}
|
}
|
||||||
if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS))
|
if (unlikely(*usize == 0 || *usize > LARGE_MAXCLASS))
|
||||||
return (true);
|
return (true);
|
||||||
*zero = MALLOCX_ZERO_GET(flags);
|
*zero = MALLOCX_ZERO_GET(flags);
|
||||||
if ((flags & MALLOCX_TCACHE_MASK) != 0) {
|
if ((flags & MALLOCX_TCACHE_MASK) != 0) {
|
||||||
@ -2084,7 +2084,7 @@ imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize,
|
|||||||
return (NULL);
|
return (NULL);
|
||||||
if (config_stats || (config_prof && opt_prof)) {
|
if (config_stats || (config_prof && opt_prof)) {
|
||||||
*usize = index2size(ind);
|
*usize = index2size(ind);
|
||||||
assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
|
assert(*usize > 0 && *usize <= LARGE_MAXCLASS);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config_prof && opt_prof) {
|
if (config_prof && opt_prof) {
|
||||||
@ -2233,7 +2233,7 @@ je_rallocx(void *ptr, size_t size, int flags)
|
|||||||
|
|
||||||
if (config_prof && opt_prof) {
|
if (config_prof && opt_prof) {
|
||||||
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
|
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
|
||||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
|
if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
|
||||||
goto label_oom;
|
goto label_oom;
|
||||||
p = irallocx_prof(tsd, extent, ptr, old_usize, size, alignment,
|
p = irallocx_prof(tsd, extent, ptr, old_usize, size, alignment,
|
||||||
&usize, zero, tcache, arena);
|
&usize, zero, tcache, arena);
|
||||||
@ -2314,17 +2314,17 @@ ixallocx_prof(tsd_t *tsd, extent_t *extent, void *ptr, size_t old_usize,
|
|||||||
*/
|
*/
|
||||||
if (alignment == 0) {
|
if (alignment == 0) {
|
||||||
usize_max = s2u(size+extra);
|
usize_max = s2u(size+extra);
|
||||||
assert(usize_max > 0 && usize_max <= HUGE_MAXCLASS);
|
assert(usize_max > 0 && usize_max <= LARGE_MAXCLASS);
|
||||||
} else {
|
} else {
|
||||||
usize_max = sa2u(size+extra, alignment);
|
usize_max = sa2u(size+extra, alignment);
|
||||||
if (unlikely(usize_max == 0 || usize_max > HUGE_MAXCLASS)) {
|
if (unlikely(usize_max == 0 || usize_max > LARGE_MAXCLASS)) {
|
||||||
/*
|
/*
|
||||||
* usize_max is out of range, and chances are that
|
* usize_max is out of range, and chances are that
|
||||||
* allocation will fail, but use the maximum possible
|
* allocation will fail, but use the maximum possible
|
||||||
* value and carry on with prof_alloc_prep(), just in
|
* value and carry on with prof_alloc_prep(), just in
|
||||||
* case allocation succeeds.
|
* case allocation succeeds.
|
||||||
*/
|
*/
|
||||||
usize_max = HUGE_MAXCLASS;
|
usize_max = LARGE_MAXCLASS;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
|
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
|
||||||
@ -2368,18 +2368,18 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
|
|||||||
/*
|
/*
|
||||||
* The API explicitly absolves itself of protecting against (size +
|
* The API explicitly absolves itself of protecting against (size +
|
||||||
* extra) numerical overflow, but we may need to clamp extra to avoid
|
* extra) numerical overflow, but we may need to clamp extra to avoid
|
||||||
* exceeding HUGE_MAXCLASS.
|
* exceeding LARGE_MAXCLASS.
|
||||||
*
|
*
|
||||||
* Ordinarily, size limit checking is handled deeper down, but here we
|
* Ordinarily, size limit checking is handled deeper down, but here we
|
||||||
* have to check as part of (size + extra) clamping, since we need the
|
* have to check as part of (size + extra) clamping, since we need the
|
||||||
* clamped value in the above helper functions.
|
* clamped value in the above helper functions.
|
||||||
*/
|
*/
|
||||||
if (unlikely(size > HUGE_MAXCLASS)) {
|
if (unlikely(size > LARGE_MAXCLASS)) {
|
||||||
usize = old_usize;
|
usize = old_usize;
|
||||||
goto label_not_resized;
|
goto label_not_resized;
|
||||||
}
|
}
|
||||||
if (unlikely(HUGE_MAXCLASS - size < extra))
|
if (unlikely(LARGE_MAXCLASS - size < extra))
|
||||||
extra = HUGE_MAXCLASS - size;
|
extra = LARGE_MAXCLASS - size;
|
||||||
|
|
||||||
if (config_prof && opt_prof) {
|
if (config_prof && opt_prof) {
|
||||||
usize = ixallocx_prof(tsd, extent, ptr, old_usize, size, extra,
|
usize = ixallocx_prof(tsd, extent, ptr, old_usize, size, extra,
|
||||||
@ -2512,7 +2512,7 @@ je_nallocx(size_t size, int flags)
|
|||||||
witness_assert_lockless(tsdn);
|
witness_assert_lockless(tsdn);
|
||||||
|
|
||||||
usize = inallocx(tsdn, size, flags);
|
usize = inallocx(tsdn, size, flags);
|
||||||
if (unlikely(usize > HUGE_MAXCLASS))
|
if (unlikely(usize > LARGE_MAXCLASS))
|
||||||
return (0);
|
return (0);
|
||||||
|
|
||||||
witness_assert_lockless(tsdn);
|
witness_assert_lockless(tsdn);
|
||||||
|
@ -1,19 +1,19 @@
|
|||||||
#define JEMALLOC_HUGE_C_
|
#define JEMALLOC_LARGE_C_
|
||||||
#include "jemalloc/internal/jemalloc_internal.h"
|
#include "jemalloc/internal/jemalloc_internal.h"
|
||||||
|
|
||||||
/******************************************************************************/
|
/******************************************************************************/
|
||||||
|
|
||||||
void *
|
void *
|
||||||
huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
|
large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
|
||||||
{
|
{
|
||||||
|
|
||||||
assert(usize == s2u(usize));
|
assert(usize == s2u(usize));
|
||||||
|
|
||||||
return (huge_palloc(tsdn, arena, usize, CACHELINE, zero));
|
return (large_palloc(tsdn, arena, usize, CACHELINE, zero));
|
||||||
}
|
}
|
||||||
|
|
||||||
void *
|
void *
|
||||||
huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
||||||
bool zero)
|
bool zero)
|
||||||
{
|
{
|
||||||
size_t ausize;
|
size_t ausize;
|
||||||
@ -24,7 +24,7 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||||||
assert(!tsdn_null(tsdn) || arena != NULL);
|
assert(!tsdn_null(tsdn) || arena != NULL);
|
||||||
|
|
||||||
ausize = sa2u(usize, alignment);
|
ausize = sa2u(usize, alignment);
|
||||||
if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
|
if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS))
|
||||||
return (NULL);
|
return (NULL);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -34,15 +34,15 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||||||
is_zeroed = zero;
|
is_zeroed = zero;
|
||||||
if (likely(!tsdn_null(tsdn)))
|
if (likely(!tsdn_null(tsdn)))
|
||||||
arena = arena_choose(tsdn_tsd(tsdn), arena);
|
arena = arena_choose(tsdn_tsd(tsdn), arena);
|
||||||
if (unlikely(arena == NULL) || (extent = arena_chunk_alloc_huge(tsdn,
|
if (unlikely(arena == NULL) || (extent = arena_chunk_alloc_large(tsdn,
|
||||||
arena, usize, alignment, &is_zeroed)) == NULL)
|
arena, usize, alignment, &is_zeroed)) == NULL)
|
||||||
return (NULL);
|
return (NULL);
|
||||||
|
|
||||||
/* Insert extent into huge. */
|
/* Insert extent into large. */
|
||||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
malloc_mutex_lock(tsdn, &arena->large_mtx);
|
||||||
ql_elm_new(extent, ql_link);
|
ql_elm_new(extent, ql_link);
|
||||||
ql_tail_insert(&arena->huge, extent, ql_link);
|
ql_tail_insert(&arena->large, extent, ql_link);
|
||||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
malloc_mutex_unlock(tsdn, &arena->large_mtx);
|
||||||
if (config_prof && arena_prof_accum(tsdn, arena, usize))
|
if (config_prof && arena_prof_accum(tsdn, arena, usize))
|
||||||
prof_idump(tsdn);
|
prof_idump(tsdn);
|
||||||
|
|
||||||
@ -61,23 +61,23 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||||||
}
|
}
|
||||||
|
|
||||||
#ifdef JEMALLOC_JET
|
#ifdef JEMALLOC_JET
|
||||||
#undef huge_dalloc_junk
|
#undef large_dalloc_junk
|
||||||
#define huge_dalloc_junk JEMALLOC_N(n_huge_dalloc_junk)
|
#define large_dalloc_junk JEMALLOC_N(n_large_dalloc_junk)
|
||||||
#endif
|
#endif
|
||||||
void
|
void
|
||||||
huge_dalloc_junk(void *ptr, size_t usize)
|
large_dalloc_junk(void *ptr, size_t usize)
|
||||||
{
|
{
|
||||||
|
|
||||||
memset(ptr, JEMALLOC_FREE_JUNK, usize);
|
memset(ptr, JEMALLOC_FREE_JUNK, usize);
|
||||||
}
|
}
|
||||||
#ifdef JEMALLOC_JET
|
#ifdef JEMALLOC_JET
|
||||||
#undef huge_dalloc_junk
|
#undef large_dalloc_junk
|
||||||
#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
|
#define large_dalloc_junk JEMALLOC_N(large_dalloc_junk)
|
||||||
huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(n_huge_dalloc_junk);
|
large_dalloc_junk_t *large_dalloc_junk = JEMALLOC_N(n_large_dalloc_junk);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
static void
|
static void
|
||||||
huge_dalloc_maybe_junk(tsdn_t *tsdn, void *ptr, size_t usize)
|
large_dalloc_maybe_junk(tsdn_t *tsdn, void *ptr, size_t usize)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (config_fill && have_dss && unlikely(opt_junk_free)) {
|
if (config_fill && have_dss && unlikely(opt_junk_free)) {
|
||||||
@ -86,13 +86,13 @@ huge_dalloc_maybe_junk(tsdn_t *tsdn, void *ptr, size_t usize)
|
|||||||
* unmapped.
|
* unmapped.
|
||||||
*/
|
*/
|
||||||
if (!config_munmap || (have_dss && chunk_in_dss(tsdn, ptr)))
|
if (!config_munmap || (have_dss && chunk_in_dss(tsdn, ptr)))
|
||||||
huge_dalloc_junk(ptr, usize);
|
large_dalloc_junk(ptr, usize);
|
||||||
memset(ptr, JEMALLOC_FREE_JUNK, usize);
|
memset(ptr, JEMALLOC_FREE_JUNK, usize);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool
|
static bool
|
||||||
huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
|
large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
|
||||||
{
|
{
|
||||||
arena_t *arena = extent_arena_get(extent);
|
arena_t *arena = extent_arena_get(extent);
|
||||||
size_t oldusize = extent_usize_get(extent);
|
size_t oldusize = extent_usize_get(extent);
|
||||||
@ -109,20 +109,20 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
|
|||||||
return (true);
|
return (true);
|
||||||
|
|
||||||
if (config_fill && unlikely(opt_junk_free)) {
|
if (config_fill && unlikely(opt_junk_free)) {
|
||||||
huge_dalloc_maybe_junk(tsdn, extent_addr_get(trail),
|
large_dalloc_maybe_junk(tsdn, extent_addr_get(trail),
|
||||||
extent_usize_get(trail));
|
extent_usize_get(trail));
|
||||||
}
|
}
|
||||||
|
|
||||||
arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks, trail);
|
arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks, trail);
|
||||||
}
|
}
|
||||||
|
|
||||||
arena_chunk_ralloc_huge_shrink(tsdn, arena, extent, oldusize);
|
arena_chunk_ralloc_large_shrink(tsdn, arena, extent, oldusize);
|
||||||
|
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool
|
static bool
|
||||||
huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
|
large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
|
||||||
bool zero)
|
bool zero)
|
||||||
{
|
{
|
||||||
arena_t *arena = extent_arena_get(extent);
|
arena_t *arena = extent_arena_get(extent);
|
||||||
@ -173,34 +173,35 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
|
|||||||
JEMALLOC_ALLOC_JUNK, usize - oldusize);
|
JEMALLOC_ALLOC_JUNK, usize - oldusize);
|
||||||
}
|
}
|
||||||
|
|
||||||
arena_chunk_ralloc_huge_expand(tsdn, arena, extent, oldusize);
|
arena_chunk_ralloc_large_expand(tsdn, arena, extent, oldusize);
|
||||||
|
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
|
large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
|
||||||
size_t usize_max, bool zero)
|
size_t usize_max, bool zero)
|
||||||
{
|
{
|
||||||
|
|
||||||
assert(s2u(extent_usize_get(extent)) == extent_usize_get(extent));
|
assert(s2u(extent_usize_get(extent)) == extent_usize_get(extent));
|
||||||
/* The following should have been caught by callers. */
|
/* The following should have been caught by callers. */
|
||||||
assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
|
assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS);
|
||||||
/* Both allocation sizes must be huge to avoid a move. */
|
/* Both allocation sizes must be large to avoid a move. */
|
||||||
assert(extent_usize_get(extent) >= LARGE_MINCLASS && usize_max >=
|
assert(extent_usize_get(extent) >= LARGE_MINCLASS && usize_max >=
|
||||||
LARGE_MINCLASS);
|
LARGE_MINCLASS);
|
||||||
|
|
||||||
if (usize_max > extent_usize_get(extent)) {
|
if (usize_max > extent_usize_get(extent)) {
|
||||||
/* Attempt to expand the allocation in-place. */
|
/* Attempt to expand the allocation in-place. */
|
||||||
if (!huge_ralloc_no_move_expand(tsdn, extent, usize_max,
|
if (!large_ralloc_no_move_expand(tsdn, extent, usize_max,
|
||||||
zero)) {
|
zero)) {
|
||||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
/* Try again, this time with usize_min. */
|
/* Try again, this time with usize_min. */
|
||||||
if (usize_min < usize_max && usize_min >
|
if (usize_min < usize_max && usize_min >
|
||||||
extent_usize_get(extent) && huge_ralloc_no_move_expand(tsdn,
|
extent_usize_get(extent) &&
|
||||||
extent, usize_min, zero)) {
|
large_ralloc_no_move_expand(tsdn, extent, usize_min,
|
||||||
|
zero)) {
|
||||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
@ -218,7 +219,7 @@ huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
|
|||||||
|
|
||||||
/* Attempt to shrink the allocation in-place. */
|
/* Attempt to shrink the allocation in-place. */
|
||||||
if (extent_usize_get(extent) > usize_max) {
|
if (extent_usize_get(extent) > usize_max) {
|
||||||
if (!huge_ralloc_no_move_shrink(tsdn, extent, usize_max)) {
|
if (!large_ralloc_no_move_shrink(tsdn, extent, usize_max)) {
|
||||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
@ -227,30 +228,30 @@ huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void *
|
static void *
|
||||||
huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||||
size_t alignment, bool zero)
|
size_t alignment, bool zero)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (alignment <= CACHELINE)
|
if (alignment <= CACHELINE)
|
||||||
return (huge_malloc(tsdn, arena, usize, zero));
|
return (large_malloc(tsdn, arena, usize, zero));
|
||||||
return (huge_palloc(tsdn, arena, usize, alignment, zero));
|
return (large_palloc(tsdn, arena, usize, alignment, zero));
|
||||||
}
|
}
|
||||||
|
|
||||||
void *
|
void *
|
||||||
huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
|
large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
|
||||||
size_t alignment, bool zero, tcache_t *tcache)
|
size_t alignment, bool zero, tcache_t *tcache)
|
||||||
{
|
{
|
||||||
void *ret;
|
void *ret;
|
||||||
size_t copysize;
|
size_t copysize;
|
||||||
|
|
||||||
/* The following should have been caught by callers. */
|
/* The following should have been caught by callers. */
|
||||||
assert(usize > 0 && usize <= HUGE_MAXCLASS);
|
assert(usize > 0 && usize <= LARGE_MAXCLASS);
|
||||||
/* Both allocation sizes must be huge to avoid a move. */
|
/* Both allocation sizes must be large to avoid a move. */
|
||||||
assert(extent_usize_get(extent) >= LARGE_MINCLASS && usize >=
|
assert(extent_usize_get(extent) >= LARGE_MINCLASS && usize >=
|
||||||
LARGE_MINCLASS);
|
LARGE_MINCLASS);
|
||||||
|
|
||||||
/* Try to avoid moving the allocation. */
|
/* Try to avoid moving the allocation. */
|
||||||
if (!huge_ralloc_no_move(tsdn, extent, usize, usize, zero))
|
if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero))
|
||||||
return (extent_addr_get(extent));
|
return (extent_addr_get(extent));
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -258,7 +259,7 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
|
|||||||
* different size class. In that case, fall back to allocating new
|
* different size class. In that case, fall back to allocating new
|
||||||
* space and copying.
|
* space and copying.
|
||||||
*/
|
*/
|
||||||
ret = huge_ralloc_move_helper(tsdn, arena, usize, alignment, zero);
|
ret = large_ralloc_move_helper(tsdn, arena, usize, alignment, zero);
|
||||||
if (ret == NULL)
|
if (ret == NULL)
|
||||||
return (NULL);
|
return (NULL);
|
||||||
|
|
||||||
@ -271,82 +272,82 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
huge_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked)
|
large_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked)
|
||||||
{
|
{
|
||||||
arena_t *arena;
|
arena_t *arena;
|
||||||
|
|
||||||
arena = extent_arena_get(extent);
|
arena = extent_arena_get(extent);
|
||||||
if (!junked_locked)
|
if (!junked_locked)
|
||||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
malloc_mutex_lock(tsdn, &arena->large_mtx);
|
||||||
ql_remove(&arena->huge, extent, ql_link);
|
ql_remove(&arena->large, extent, ql_link);
|
||||||
if (!junked_locked) {
|
if (!junked_locked) {
|
||||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
malloc_mutex_unlock(tsdn, &arena->large_mtx);
|
||||||
|
|
||||||
huge_dalloc_maybe_junk(tsdn, extent_addr_get(extent),
|
large_dalloc_maybe_junk(tsdn, extent_addr_get(extent),
|
||||||
extent_usize_get(extent));
|
extent_usize_get(extent));
|
||||||
}
|
}
|
||||||
arena_chunk_dalloc_huge(tsdn, arena, extent, junked_locked);
|
arena_chunk_dalloc_large(tsdn, arena, extent, junked_locked);
|
||||||
|
|
||||||
if (!junked_locked)
|
if (!junked_locked)
|
||||||
arena_decay_tick(tsdn, arena);
|
arena_decay_tick(tsdn, arena);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
huge_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent)
|
large_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent)
|
||||||
{
|
{
|
||||||
|
|
||||||
huge_dalloc_impl(tsdn, extent, true);
|
large_dalloc_impl(tsdn, extent, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
huge_dalloc(tsdn_t *tsdn, extent_t *extent)
|
large_dalloc(tsdn_t *tsdn, extent_t *extent)
|
||||||
{
|
{
|
||||||
|
|
||||||
huge_dalloc_impl(tsdn, extent, false);
|
large_dalloc_impl(tsdn, extent, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t
|
size_t
|
||||||
huge_salloc(tsdn_t *tsdn, const extent_t *extent)
|
large_salloc(tsdn_t *tsdn, const extent_t *extent)
|
||||||
{
|
{
|
||||||
size_t usize;
|
size_t usize;
|
||||||
arena_t *arena;
|
arena_t *arena;
|
||||||
|
|
||||||
arena = extent_arena_get(extent);
|
arena = extent_arena_get(extent);
|
||||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
malloc_mutex_lock(tsdn, &arena->large_mtx);
|
||||||
usize = extent_usize_get(extent);
|
usize = extent_usize_get(extent);
|
||||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
malloc_mutex_unlock(tsdn, &arena->large_mtx);
|
||||||
|
|
||||||
return (usize);
|
return (usize);
|
||||||
}
|
}
|
||||||
|
|
||||||
prof_tctx_t *
|
prof_tctx_t *
|
||||||
huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent)
|
large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent)
|
||||||
{
|
{
|
||||||
prof_tctx_t *tctx;
|
prof_tctx_t *tctx;
|
||||||
arena_t *arena;
|
arena_t *arena;
|
||||||
|
|
||||||
arena = extent_arena_get(extent);
|
arena = extent_arena_get(extent);
|
||||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
malloc_mutex_lock(tsdn, &arena->large_mtx);
|
||||||
tctx = extent_prof_tctx_get(extent);
|
tctx = extent_prof_tctx_get(extent);
|
||||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
malloc_mutex_unlock(tsdn, &arena->large_mtx);
|
||||||
|
|
||||||
return (tctx);
|
return (tctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx)
|
large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx)
|
||||||
{
|
{
|
||||||
arena_t *arena;
|
arena_t *arena;
|
||||||
|
|
||||||
arena = extent_arena_get(extent);
|
arena = extent_arena_get(extent);
|
||||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
malloc_mutex_lock(tsdn, &arena->large_mtx);
|
||||||
extent_prof_tctx_set(extent, tctx);
|
extent_prof_tctx_set(extent, tctx);
|
||||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
malloc_mutex_unlock(tsdn, &arena->large_mtx);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent)
|
large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent)
|
||||||
{
|
{
|
||||||
|
|
||||||
huge_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
|
large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
|
||||||
}
|
}
|
71
src/stats.c
71
src/stats.c
@ -37,10 +37,10 @@ size_t stats_cactive = 0;
|
|||||||
|
|
||||||
static void stats_arena_bins_print(void (*write_cb)(void *, const char *),
|
static void stats_arena_bins_print(void (*write_cb)(void *, const char *),
|
||||||
void *cbopaque, unsigned i);
|
void *cbopaque, unsigned i);
|
||||||
static void stats_arena_hchunks_print(
|
static void stats_arena_lextents_print(
|
||||||
void (*write_cb)(void *, const char *), void *cbopaque, unsigned i);
|
void (*write_cb)(void *, const char *), void *cbopaque, unsigned i);
|
||||||
static void stats_arena_print(void (*write_cb)(void *, const char *),
|
static void stats_arena_print(void (*write_cb)(void *, const char *),
|
||||||
void *cbopaque, unsigned i, bool bins, bool huge);
|
void *cbopaque, unsigned i, bool bins, bool large);
|
||||||
|
|
||||||
/******************************************************************************/
|
/******************************************************************************/
|
||||||
|
|
||||||
@ -157,34 +157,34 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
stats_arena_hchunks_print(void (*write_cb)(void *, const char *),
|
stats_arena_lextents_print(void (*write_cb)(void *, const char *),
|
||||||
void *cbopaque, unsigned i)
|
void *cbopaque, unsigned i)
|
||||||
{
|
{
|
||||||
unsigned nbins, nhchunks, j;
|
unsigned nbins, nlextents, j;
|
||||||
bool in_gap;
|
bool in_gap;
|
||||||
|
|
||||||
malloc_cprintf(write_cb, cbopaque,
|
malloc_cprintf(write_cb, cbopaque,
|
||||||
"huge: size ind allocated nmalloc ndalloc"
|
"large: size ind allocated nmalloc ndalloc"
|
||||||
" nrequests curhchunks\n");
|
" nrequests curlextents\n");
|
||||||
CTL_GET("arenas.nbins", &nbins, unsigned);
|
CTL_GET("arenas.nbins", &nbins, unsigned);
|
||||||
CTL_GET("arenas.nhchunks", &nhchunks, unsigned);
|
CTL_GET("arenas.nlextents", &nlextents, unsigned);
|
||||||
for (j = 0, in_gap = false; j < nhchunks; j++) {
|
for (j = 0, in_gap = false; j < nlextents; j++) {
|
||||||
uint64_t nmalloc, ndalloc, nrequests;
|
uint64_t nmalloc, ndalloc, nrequests;
|
||||||
size_t hchunk_size, curhchunks;
|
size_t lextent_size, curlextents;
|
||||||
|
|
||||||
CTL_M2_M4_GET("stats.arenas.0.hchunks.0.nmalloc", i, j,
|
CTL_M2_M4_GET("stats.arenas.0.lextents.0.nmalloc", i, j,
|
||||||
&nmalloc, uint64_t);
|
&nmalloc, uint64_t);
|
||||||
CTL_M2_M4_GET("stats.arenas.0.hchunks.0.ndalloc", i, j,
|
CTL_M2_M4_GET("stats.arenas.0.lextents.0.ndalloc", i, j,
|
||||||
&ndalloc, uint64_t);
|
&ndalloc, uint64_t);
|
||||||
CTL_M2_M4_GET("stats.arenas.0.hchunks.0.nrequests", i, j,
|
CTL_M2_M4_GET("stats.arenas.0.lextents.0.nrequests", i, j,
|
||||||
&nrequests, uint64_t);
|
&nrequests, uint64_t);
|
||||||
if (nrequests == 0)
|
if (nrequests == 0)
|
||||||
in_gap = true;
|
in_gap = true;
|
||||||
else {
|
else {
|
||||||
CTL_M2_GET("arenas.hchunk.0.size", j, &hchunk_size,
|
CTL_M2_GET("arenas.lextent.0.size", j, &lextent_size,
|
||||||
size_t);
|
size_t);
|
||||||
CTL_M2_M4_GET("stats.arenas.0.hchunks.0.curhchunks", i,
|
CTL_M2_M4_GET("stats.arenas.0.lextents.0.curlextents",
|
||||||
j, &curhchunks, size_t);
|
i, j, &curlextents, size_t);
|
||||||
if (in_gap) {
|
if (in_gap) {
|
||||||
malloc_cprintf(write_cb, cbopaque,
|
malloc_cprintf(write_cb, cbopaque,
|
||||||
" ---\n");
|
" ---\n");
|
||||||
@ -193,9 +193,9 @@ stats_arena_hchunks_print(void (*write_cb)(void *, const char *),
|
|||||||
malloc_cprintf(write_cb, cbopaque,
|
malloc_cprintf(write_cb, cbopaque,
|
||||||
"%20zu %3u %12zu %12"FMTu64" %12"FMTu64
|
"%20zu %3u %12zu %12"FMTu64" %12"FMTu64
|
||||||
" %12"FMTu64" %12zu\n",
|
" %12"FMTu64" %12zu\n",
|
||||||
hchunk_size, nbins + j,
|
lextent_size, nbins + j,
|
||||||
curhchunks * hchunk_size, nmalloc, ndalloc,
|
curlextents * lextent_size, nmalloc, ndalloc,
|
||||||
nrequests, curhchunks);
|
nrequests, curlextents);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (in_gap) {
|
if (in_gap) {
|
||||||
@ -206,7 +206,7 @@ stats_arena_hchunks_print(void (*write_cb)(void *, const char *),
|
|||||||
|
|
||||||
static void
|
static void
|
||||||
stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||||
unsigned i, bool bins, bool huge)
|
unsigned i, bool bins, bool large)
|
||||||
{
|
{
|
||||||
unsigned nthreads;
|
unsigned nthreads;
|
||||||
const char *dss;
|
const char *dss;
|
||||||
@ -216,8 +216,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
|||||||
uint64_t npurge, nmadvise, purged;
|
uint64_t npurge, nmadvise, purged;
|
||||||
size_t small_allocated;
|
size_t small_allocated;
|
||||||
uint64_t small_nmalloc, small_ndalloc, small_nrequests;
|
uint64_t small_nmalloc, small_ndalloc, small_nrequests;
|
||||||
size_t huge_allocated;
|
size_t large_allocated;
|
||||||
uint64_t huge_nmalloc, huge_ndalloc, huge_nrequests;
|
uint64_t large_nmalloc, large_ndalloc, large_nrequests;
|
||||||
|
|
||||||
CTL_GET("arenas.page", &page, size_t);
|
CTL_GET("arenas.page", &page, size_t);
|
||||||
|
|
||||||
@ -268,20 +268,21 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
|||||||
"small: %12zu %12"FMTu64" %12"FMTu64
|
"small: %12zu %12"FMTu64" %12"FMTu64
|
||||||
" %12"FMTu64"\n",
|
" %12"FMTu64"\n",
|
||||||
small_allocated, small_nmalloc, small_ndalloc, small_nrequests);
|
small_allocated, small_nmalloc, small_ndalloc, small_nrequests);
|
||||||
CTL_M2_GET("stats.arenas.0.huge.allocated", i, &huge_allocated, size_t);
|
CTL_M2_GET("stats.arenas.0.large.allocated", i, &large_allocated,
|
||||||
CTL_M2_GET("stats.arenas.0.huge.nmalloc", i, &huge_nmalloc, uint64_t);
|
size_t);
|
||||||
CTL_M2_GET("stats.arenas.0.huge.ndalloc", i, &huge_ndalloc, uint64_t);
|
CTL_M2_GET("stats.arenas.0.large.nmalloc", i, &large_nmalloc, uint64_t);
|
||||||
CTL_M2_GET("stats.arenas.0.huge.nrequests", i, &huge_nrequests,
|
CTL_M2_GET("stats.arenas.0.large.ndalloc", i, &large_ndalloc, uint64_t);
|
||||||
|
CTL_M2_GET("stats.arenas.0.large.nrequests", i, &large_nrequests,
|
||||||
uint64_t);
|
uint64_t);
|
||||||
malloc_cprintf(write_cb, cbopaque,
|
malloc_cprintf(write_cb, cbopaque,
|
||||||
"huge: %12zu %12"FMTu64" %12"FMTu64
|
"large: %12zu %12"FMTu64" %12"FMTu64
|
||||||
" %12"FMTu64"\n",
|
" %12"FMTu64"\n",
|
||||||
huge_allocated, huge_nmalloc, huge_ndalloc, huge_nrequests);
|
large_allocated, large_nmalloc, large_ndalloc, large_nrequests);
|
||||||
malloc_cprintf(write_cb, cbopaque,
|
malloc_cprintf(write_cb, cbopaque,
|
||||||
"total: %12zu %12"FMTu64" %12"FMTu64
|
"total: %12zu %12"FMTu64" %12"FMTu64
|
||||||
" %12"FMTu64"\n",
|
" %12"FMTu64"\n",
|
||||||
small_allocated + huge_allocated, small_nmalloc + huge_nmalloc,
|
small_allocated + large_allocated, small_nmalloc + large_nmalloc,
|
||||||
small_ndalloc + huge_ndalloc, small_nrequests + huge_nrequests);
|
small_ndalloc + large_ndalloc, small_nrequests + large_nrequests);
|
||||||
malloc_cprintf(write_cb, cbopaque,
|
malloc_cprintf(write_cb, cbopaque,
|
||||||
"active: %12zu\n", pactive * page);
|
"active: %12zu\n", pactive * page);
|
||||||
CTL_M2_GET("stats.arenas.0.mapped", i, &mapped, size_t);
|
CTL_M2_GET("stats.arenas.0.mapped", i, &mapped, size_t);
|
||||||
@ -300,8 +301,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
|||||||
|
|
||||||
if (bins)
|
if (bins)
|
||||||
stats_arena_bins_print(write_cb, cbopaque, i);
|
stats_arena_bins_print(write_cb, cbopaque, i);
|
||||||
if (huge)
|
if (large)
|
||||||
stats_arena_hchunks_print(write_cb, cbopaque, i);
|
stats_arena_lextents_print(write_cb, cbopaque, i);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
@ -315,7 +316,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
|||||||
bool merged = true;
|
bool merged = true;
|
||||||
bool unmerged = true;
|
bool unmerged = true;
|
||||||
bool bins = true;
|
bool bins = true;
|
||||||
bool huge = true;
|
bool large = true;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Refresh stats, in case mallctl() was called by the application.
|
* Refresh stats, in case mallctl() was called by the application.
|
||||||
@ -356,7 +357,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
|||||||
bins = false;
|
bins = false;
|
||||||
break;
|
break;
|
||||||
case 'l':
|
case 'l':
|
||||||
huge = false;
|
large = false;
|
||||||
break;
|
break;
|
||||||
default:;
|
default:;
|
||||||
}
|
}
|
||||||
@ -568,7 +569,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
|||||||
malloc_cprintf(write_cb, cbopaque,
|
malloc_cprintf(write_cb, cbopaque,
|
||||||
"\nMerged arenas stats:\n");
|
"\nMerged arenas stats:\n");
|
||||||
stats_arena_print(write_cb, cbopaque,
|
stats_arena_print(write_cb, cbopaque,
|
||||||
narenas, bins, huge);
|
narenas, bins, large);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -594,7 +595,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
|||||||
cbopaque,
|
cbopaque,
|
||||||
"\narenas[%u]:\n", i);
|
"\narenas[%u]:\n", i);
|
||||||
stats_arena_print(write_cb,
|
stats_arena_print(write_cb,
|
||||||
cbopaque, i, bins, huge);
|
cbopaque, i, bins, large);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
28
src/tcache.c
28
src/tcache.c
@ -46,7 +46,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache)
|
|||||||
tbin->ncached - tbin->low_water + (tbin->low_water
|
tbin->ncached - tbin->low_water + (tbin->low_water
|
||||||
>> 2));
|
>> 2));
|
||||||
} else {
|
} else {
|
||||||
tcache_bin_flush_huge(tsd, tbin, binind, tbin->ncached
|
tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached
|
||||||
- tbin->low_water + (tbin->low_water >> 2), tcache);
|
- tbin->low_water + (tbin->low_water >> 2), tcache);
|
||||||
}
|
}
|
||||||
/*
|
/*
|
||||||
@ -164,7 +164,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
|
|||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
||||||
unsigned rem, tcache_t *tcache)
|
unsigned rem, tcache_t *tcache)
|
||||||
{
|
{
|
||||||
arena_t *arena;
|
arena_t *arena;
|
||||||
@ -194,9 +194,9 @@ tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
|||||||
}
|
}
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
merged_stats = true;
|
merged_stats = true;
|
||||||
arena->stats.nrequests_huge +=
|
arena->stats.nrequests_large +=
|
||||||
tbin->tstats.nrequests;
|
tbin->tstats.nrequests;
|
||||||
arena->stats.hstats[binind - NBINS].nrequests +=
|
arena->stats.lstats[binind - NBINS].nrequests +=
|
||||||
tbin->tstats.nrequests;
|
tbin->tstats.nrequests;
|
||||||
tbin->tstats.nrequests = 0;
|
tbin->tstats.nrequests = 0;
|
||||||
}
|
}
|
||||||
@ -207,7 +207,7 @@ tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
|||||||
assert(ptr != NULL);
|
assert(ptr != NULL);
|
||||||
extent = iealloc(tsd_tsdn(tsd), ptr);
|
extent = iealloc(tsd_tsdn(tsd), ptr);
|
||||||
if (extent_arena_get(extent) == locked_arena) {
|
if (extent_arena_get(extent) == locked_arena) {
|
||||||
huge_dalloc_junked_locked(tsd_tsdn(tsd),
|
large_dalloc_junked_locked(tsd_tsdn(tsd),
|
||||||
extent);
|
extent);
|
||||||
} else {
|
} else {
|
||||||
/*
|
/*
|
||||||
@ -232,8 +232,8 @@ tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
|||||||
* arena, so the stats didn't get merged. Manually do so now.
|
* arena, so the stats didn't get merged. Manually do so now.
|
||||||
*/
|
*/
|
||||||
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
|
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
|
||||||
arena->stats.nrequests_huge += tbin->tstats.nrequests;
|
arena->stats.nrequests_large += tbin->tstats.nrequests;
|
||||||
arena->stats.hstats[binind - NBINS].nrequests +=
|
arena->stats.lstats[binind - NBINS].nrequests +=
|
||||||
tbin->tstats.nrequests;
|
tbin->tstats.nrequests;
|
||||||
tbin->tstats.nrequests = 0;
|
tbin->tstats.nrequests = 0;
|
||||||
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
|
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
|
||||||
@ -371,12 +371,12 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
|
|||||||
|
|
||||||
for (; i < nhbins; i++) {
|
for (; i < nhbins; i++) {
|
||||||
tcache_bin_t *tbin = &tcache->tbins[i];
|
tcache_bin_t *tbin = &tcache->tbins[i];
|
||||||
tcache_bin_flush_huge(tsd, tbin, i, 0, tcache);
|
tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
|
||||||
|
|
||||||
if (config_stats && tbin->tstats.nrequests != 0) {
|
if (config_stats && tbin->tstats.nrequests != 0) {
|
||||||
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
|
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
|
||||||
arena->stats.nrequests_huge += tbin->tstats.nrequests;
|
arena->stats.nrequests_large += tbin->tstats.nrequests;
|
||||||
arena->stats.hstats[i - NBINS].nrequests +=
|
arena->stats.lstats[i - NBINS].nrequests +=
|
||||||
tbin->tstats.nrequests;
|
tbin->tstats.nrequests;
|
||||||
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
|
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
|
||||||
}
|
}
|
||||||
@ -431,10 +431,10 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (; i < nhbins; i++) {
|
for (; i < nhbins; i++) {
|
||||||
malloc_huge_stats_t *hstats = &arena->stats.hstats[i - NBINS];
|
malloc_large_stats_t *lstats = &arena->stats.lstats[i - NBINS];
|
||||||
tcache_bin_t *tbin = &tcache->tbins[i];
|
tcache_bin_t *tbin = &tcache->tbins[i];
|
||||||
arena->stats.nrequests_huge += tbin->tstats.nrequests;
|
arena->stats.nrequests_large += tbin->tstats.nrequests;
|
||||||
hstats->nrequests += tbin->tstats.nrequests;
|
lstats->nrequests += tbin->tstats.nrequests;
|
||||||
tbin->tstats.nrequests = 0;
|
tbin->tstats.nrequests = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -537,7 +537,7 @@ tcache_boot(tsdn_t *tsdn)
|
|||||||
stack_nelms += tcache_bin_info[i].ncached_max;
|
stack_nelms += tcache_bin_info[i].ncached_max;
|
||||||
}
|
}
|
||||||
for (; i < nhbins; i++) {
|
for (; i < nhbins; i++) {
|
||||||
tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_HUGE;
|
tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
|
||||||
stack_nelms += tcache_bin_info[i].ncached_max;
|
stack_nelms += tcache_bin_info[i].ncached_max;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -120,7 +120,7 @@ chunk_merge(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
|
|||||||
TEST_BEGIN(test_chunk)
|
TEST_BEGIN(test_chunk)
|
||||||
{
|
{
|
||||||
void *p;
|
void *p;
|
||||||
size_t old_size, new_size, huge0, huge1, huge2, sz;
|
size_t old_size, new_size, large0, large1, large2, sz;
|
||||||
unsigned arena_ind;
|
unsigned arena_ind;
|
||||||
int flags;
|
int flags;
|
||||||
size_t hooks_mib[3], purge_mib[3];
|
size_t hooks_mib[3], purge_mib[3];
|
||||||
@ -162,14 +162,14 @@ TEST_BEGIN(test_chunk)
|
|||||||
assert_ptr_ne(old_hooks.split, chunk_split, "Unexpected split error");
|
assert_ptr_ne(old_hooks.split, chunk_split, "Unexpected split error");
|
||||||
assert_ptr_ne(old_hooks.merge, chunk_merge, "Unexpected merge error");
|
assert_ptr_ne(old_hooks.merge, chunk_merge, "Unexpected merge error");
|
||||||
|
|
||||||
/* Get huge size classes. */
|
/* Get large size classes. */
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.lextent.0.size", &large0, &sz, NULL, 0), 0,
|
||||||
"Unexpected arenas.hchunk.0.size failure");
|
"Unexpected arenas.lextent.0.size failure");
|
||||||
assert_d_eq(mallctl("arenas.hchunk.1.size", &huge1, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.lextent.1.size", &large1, &sz, NULL, 0), 0,
|
||||||
"Unexpected arenas.hchunk.1.size failure");
|
"Unexpected arenas.lextent.1.size failure");
|
||||||
assert_d_eq(mallctl("arenas.hchunk.2.size", &huge2, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.lextent.2.size", &large2, &sz, NULL, 0), 0,
|
||||||
"Unexpected arenas.hchunk.2.size failure");
|
"Unexpected arenas.lextent.2.size failure");
|
||||||
|
|
||||||
/* Test dalloc/decommit/purge cascade. */
|
/* Test dalloc/decommit/purge cascade. */
|
||||||
purge_miblen = sizeof(purge_mib)/sizeof(size_t);
|
purge_miblen = sizeof(purge_mib)/sizeof(size_t);
|
||||||
@ -178,13 +178,13 @@ TEST_BEGIN(test_chunk)
|
|||||||
purge_mib[1] = (size_t)arena_ind;
|
purge_mib[1] = (size_t)arena_ind;
|
||||||
do_dalloc = false;
|
do_dalloc = false;
|
||||||
do_decommit = false;
|
do_decommit = false;
|
||||||
p = mallocx(huge0 * 2, flags);
|
p = mallocx(large0 * 2, flags);
|
||||||
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
||||||
did_dalloc = false;
|
did_dalloc = false;
|
||||||
did_decommit = false;
|
did_decommit = false;
|
||||||
did_purge = false;
|
did_purge = false;
|
||||||
did_split = false;
|
did_split = false;
|
||||||
xallocx_success_a = (xallocx(p, huge0, 0, flags) == huge0);
|
xallocx_success_a = (xallocx(p, large0, 0, flags) == large0);
|
||||||
assert_d_eq(mallctlbymib(purge_mib, purge_miblen, NULL, NULL, NULL, 0),
|
assert_d_eq(mallctlbymib(purge_mib, purge_miblen, NULL, NULL, NULL, 0),
|
||||||
0, "Unexpected arena.%u.purge error", arena_ind);
|
0, "Unexpected arena.%u.purge error", arena_ind);
|
||||||
if (xallocx_success_a) {
|
if (xallocx_success_a) {
|
||||||
@ -199,18 +199,18 @@ TEST_BEGIN(test_chunk)
|
|||||||
/* Test decommit/commit and observe split/merge. */
|
/* Test decommit/commit and observe split/merge. */
|
||||||
do_dalloc = false;
|
do_dalloc = false;
|
||||||
do_decommit = true;
|
do_decommit = true;
|
||||||
p = mallocx(huge0 * 2, flags);
|
p = mallocx(large0 * 2, flags);
|
||||||
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
||||||
did_decommit = false;
|
did_decommit = false;
|
||||||
did_commit = false;
|
did_commit = false;
|
||||||
did_split = false;
|
did_split = false;
|
||||||
did_merge = false;
|
did_merge = false;
|
||||||
xallocx_success_b = (xallocx(p, huge0, 0, flags) == huge0);
|
xallocx_success_b = (xallocx(p, large0, 0, flags) == large0);
|
||||||
assert_d_eq(mallctlbymib(purge_mib, purge_miblen, NULL, NULL, NULL, 0),
|
assert_d_eq(mallctlbymib(purge_mib, purge_miblen, NULL, NULL, NULL, 0),
|
||||||
0, "Unexpected arena.%u.purge error", arena_ind);
|
0, "Unexpected arena.%u.purge error", arena_ind);
|
||||||
if (xallocx_success_b)
|
if (xallocx_success_b)
|
||||||
assert_true(did_split, "Expected split");
|
assert_true(did_split, "Expected split");
|
||||||
xallocx_success_c = (xallocx(p, huge0 * 2, 0, flags) == huge0 * 2);
|
xallocx_success_c = (xallocx(p, large0 * 2, 0, flags) == large0 * 2);
|
||||||
assert_b_eq(did_decommit, did_commit, "Expected decommit/commit match");
|
assert_b_eq(did_decommit, did_commit, "Expected decommit/commit match");
|
||||||
if (xallocx_success_b && xallocx_success_c)
|
if (xallocx_success_b && xallocx_success_c)
|
||||||
assert_true(did_merge, "Expected merge");
|
assert_true(did_merge, "Expected merge");
|
||||||
@ -218,7 +218,7 @@ TEST_BEGIN(test_chunk)
|
|||||||
do_dalloc = true;
|
do_dalloc = true;
|
||||||
do_decommit = false;
|
do_decommit = false;
|
||||||
|
|
||||||
/* Make sure non-huge allocation succeeds. */
|
/* Make sure non-large allocation succeeds. */
|
||||||
p = mallocx(42, flags);
|
p = mallocx(42, flags);
|
||||||
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
||||||
dallocx(p, flags);
|
dallocx(p, flags);
|
||||||
|
@ -18,10 +18,10 @@ get_nsizes_impl(const char *cmd)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static unsigned
|
static unsigned
|
||||||
get_nhuge(void)
|
get_nlarge(void)
|
||||||
{
|
{
|
||||||
|
|
||||||
return (get_nsizes_impl("arenas.nhchunks"));
|
return (get_nsizes_impl("arenas.nlextents"));
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
@ -44,20 +44,20 @@ get_size_impl(const char *cmd, size_t ind)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
get_huge_size(size_t ind)
|
get_large_size(size_t ind)
|
||||||
{
|
{
|
||||||
|
|
||||||
return (get_size_impl("arenas.hchunk.0.size", ind));
|
return (get_size_impl("arenas.lextent.0.size", ind));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_BEGIN(test_overflow)
|
TEST_BEGIN(test_overflow)
|
||||||
{
|
{
|
||||||
size_t hugemax;
|
size_t largemax;
|
||||||
|
|
||||||
hugemax = get_huge_size(get_nhuge()-1);
|
largemax = get_large_size(get_nlarge()-1);
|
||||||
|
|
||||||
assert_ptr_null(mallocx(hugemax+1, 0),
|
assert_ptr_null(mallocx(largemax+1, 0),
|
||||||
"Expected OOM for mallocx(size=%#zx, 0)", hugemax+1);
|
"Expected OOM for mallocx(size=%#zx, 0)", largemax+1);
|
||||||
|
|
||||||
assert_ptr_null(mallocx(ZU(PTRDIFF_MAX)+1, 0),
|
assert_ptr_null(mallocx(ZU(PTRDIFF_MAX)+1, 0),
|
||||||
"Expected OOM for mallocx(size=%#zx, 0)", ZU(PTRDIFF_MAX)+1);
|
"Expected OOM for mallocx(size=%#zx, 0)", ZU(PTRDIFF_MAX)+1);
|
||||||
@ -73,7 +73,7 @@ TEST_END
|
|||||||
|
|
||||||
TEST_BEGIN(test_oom)
|
TEST_BEGIN(test_oom)
|
||||||
{
|
{
|
||||||
size_t hugemax;
|
size_t largemax;
|
||||||
bool oom;
|
bool oom;
|
||||||
void *ptrs[3];
|
void *ptrs[3];
|
||||||
unsigned i;
|
unsigned i;
|
||||||
@ -82,16 +82,16 @@ TEST_BEGIN(test_oom)
|
|||||||
* It should be impossible to allocate three objects that each consume
|
* It should be impossible to allocate three objects that each consume
|
||||||
* nearly half the virtual address space.
|
* nearly half the virtual address space.
|
||||||
*/
|
*/
|
||||||
hugemax = get_huge_size(get_nhuge()-1);
|
largemax = get_large_size(get_nlarge()-1);
|
||||||
oom = false;
|
oom = false;
|
||||||
for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) {
|
for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) {
|
||||||
ptrs[i] = mallocx(hugemax, 0);
|
ptrs[i] = mallocx(largemax, 0);
|
||||||
if (ptrs[i] == NULL)
|
if (ptrs[i] == NULL)
|
||||||
oom = true;
|
oom = true;
|
||||||
}
|
}
|
||||||
assert_true(oom,
|
assert_true(oom,
|
||||||
"Expected OOM during series of calls to mallocx(size=%zu, 0)",
|
"Expected OOM during series of calls to mallocx(size=%zu, 0)",
|
||||||
hugemax);
|
largemax);
|
||||||
for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) {
|
for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) {
|
||||||
if (ptrs[i] != NULL)
|
if (ptrs[i] != NULL)
|
||||||
dallocx(ptrs[i], 0);
|
dallocx(ptrs[i], 0);
|
||||||
|
@ -2,19 +2,19 @@
|
|||||||
|
|
||||||
TEST_BEGIN(test_overflow)
|
TEST_BEGIN(test_overflow)
|
||||||
{
|
{
|
||||||
unsigned nhchunks;
|
unsigned nlextents;
|
||||||
size_t mib[4];
|
size_t mib[4];
|
||||||
size_t sz, miblen, max_size_class;
|
size_t sz, miblen, max_size_class;
|
||||||
void *p;
|
void *p;
|
||||||
|
|
||||||
sz = sizeof(unsigned);
|
sz = sizeof(unsigned);
|
||||||
assert_d_eq(mallctl("arenas.nhchunks", &nhchunks, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.nlextents", &nlextents, &sz, NULL, 0), 0,
|
||||||
"Unexpected mallctl() error");
|
"Unexpected mallctl() error");
|
||||||
|
|
||||||
miblen = sizeof(mib) / sizeof(size_t);
|
miblen = sizeof(mib) / sizeof(size_t);
|
||||||
assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0,
|
assert_d_eq(mallctlnametomib("arenas.lextent.0.size", mib, &miblen), 0,
|
||||||
"Unexpected mallctlnametomib() error");
|
"Unexpected mallctlnametomib() error");
|
||||||
mib[2] = nhchunks - 1;
|
mib[2] = nlextents - 1;
|
||||||
|
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0,
|
assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0,
|
||||||
|
@ -14,10 +14,10 @@ get_nsizes_impl(const char *cmd)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static unsigned
|
static unsigned
|
||||||
get_nhuge(void)
|
get_nlarge(void)
|
||||||
{
|
{
|
||||||
|
|
||||||
return (get_nsizes_impl("arenas.nhchunks"));
|
return (get_nsizes_impl("arenas.nlextents"));
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
@ -40,10 +40,10 @@ get_size_impl(const char *cmd, size_t ind)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
get_huge_size(size_t ind)
|
get_large_size(size_t ind)
|
||||||
{
|
{
|
||||||
|
|
||||||
return (get_size_impl("arenas.hchunk.0.size", ind));
|
return (get_size_impl("arenas.lextent.0.size", ind));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_BEGIN(test_grow_and_shrink)
|
TEST_BEGIN(test_grow_and_shrink)
|
||||||
@ -221,16 +221,16 @@ TEST_END
|
|||||||
|
|
||||||
TEST_BEGIN(test_overflow)
|
TEST_BEGIN(test_overflow)
|
||||||
{
|
{
|
||||||
size_t hugemax;
|
size_t largemax;
|
||||||
void *p;
|
void *p;
|
||||||
|
|
||||||
hugemax = get_huge_size(get_nhuge()-1);
|
largemax = get_large_size(get_nlarge()-1);
|
||||||
|
|
||||||
p = mallocx(1, 0);
|
p = mallocx(1, 0);
|
||||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||||
|
|
||||||
assert_ptr_null(rallocx(p, hugemax+1, 0),
|
assert_ptr_null(rallocx(p, largemax+1, 0),
|
||||||
"Expected OOM for rallocx(p, size=%#zx, 0)", hugemax+1);
|
"Expected OOM for rallocx(p, size=%#zx, 0)", largemax+1);
|
||||||
|
|
||||||
assert_ptr_null(rallocx(p, ZU(PTRDIFF_MAX)+1, 0),
|
assert_ptr_null(rallocx(p, ZU(PTRDIFF_MAX)+1, 0),
|
||||||
"Expected OOM for rallocx(p, size=%#zx, 0)", ZU(PTRDIFF_MAX)+1);
|
"Expected OOM for rallocx(p, size=%#zx, 0)", ZU(PTRDIFF_MAX)+1);
|
||||||
|
@ -92,10 +92,10 @@ get_nsmall(void)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static unsigned
|
static unsigned
|
||||||
get_nhuge(void)
|
get_nlarge(void)
|
||||||
{
|
{
|
||||||
|
|
||||||
return (get_nsizes_impl("arenas.nhchunks"));
|
return (get_nsizes_impl("arenas.nlextents"));
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
@ -125,20 +125,20 @@ get_small_size(size_t ind)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
get_huge_size(size_t ind)
|
get_large_size(size_t ind)
|
||||||
{
|
{
|
||||||
|
|
||||||
return (get_size_impl("arenas.hchunk.0.size", ind));
|
return (get_size_impl("arenas.lextent.0.size", ind));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_BEGIN(test_size)
|
TEST_BEGIN(test_size)
|
||||||
{
|
{
|
||||||
size_t small0, hugemax;
|
size_t small0, largemax;
|
||||||
void *p;
|
void *p;
|
||||||
|
|
||||||
/* Get size classes. */
|
/* Get size classes. */
|
||||||
small0 = get_small_size(0);
|
small0 = get_small_size(0);
|
||||||
hugemax = get_huge_size(get_nhuge()-1);
|
largemax = get_large_size(get_nlarge()-1);
|
||||||
|
|
||||||
p = mallocx(small0, 0);
|
p = mallocx(small0, 0);
|
||||||
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
||||||
@ -148,13 +148,13 @@ TEST_BEGIN(test_size)
|
|||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
/* Test largest supported size. */
|
/* Test largest supported size. */
|
||||||
assert_zu_le(xallocx(p, hugemax, 0, 0), hugemax,
|
assert_zu_le(xallocx(p, largemax, 0, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
/* Test size overflow. */
|
/* Test size overflow. */
|
||||||
assert_zu_le(xallocx(p, hugemax+1, 0, 0), hugemax,
|
assert_zu_le(xallocx(p, largemax+1, 0, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_le(xallocx(p, SIZE_T_MAX, 0, 0), hugemax,
|
assert_zu_le(xallocx(p, SIZE_T_MAX, 0, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
dallocx(p, 0);
|
dallocx(p, 0);
|
||||||
@ -163,30 +163,30 @@ TEST_END
|
|||||||
|
|
||||||
TEST_BEGIN(test_size_extra_overflow)
|
TEST_BEGIN(test_size_extra_overflow)
|
||||||
{
|
{
|
||||||
size_t small0, hugemax;
|
size_t small0, largemax;
|
||||||
void *p;
|
void *p;
|
||||||
|
|
||||||
/* Get size classes. */
|
/* Get size classes. */
|
||||||
small0 = get_small_size(0);
|
small0 = get_small_size(0);
|
||||||
hugemax = get_huge_size(get_nhuge()-1);
|
largemax = get_large_size(get_nlarge()-1);
|
||||||
|
|
||||||
p = mallocx(small0, 0);
|
p = mallocx(small0, 0);
|
||||||
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
||||||
|
|
||||||
/* Test overflows that can be resolved by clamping extra. */
|
/* Test overflows that can be resolved by clamping extra. */
|
||||||
assert_zu_le(xallocx(p, hugemax-1, 2, 0), hugemax,
|
assert_zu_le(xallocx(p, largemax-1, 2, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_le(xallocx(p, hugemax, 1, 0), hugemax,
|
assert_zu_le(xallocx(p, largemax, 1, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
/* Test overflow such that hugemax-size underflows. */
|
/* Test overflow such that largemax-size underflows. */
|
||||||
assert_zu_le(xallocx(p, hugemax+1, 2, 0), hugemax,
|
assert_zu_le(xallocx(p, largemax+1, 2, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_le(xallocx(p, hugemax+2, 3, 0), hugemax,
|
assert_zu_le(xallocx(p, largemax+2, 3, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_le(xallocx(p, SIZE_T_MAX-2, 2, 0), hugemax,
|
assert_zu_le(xallocx(p, SIZE_T_MAX-2, 2, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_le(xallocx(p, SIZE_T_MAX-1, 1, 0), hugemax,
|
assert_zu_le(xallocx(p, SIZE_T_MAX-1, 1, 0), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
dallocx(p, 0);
|
dallocx(p, 0);
|
||||||
@ -195,13 +195,13 @@ TEST_END
|
|||||||
|
|
||||||
TEST_BEGIN(test_extra_small)
|
TEST_BEGIN(test_extra_small)
|
||||||
{
|
{
|
||||||
size_t small0, small1, hugemax;
|
size_t small0, small1, largemax;
|
||||||
void *p;
|
void *p;
|
||||||
|
|
||||||
/* Get size classes. */
|
/* Get size classes. */
|
||||||
small0 = get_small_size(0);
|
small0 = get_small_size(0);
|
||||||
small1 = get_small_size(1);
|
small1 = get_small_size(1);
|
||||||
hugemax = get_huge_size(get_nhuge()-1);
|
largemax = get_large_size(get_nlarge()-1);
|
||||||
|
|
||||||
p = mallocx(small0, 0);
|
p = mallocx(small0, 0);
|
||||||
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
||||||
@ -216,7 +216,7 @@ TEST_BEGIN(test_extra_small)
|
|||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
/* Test size+extra overflow. */
|
/* Test size+extra overflow. */
|
||||||
assert_zu_eq(xallocx(p, small0, hugemax - small0 + 1, 0), small0,
|
assert_zu_eq(xallocx(p, small0, largemax - small0 + 1, 0), small0,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_eq(xallocx(p, small0, SIZE_T_MAX - small0, 0), small0,
|
assert_zu_eq(xallocx(p, small0, SIZE_T_MAX - small0, 0), small0,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
@ -225,66 +225,66 @@ TEST_BEGIN(test_extra_small)
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
TEST_BEGIN(test_extra_huge)
|
TEST_BEGIN(test_extra_large)
|
||||||
{
|
{
|
||||||
int flags = MALLOCX_ARENA(arena_ind());
|
int flags = MALLOCX_ARENA(arena_ind());
|
||||||
size_t smallmax, huge1, huge2, huge3, hugemax;
|
size_t smallmax, large1, large2, large3, largemax;
|
||||||
void *p;
|
void *p;
|
||||||
|
|
||||||
/* Get size classes. */
|
/* Get size classes. */
|
||||||
smallmax = get_small_size(get_nsmall()-1);
|
smallmax = get_small_size(get_nsmall()-1);
|
||||||
huge1 = get_huge_size(1);
|
large1 = get_large_size(1);
|
||||||
huge2 = get_huge_size(2);
|
large2 = get_large_size(2);
|
||||||
huge3 = get_huge_size(3);
|
large3 = get_large_size(3);
|
||||||
hugemax = get_huge_size(get_nhuge()-1);
|
largemax = get_large_size(get_nlarge()-1);
|
||||||
|
|
||||||
p = mallocx(huge3, flags);
|
p = mallocx(large3, flags);
|
||||||
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
||||||
|
|
||||||
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3,
|
assert_zu_eq(xallocx(p, large3, 0, flags), large3,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
/* Test size decrease with zero extra. */
|
/* Test size decrease with zero extra. */
|
||||||
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1,
|
assert_zu_ge(xallocx(p, large1, 0, flags), large1,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_ge(xallocx(p, smallmax, 0, flags), huge1,
|
assert_zu_ge(xallocx(p, smallmax, 0, flags), large1,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3,
|
assert_zu_eq(xallocx(p, large3, 0, flags), large3,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
/* Test size decrease with non-zero extra. */
|
/* Test size decrease with non-zero extra. */
|
||||||
assert_zu_eq(xallocx(p, huge1, huge3 - huge1, flags), huge3,
|
assert_zu_eq(xallocx(p, large1, large3 - large1, flags), large3,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_eq(xallocx(p, huge2, huge3 - huge2, flags), huge3,
|
assert_zu_eq(xallocx(p, large2, large3 - large2, flags), large3,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_eq(xallocx(p, huge1, huge2 - huge1, flags), huge2,
|
assert_zu_eq(xallocx(p, large1, large2 - large1, flags), large2,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_ge(xallocx(p, smallmax, huge1 - smallmax, flags), huge1,
|
assert_zu_ge(xallocx(p, smallmax, large1 - smallmax, flags), large1,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1,
|
assert_zu_ge(xallocx(p, large1, 0, flags), large1,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
/* Test size increase with zero extra. */
|
/* Test size increase with zero extra. */
|
||||||
assert_zu_le(xallocx(p, huge3, 0, flags), huge3,
|
assert_zu_le(xallocx(p, large3, 0, flags), large3,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
assert_zu_le(xallocx(p, hugemax+1, 0, flags), huge3,
|
assert_zu_le(xallocx(p, largemax+1, 0, flags), large3,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1,
|
assert_zu_ge(xallocx(p, large1, 0, flags), large1,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
/* Test size increase with non-zero extra. */
|
/* Test size increase with non-zero extra. */
|
||||||
assert_zu_le(xallocx(p, huge1, SIZE_T_MAX - huge1, flags), hugemax,
|
assert_zu_le(xallocx(p, large1, SIZE_T_MAX - large1, flags), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1,
|
assert_zu_ge(xallocx(p, large1, 0, flags), large1,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
/* Test size increase with non-zero extra. */
|
/* Test size increase with non-zero extra. */
|
||||||
assert_zu_le(xallocx(p, huge1, huge3 - huge1, flags), huge3,
|
assert_zu_le(xallocx(p, large1, large3 - large1, flags), large3,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3,
|
assert_zu_eq(xallocx(p, large3, 0, flags), large3,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
/* Test size+extra overflow. */
|
/* Test size+extra overflow. */
|
||||||
assert_zu_le(xallocx(p, huge3, hugemax - huge3 + 1, flags), hugemax,
|
assert_zu_le(xallocx(p, large3, largemax - large3 + 1, flags), largemax,
|
||||||
"Unexpected xallocx() behavior");
|
"Unexpected xallocx() behavior");
|
||||||
|
|
||||||
dallocx(p, flags);
|
dallocx(p, flags);
|
||||||
@ -374,15 +374,15 @@ test_zero(size_t szmin, size_t szmax)
|
|||||||
dallocx(p, flags);
|
dallocx(p, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_BEGIN(test_zero_huge)
|
TEST_BEGIN(test_zero_large)
|
||||||
{
|
{
|
||||||
size_t huge0, huge1;
|
size_t large0, large1;
|
||||||
|
|
||||||
/* Get size classes. */
|
/* Get size classes. */
|
||||||
huge0 = get_huge_size(0);
|
large0 = get_large_size(0);
|
||||||
huge1 = get_huge_size(1);
|
large1 = get_large_size(1);
|
||||||
|
|
||||||
test_zero(huge1, huge0 * 2);
|
test_zero(large1, large0 * 2);
|
||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
@ -397,6 +397,6 @@ main(void)
|
|||||||
test_size,
|
test_size,
|
||||||
test_size_extra_overflow,
|
test_size_extra_overflow,
|
||||||
test_extra_small,
|
test_extra_small,
|
||||||
test_extra_huge,
|
test_extra_large,
|
||||||
test_zero_huge));
|
test_zero_large));
|
||||||
}
|
}
|
||||||
|
@ -25,10 +25,10 @@ get_nsmall(void)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static unsigned
|
static unsigned
|
||||||
get_nhuge(void)
|
get_nlarge(void)
|
||||||
{
|
{
|
||||||
|
|
||||||
return (get_nsizes_impl("arenas.nhchunks"));
|
return (get_nsizes_impl("arenas.nlextents"));
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
@ -58,10 +58,10 @@ get_small_size(size_t ind)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
get_huge_size(size_t ind)
|
get_large_size(size_t ind)
|
||||||
{
|
{
|
||||||
|
|
||||||
return (get_size_impl("arenas.hchunk.0.size", ind));
|
return (get_size_impl("arenas.lextent.0.size", ind));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Like ivsalloc(), but safe to call on discarded allocations. */
|
/* Like ivsalloc(), but safe to call on discarded allocations. */
|
||||||
@ -81,8 +81,8 @@ vsalloc(tsdn_t *tsdn, const void *ptr)
|
|||||||
|
|
||||||
TEST_BEGIN(test_arena_reset)
|
TEST_BEGIN(test_arena_reset)
|
||||||
{
|
{
|
||||||
#define NHUGE 32
|
#define NLARGE 32
|
||||||
unsigned arena_ind, nsmall, nhuge, nptrs, i;
|
unsigned arena_ind, nsmall, nlarge, nptrs, i;
|
||||||
size_t sz, miblen;
|
size_t sz, miblen;
|
||||||
void **ptrs;
|
void **ptrs;
|
||||||
int flags;
|
int flags;
|
||||||
@ -96,8 +96,8 @@ TEST_BEGIN(test_arena_reset)
|
|||||||
flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
|
flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
|
||||||
|
|
||||||
nsmall = get_nsmall();
|
nsmall = get_nsmall();
|
||||||
nhuge = get_nhuge() > NHUGE ? NHUGE : get_nhuge();
|
nlarge = get_nlarge() > NLARGE ? NLARGE : get_nlarge();
|
||||||
nptrs = nsmall + nhuge;
|
nptrs = nsmall + nlarge;
|
||||||
ptrs = (void **)malloc(nptrs * sizeof(void *));
|
ptrs = (void **)malloc(nptrs * sizeof(void *));
|
||||||
assert_ptr_not_null(ptrs, "Unexpected malloc() failure");
|
assert_ptr_not_null(ptrs, "Unexpected malloc() failure");
|
||||||
|
|
||||||
@ -108,8 +108,8 @@ TEST_BEGIN(test_arena_reset)
|
|||||||
assert_ptr_not_null(ptrs[i],
|
assert_ptr_not_null(ptrs[i],
|
||||||
"Unexpected mallocx(%zu, %#x) failure", sz, flags);
|
"Unexpected mallocx(%zu, %#x) failure", sz, flags);
|
||||||
}
|
}
|
||||||
for (i = 0; i < nhuge; i++) {
|
for (i = 0; i < nlarge; i++) {
|
||||||
sz = get_huge_size(i);
|
sz = get_large_size(i);
|
||||||
ptrs[nsmall + i] = mallocx(sz, flags);
|
ptrs[nsmall + i] = mallocx(sz, flags);
|
||||||
assert_ptr_not_null(ptrs[i],
|
assert_ptr_not_null(ptrs[i],
|
||||||
"Unexpected mallocx(%zu, %#x) failure", sz, flags);
|
"Unexpected mallocx(%zu, %#x) failure", sz, flags);
|
||||||
|
@ -22,7 +22,7 @@ TEST_BEGIN(test_decay_ticks)
|
|||||||
{
|
{
|
||||||
ticker_t *decay_ticker;
|
ticker_t *decay_ticker;
|
||||||
unsigned tick0, tick1;
|
unsigned tick0, tick1;
|
||||||
size_t sz, huge0;
|
size_t sz, large0;
|
||||||
void *p;
|
void *p;
|
||||||
|
|
||||||
test_skip_if(opt_purge != purge_mode_decay);
|
test_skip_if(opt_purge != purge_mode_decay);
|
||||||
@ -32,18 +32,18 @@ TEST_BEGIN(test_decay_ticks)
|
|||||||
"Unexpected failure getting decay ticker");
|
"Unexpected failure getting decay ticker");
|
||||||
|
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.lextent.0.size", &large0, &sz, NULL, 0), 0,
|
||||||
"Unexpected mallctl failure");
|
"Unexpected mallctl failure");
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Test the standard APIs using a huge size class, since we can't
|
* Test the standard APIs using a large size class, since we can't
|
||||||
* control tcache interactions for small size classes (except by
|
* control tcache interactions for small size classes (except by
|
||||||
* completely disabling tcache for the entire test program).
|
* completely disabling tcache for the entire test program).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* malloc(). */
|
/* malloc(). */
|
||||||
tick0 = ticker_read(decay_ticker);
|
tick0 = ticker_read(decay_ticker);
|
||||||
p = malloc(huge0);
|
p = malloc(large0);
|
||||||
assert_ptr_not_null(p, "Unexpected malloc() failure");
|
assert_ptr_not_null(p, "Unexpected malloc() failure");
|
||||||
tick1 = ticker_read(decay_ticker);
|
tick1 = ticker_read(decay_ticker);
|
||||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during malloc()");
|
assert_u32_ne(tick1, tick0, "Expected ticker to tick during malloc()");
|
||||||
@ -55,7 +55,7 @@ TEST_BEGIN(test_decay_ticks)
|
|||||||
|
|
||||||
/* calloc(). */
|
/* calloc(). */
|
||||||
tick0 = ticker_read(decay_ticker);
|
tick0 = ticker_read(decay_ticker);
|
||||||
p = calloc(1, huge0);
|
p = calloc(1, large0);
|
||||||
assert_ptr_not_null(p, "Unexpected calloc() failure");
|
assert_ptr_not_null(p, "Unexpected calloc() failure");
|
||||||
tick1 = ticker_read(decay_ticker);
|
tick1 = ticker_read(decay_ticker);
|
||||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during calloc()");
|
assert_u32_ne(tick1, tick0, "Expected ticker to tick during calloc()");
|
||||||
@ -63,7 +63,7 @@ TEST_BEGIN(test_decay_ticks)
|
|||||||
|
|
||||||
/* posix_memalign(). */
|
/* posix_memalign(). */
|
||||||
tick0 = ticker_read(decay_ticker);
|
tick0 = ticker_read(decay_ticker);
|
||||||
assert_d_eq(posix_memalign(&p, sizeof(size_t), huge0), 0,
|
assert_d_eq(posix_memalign(&p, sizeof(size_t), large0), 0,
|
||||||
"Unexpected posix_memalign() failure");
|
"Unexpected posix_memalign() failure");
|
||||||
tick1 = ticker_read(decay_ticker);
|
tick1 = ticker_read(decay_ticker);
|
||||||
assert_u32_ne(tick1, tick0,
|
assert_u32_ne(tick1, tick0,
|
||||||
@ -72,7 +72,7 @@ TEST_BEGIN(test_decay_ticks)
|
|||||||
|
|
||||||
/* aligned_alloc(). */
|
/* aligned_alloc(). */
|
||||||
tick0 = ticker_read(decay_ticker);
|
tick0 = ticker_read(decay_ticker);
|
||||||
p = aligned_alloc(sizeof(size_t), huge0);
|
p = aligned_alloc(sizeof(size_t), large0);
|
||||||
assert_ptr_not_null(p, "Unexpected aligned_alloc() failure");
|
assert_ptr_not_null(p, "Unexpected aligned_alloc() failure");
|
||||||
tick1 = ticker_read(decay_ticker);
|
tick1 = ticker_read(decay_ticker);
|
||||||
assert_u32_ne(tick1, tick0,
|
assert_u32_ne(tick1, tick0,
|
||||||
@ -82,13 +82,13 @@ TEST_BEGIN(test_decay_ticks)
|
|||||||
/* realloc(). */
|
/* realloc(). */
|
||||||
/* Allocate. */
|
/* Allocate. */
|
||||||
tick0 = ticker_read(decay_ticker);
|
tick0 = ticker_read(decay_ticker);
|
||||||
p = realloc(NULL, huge0);
|
p = realloc(NULL, large0);
|
||||||
assert_ptr_not_null(p, "Unexpected realloc() failure");
|
assert_ptr_not_null(p, "Unexpected realloc() failure");
|
||||||
tick1 = ticker_read(decay_ticker);
|
tick1 = ticker_read(decay_ticker);
|
||||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
||||||
/* Reallocate. */
|
/* Reallocate. */
|
||||||
tick0 = ticker_read(decay_ticker);
|
tick0 = ticker_read(decay_ticker);
|
||||||
p = realloc(p, huge0);
|
p = realloc(p, large0);
|
||||||
assert_ptr_not_null(p, "Unexpected realloc() failure");
|
assert_ptr_not_null(p, "Unexpected realloc() failure");
|
||||||
tick1 = ticker_read(decay_ticker);
|
tick1 = ticker_read(decay_ticker);
|
||||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
||||||
@ -99,13 +99,13 @@ TEST_BEGIN(test_decay_ticks)
|
|||||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Test the *allocx() APIs using huge and small size classes, with
|
* Test the *allocx() APIs using large and small size classes, with
|
||||||
* tcache explicitly disabled.
|
* tcache explicitly disabled.
|
||||||
*/
|
*/
|
||||||
{
|
{
|
||||||
unsigned i;
|
unsigned i;
|
||||||
size_t allocx_sizes[2];
|
size_t allocx_sizes[2];
|
||||||
allocx_sizes[0] = huge0;
|
allocx_sizes[0] = large0;
|
||||||
allocx_sizes[1] = 1;
|
allocx_sizes[1] = 1;
|
||||||
|
|
||||||
for (i = 0; i < sizeof(allocx_sizes) / sizeof(size_t); i++) {
|
for (i = 0; i < sizeof(allocx_sizes) / sizeof(size_t); i++) {
|
||||||
@ -154,13 +154,13 @@ TEST_BEGIN(test_decay_ticks)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Test tcache fill/flush interactions for huge and small size classes,
|
* Test tcache fill/flush interactions for large and small size classes,
|
||||||
* using an explicit tcache.
|
* using an explicit tcache.
|
||||||
*/
|
*/
|
||||||
if (config_tcache) {
|
if (config_tcache) {
|
||||||
unsigned tcache_ind, i;
|
unsigned tcache_ind, i;
|
||||||
size_t tcache_sizes[2];
|
size_t tcache_sizes[2];
|
||||||
tcache_sizes[0] = huge0;
|
tcache_sizes[0] = large0;
|
||||||
tcache_sizes[1] = 1;
|
tcache_sizes[1] = 1;
|
||||||
|
|
||||||
sz = sizeof(unsigned);
|
sz = sizeof(unsigned);
|
||||||
@ -201,14 +201,14 @@ TEST_BEGIN(test_decay_ticker)
|
|||||||
uint64_t epoch;
|
uint64_t epoch;
|
||||||
uint64_t npurge0 = 0;
|
uint64_t npurge0 = 0;
|
||||||
uint64_t npurge1 = 0;
|
uint64_t npurge1 = 0;
|
||||||
size_t sz, huge;
|
size_t sz, large;
|
||||||
unsigned i, nupdates0;
|
unsigned i, nupdates0;
|
||||||
nstime_t time, decay_time, deadline;
|
nstime_t time, decay_time, deadline;
|
||||||
|
|
||||||
test_skip_if(opt_purge != purge_mode_decay);
|
test_skip_if(opt_purge != purge_mode_decay);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Allocate a bunch of huge objects, pause the clock, deallocate the
|
* Allocate a bunch of large objects, pause the clock, deallocate the
|
||||||
* objects, restore the clock, then [md]allocx() in a tight loop to
|
* objects, restore the clock, then [md]allocx() in a tight loop to
|
||||||
* verify the ticker triggers purging.
|
* verify the ticker triggers purging.
|
||||||
*/
|
*/
|
||||||
@ -219,10 +219,10 @@ TEST_BEGIN(test_decay_ticker)
|
|||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("arenas.tcache_max", &tcache_max, &sz, NULL,
|
assert_d_eq(mallctl("arenas.tcache_max", &tcache_max, &sz, NULL,
|
||||||
0), 0, "Unexpected mallctl failure");
|
0), 0, "Unexpected mallctl failure");
|
||||||
huge = nallocx(tcache_max + 1, flags);
|
large = nallocx(tcache_max + 1, flags);
|
||||||
} else {
|
} else {
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge, &sz, NULL,
|
assert_d_eq(mallctl("arenas.lextent.0.size", &large, &sz, NULL,
|
||||||
0), 0, "Unexpected mallctl failure");
|
0), 0, "Unexpected mallctl failure");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -235,7 +235,7 @@ TEST_BEGIN(test_decay_ticker)
|
|||||||
config_stats ? 0 : ENOENT, "Unexpected mallctl result");
|
config_stats ? 0 : ENOENT, "Unexpected mallctl result");
|
||||||
|
|
||||||
for (i = 0; i < NPS; i++) {
|
for (i = 0; i < NPS; i++) {
|
||||||
ps[i] = mallocx(huge, flags);
|
ps[i] = mallocx(large, flags);
|
||||||
assert_ptr_not_null(ps[i], "Unexpected mallocx() failure");
|
assert_ptr_not_null(ps[i], "Unexpected mallocx() failure");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -293,13 +293,13 @@ TEST_BEGIN(test_decay_nonmonotonic)
|
|||||||
uint64_t epoch;
|
uint64_t epoch;
|
||||||
uint64_t npurge0 = 0;
|
uint64_t npurge0 = 0;
|
||||||
uint64_t npurge1 = 0;
|
uint64_t npurge1 = 0;
|
||||||
size_t sz, huge0;
|
size_t sz, large0;
|
||||||
unsigned i, nupdates0;
|
unsigned i, nupdates0;
|
||||||
|
|
||||||
test_skip_if(opt_purge != purge_mode_decay);
|
test_skip_if(opt_purge != purge_mode_decay);
|
||||||
|
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.lextent.0.size", &large0, &sz, NULL, 0), 0,
|
||||||
"Unexpected mallctl failure");
|
"Unexpected mallctl failure");
|
||||||
|
|
||||||
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
|
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
|
||||||
@ -319,7 +319,7 @@ TEST_BEGIN(test_decay_nonmonotonic)
|
|||||||
nstime_update = nstime_update_mock;
|
nstime_update = nstime_update_mock;
|
||||||
|
|
||||||
for (i = 0; i < NPS; i++) {
|
for (i = 0; i < NPS; i++) {
|
||||||
ps[i] = mallocx(huge0, flags);
|
ps[i] = mallocx(large0, flags);
|
||||||
assert_ptr_not_null(ps[i], "Unexpected mallocx() failure");
|
assert_ptr_not_null(ps[i], "Unexpected mallocx() failure");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -35,16 +35,16 @@ TEST_BEGIN(test_small_extent_size)
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
TEST_BEGIN(test_huge_extent_size)
|
TEST_BEGIN(test_large_extent_size)
|
||||||
{
|
{
|
||||||
bool cache_oblivious;
|
bool cache_oblivious;
|
||||||
unsigned nhchunks, i;
|
unsigned nlextents, i;
|
||||||
size_t sz, extent_size_prev, ceil_prev;
|
size_t sz, extent_size_prev, ceil_prev;
|
||||||
size_t mib[4];
|
size_t mib[4];
|
||||||
size_t miblen = sizeof(mib) / sizeof(size_t);
|
size_t miblen = sizeof(mib) / sizeof(size_t);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Iterate over all huge size classes, get their extent sizes, and
|
* Iterate over all large size classes, get their extent sizes, and
|
||||||
* verify that the quantized size is the same as the extent size.
|
* verify that the quantized size is the same as the extent size.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@ -53,12 +53,12 @@ TEST_BEGIN(test_huge_extent_size)
|
|||||||
NULL, 0), 0, "Unexpected mallctl failure");
|
NULL, 0), 0, "Unexpected mallctl failure");
|
||||||
|
|
||||||
sz = sizeof(unsigned);
|
sz = sizeof(unsigned);
|
||||||
assert_d_eq(mallctl("arenas.nhchunks", &nhchunks, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.nlextents", &nlextents, &sz, NULL, 0), 0,
|
||||||
"Unexpected mallctl failure");
|
"Unexpected mallctl failure");
|
||||||
|
|
||||||
assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0,
|
assert_d_eq(mallctlnametomib("arenas.lextent.0.size", mib, &miblen), 0,
|
||||||
"Unexpected mallctlnametomib failure");
|
"Unexpected mallctlnametomib failure");
|
||||||
for (i = 0; i < nhchunks; i++) {
|
for (i = 0; i < nlextents; i++) {
|
||||||
size_t lextent_size, extent_size, floor, ceil;
|
size_t lextent_size, extent_size, floor, ceil;
|
||||||
|
|
||||||
mib[2] = i;
|
mib[2] = i;
|
||||||
@ -91,7 +91,7 @@ TEST_BEGIN(test_huge_extent_size)
|
|||||||
ceil_prev, extent_size);
|
ceil_prev, extent_size);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (i + 1 < nhchunks) {
|
if (i + 1 < nlextents) {
|
||||||
extent_size_prev = floor;
|
extent_size_prev = floor;
|
||||||
ceil_prev = extent_size_quantize_ceil(extent_size +
|
ceil_prev = extent_size_quantize_ceil(extent_size +
|
||||||
PAGE);
|
PAGE);
|
||||||
@ -141,6 +141,6 @@ main(void)
|
|||||||
|
|
||||||
return (test(
|
return (test(
|
||||||
test_small_extent_size,
|
test_small_extent_size,
|
||||||
test_huge_extent_size,
|
test_large_extent_size,
|
||||||
test_monotonic));
|
test_monotonic));
|
||||||
}
|
}
|
||||||
|
@ -9,7 +9,7 @@ const char *malloc_conf =
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig;
|
static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig;
|
||||||
static huge_dalloc_junk_t *huge_dalloc_junk_orig;
|
static large_dalloc_junk_t *large_dalloc_junk_orig;
|
||||||
static void *watch_for_junking;
|
static void *watch_for_junking;
|
||||||
static bool saw_junking;
|
static bool saw_junking;
|
||||||
|
|
||||||
@ -37,10 +37,10 @@ arena_dalloc_junk_small_intercept(void *ptr, const arena_bin_info_t *bin_info)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
huge_dalloc_junk_intercept(void *ptr, size_t usize)
|
large_dalloc_junk_intercept(void *ptr, size_t usize)
|
||||||
{
|
{
|
||||||
|
|
||||||
huge_dalloc_junk_orig(ptr, usize);
|
large_dalloc_junk_orig(ptr, usize);
|
||||||
/*
|
/*
|
||||||
* The conditions under which junk filling actually occurs are nuanced
|
* The conditions under which junk filling actually occurs are nuanced
|
||||||
* enough that it doesn't make sense to duplicate the decision logic in
|
* enough that it doesn't make sense to duplicate the decision logic in
|
||||||
@ -59,8 +59,8 @@ test_junk(size_t sz_min, size_t sz_max)
|
|||||||
if (opt_junk_free) {
|
if (opt_junk_free) {
|
||||||
arena_dalloc_junk_small_orig = arena_dalloc_junk_small;
|
arena_dalloc_junk_small_orig = arena_dalloc_junk_small;
|
||||||
arena_dalloc_junk_small = arena_dalloc_junk_small_intercept;
|
arena_dalloc_junk_small = arena_dalloc_junk_small_intercept;
|
||||||
huge_dalloc_junk_orig = huge_dalloc_junk;
|
large_dalloc_junk_orig = large_dalloc_junk;
|
||||||
huge_dalloc_junk = huge_dalloc_junk_intercept;
|
large_dalloc_junk = large_dalloc_junk_intercept;
|
||||||
}
|
}
|
||||||
|
|
||||||
sz_prev = 0;
|
sz_prev = 0;
|
||||||
@ -110,7 +110,7 @@ test_junk(size_t sz_min, size_t sz_max)
|
|||||||
|
|
||||||
if (opt_junk_free) {
|
if (opt_junk_free) {
|
||||||
arena_dalloc_junk_small = arena_dalloc_junk_small_orig;
|
arena_dalloc_junk_small = arena_dalloc_junk_small_orig;
|
||||||
huge_dalloc_junk = huge_dalloc_junk_orig;
|
large_dalloc_junk = large_dalloc_junk_orig;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -122,7 +122,7 @@ TEST_BEGIN(test_junk_small)
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
TEST_BEGIN(test_junk_huge)
|
TEST_BEGIN(test_junk_large)
|
||||||
{
|
{
|
||||||
|
|
||||||
test_skip_if(!config_fill);
|
test_skip_if(!config_fill);
|
||||||
@ -136,5 +136,5 @@ main(void)
|
|||||||
|
|
||||||
return (test(
|
return (test(
|
||||||
test_junk_small,
|
test_junk_small,
|
||||||
test_junk_huge));
|
test_junk_large));
|
||||||
}
|
}
|
||||||
|
@ -596,7 +596,7 @@ TEST_BEGIN(test_arenas_constants)
|
|||||||
TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM);
|
TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM);
|
||||||
TEST_ARENAS_CONSTANT(size_t, page, PAGE);
|
TEST_ARENAS_CONSTANT(size_t, page, PAGE);
|
||||||
TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS);
|
TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS);
|
||||||
TEST_ARENAS_CONSTANT(unsigned, nhchunks, NSIZES - NBINS);
|
TEST_ARENAS_CONSTANT(unsigned, nlextents, NSIZES - NBINS);
|
||||||
|
|
||||||
#undef TEST_ARENAS_CONSTANT
|
#undef TEST_ARENAS_CONSTANT
|
||||||
}
|
}
|
||||||
@ -622,13 +622,13 @@ TEST_BEGIN(test_arenas_bin_constants)
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
TEST_BEGIN(test_arenas_hchunk_constants)
|
TEST_BEGIN(test_arenas_lextent_constants)
|
||||||
{
|
{
|
||||||
|
|
||||||
#define TEST_ARENAS_HCHUNK_CONSTANT(t, name, expected) do { \
|
#define TEST_ARENAS_HCHUNK_CONSTANT(t, name, expected) do { \
|
||||||
t name; \
|
t name; \
|
||||||
size_t sz = sizeof(t); \
|
size_t sz = sizeof(t); \
|
||||||
assert_d_eq(mallctl("arenas.hchunk.0."#name, &name, &sz, NULL, \
|
assert_d_eq(mallctl("arenas.lextent.0."#name, &name, &sz, NULL, \
|
||||||
0), 0, "Unexpected mallctl() failure"); \
|
0), 0, "Unexpected mallctl() failure"); \
|
||||||
assert_zu_eq(name, expected, "Incorrect "#name" size"); \
|
assert_zu_eq(name, expected, "Incorrect "#name" size"); \
|
||||||
} while (0)
|
} while (0)
|
||||||
@ -704,7 +704,7 @@ main(void)
|
|||||||
test_arenas_decay_time,
|
test_arenas_decay_time,
|
||||||
test_arenas_constants,
|
test_arenas_constants,
|
||||||
test_arenas_bin_constants,
|
test_arenas_bin_constants,
|
||||||
test_arenas_hchunk_constants,
|
test_arenas_lextent_constants,
|
||||||
test_arenas_extend,
|
test_arenas_extend,
|
||||||
test_stats_arenas));
|
test_stats_arenas));
|
||||||
}
|
}
|
||||||
|
@ -3,18 +3,18 @@
|
|||||||
static size_t
|
static size_t
|
||||||
get_max_size_class(void)
|
get_max_size_class(void)
|
||||||
{
|
{
|
||||||
unsigned nhchunks;
|
unsigned nlextents;
|
||||||
size_t mib[4];
|
size_t mib[4];
|
||||||
size_t sz, miblen, max_size_class;
|
size_t sz, miblen, max_size_class;
|
||||||
|
|
||||||
sz = sizeof(unsigned);
|
sz = sizeof(unsigned);
|
||||||
assert_d_eq(mallctl("arenas.nhchunks", &nhchunks, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.nlextents", &nlextents, &sz, NULL, 0), 0,
|
||||||
"Unexpected mallctl() error");
|
"Unexpected mallctl() error");
|
||||||
|
|
||||||
miblen = sizeof(mib) / sizeof(size_t);
|
miblen = sizeof(mib) / sizeof(size_t);
|
||||||
assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0,
|
assert_d_eq(mallctlnametomib("arenas.lextent.0.size", mib, &miblen), 0,
|
||||||
"Unexpected mallctlnametomib() error");
|
"Unexpected mallctlnametomib() error");
|
||||||
mib[2] = nhchunks - 1;
|
mib[2] = nlextents - 1;
|
||||||
|
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0,
|
assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0,
|
||||||
|
@ -33,7 +33,7 @@ TEST_BEGIN(test_stats_summary)
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
TEST_BEGIN(test_stats_huge)
|
TEST_BEGIN(test_stats_large)
|
||||||
{
|
{
|
||||||
void *p;
|
void *p;
|
||||||
uint64_t epoch;
|
uint64_t epoch;
|
||||||
@ -49,14 +49,14 @@ TEST_BEGIN(test_stats_huge)
|
|||||||
"Unexpected mallctl() failure");
|
"Unexpected mallctl() failure");
|
||||||
|
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("stats.arenas.0.huge.allocated", &allocated, &sz,
|
assert_d_eq(mallctl("stats.arenas.0.large.allocated", &allocated, &sz,
|
||||||
NULL, 0), expected, "Unexpected mallctl() result");
|
NULL, 0), expected, "Unexpected mallctl() result");
|
||||||
sz = sizeof(uint64_t);
|
sz = sizeof(uint64_t);
|
||||||
assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", &nmalloc, &sz, NULL,
|
assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", &nmalloc, &sz, NULL,
|
||||||
0), expected, "Unexpected mallctl() result");
|
0), expected, "Unexpected mallctl() result");
|
||||||
assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", &ndalloc, &sz, NULL,
|
assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", &ndalloc, &sz, NULL,
|
||||||
0), expected, "Unexpected mallctl() result");
|
0), expected, "Unexpected mallctl() result");
|
||||||
assert_d_eq(mallctl("stats.arenas.0.huge.nrequests", &nrequests, &sz,
|
assert_d_eq(mallctl("stats.arenas.0.large.nrequests", &nrequests, &sz,
|
||||||
NULL, 0), expected, "Unexpected mallctl() result");
|
NULL, 0), expected, "Unexpected mallctl() result");
|
||||||
|
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
@ -75,7 +75,7 @@ TEST_END
|
|||||||
TEST_BEGIN(test_stats_arenas_summary)
|
TEST_BEGIN(test_stats_arenas_summary)
|
||||||
{
|
{
|
||||||
unsigned arena;
|
unsigned arena;
|
||||||
void *little, *huge;
|
void *little, *large;
|
||||||
uint64_t epoch;
|
uint64_t epoch;
|
||||||
size_t sz;
|
size_t sz;
|
||||||
int expected = config_stats ? 0 : ENOENT;
|
int expected = config_stats ? 0 : ENOENT;
|
||||||
@ -88,11 +88,11 @@ TEST_BEGIN(test_stats_arenas_summary)
|
|||||||
|
|
||||||
little = mallocx(SMALL_MAXCLASS, 0);
|
little = mallocx(SMALL_MAXCLASS, 0);
|
||||||
assert_ptr_not_null(little, "Unexpected mallocx() failure");
|
assert_ptr_not_null(little, "Unexpected mallocx() failure");
|
||||||
huge = mallocx(chunksize, 0);
|
large = mallocx(chunksize, 0);
|
||||||
assert_ptr_not_null(huge, "Unexpected mallocx() failure");
|
assert_ptr_not_null(large, "Unexpected mallocx() failure");
|
||||||
|
|
||||||
dallocx(little, 0);
|
dallocx(little, 0);
|
||||||
dallocx(huge, 0);
|
dallocx(large, 0);
|
||||||
|
|
||||||
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
|
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
|
||||||
"Unexpected mallctl() failure");
|
"Unexpected mallctl() failure");
|
||||||
@ -185,7 +185,7 @@ TEST_BEGIN(test_stats_arenas_small)
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
TEST_BEGIN(test_stats_arenas_huge)
|
TEST_BEGIN(test_stats_arenas_large)
|
||||||
{
|
{
|
||||||
unsigned arena;
|
unsigned arena;
|
||||||
void *p;
|
void *p;
|
||||||
@ -204,12 +204,12 @@ TEST_BEGIN(test_stats_arenas_huge)
|
|||||||
"Unexpected mallctl() failure");
|
"Unexpected mallctl() failure");
|
||||||
|
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("stats.arenas.0.huge.allocated", &allocated, &sz,
|
assert_d_eq(mallctl("stats.arenas.0.large.allocated", &allocated, &sz,
|
||||||
NULL, 0), expected, "Unexpected mallctl() result");
|
NULL, 0), expected, "Unexpected mallctl() result");
|
||||||
sz = sizeof(uint64_t);
|
sz = sizeof(uint64_t);
|
||||||
assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", &nmalloc, &sz,
|
assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", &nmalloc, &sz,
|
||||||
NULL, 0), expected, "Unexpected mallctl() result");
|
NULL, 0), expected, "Unexpected mallctl() result");
|
||||||
assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", &ndalloc, &sz,
|
assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", &ndalloc, &sz,
|
||||||
NULL, 0), expected, "Unexpected mallctl() result");
|
NULL, 0), expected, "Unexpected mallctl() result");
|
||||||
|
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
@ -299,12 +299,12 @@ TEST_BEGIN(test_stats_arenas_bins)
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
TEST_BEGIN(test_stats_arenas_hchunks)
|
TEST_BEGIN(test_stats_arenas_lextents)
|
||||||
{
|
{
|
||||||
unsigned arena;
|
unsigned arena;
|
||||||
void *p;
|
void *p;
|
||||||
uint64_t epoch, nmalloc, ndalloc;
|
uint64_t epoch, nmalloc, ndalloc;
|
||||||
size_t curhchunks, sz, hsize;
|
size_t curlextents, sz, hsize;
|
||||||
int expected = config_stats ? 0 : ENOENT;
|
int expected = config_stats ? 0 : ENOENT;
|
||||||
|
|
||||||
arena = 0;
|
arena = 0;
|
||||||
@ -312,7 +312,7 @@ TEST_BEGIN(test_stats_arenas_hchunks)
|
|||||||
0, "Unexpected mallctl() failure");
|
0, "Unexpected mallctl() failure");
|
||||||
|
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("arenas.hchunk.0.size", &hsize, &sz, NULL, 0), 0,
|
assert_d_eq(mallctl("arenas.lextent.0.size", &hsize, &sz, NULL, 0), 0,
|
||||||
"Unexpected mallctl() failure");
|
"Unexpected mallctl() failure");
|
||||||
|
|
||||||
p = mallocx(hsize, 0);
|
p = mallocx(hsize, 0);
|
||||||
@ -322,20 +322,20 @@ TEST_BEGIN(test_stats_arenas_hchunks)
|
|||||||
"Unexpected mallctl() failure");
|
"Unexpected mallctl() failure");
|
||||||
|
|
||||||
sz = sizeof(uint64_t);
|
sz = sizeof(uint64_t);
|
||||||
assert_d_eq(mallctl("stats.arenas.0.hchunks.0.nmalloc", &nmalloc, &sz,
|
assert_d_eq(mallctl("stats.arenas.0.lextents.0.nmalloc", &nmalloc, &sz,
|
||||||
NULL, 0), expected, "Unexpected mallctl() result");
|
NULL, 0), expected, "Unexpected mallctl() result");
|
||||||
assert_d_eq(mallctl("stats.arenas.0.hchunks.0.ndalloc", &ndalloc, &sz,
|
assert_d_eq(mallctl("stats.arenas.0.lextents.0.ndalloc", &ndalloc, &sz,
|
||||||
NULL, 0), expected, "Unexpected mallctl() result");
|
NULL, 0), expected, "Unexpected mallctl() result");
|
||||||
sz = sizeof(size_t);
|
sz = sizeof(size_t);
|
||||||
assert_d_eq(mallctl("stats.arenas.0.hchunks.0.curhchunks", &curhchunks,
|
assert_d_eq(mallctl("stats.arenas.0.lextents.0.curlextents",
|
||||||
&sz, NULL, 0), expected, "Unexpected mallctl() result");
|
&curlextents, &sz, NULL, 0), expected, "Unexpected mallctl() result");
|
||||||
|
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
assert_u64_gt(nmalloc, 0,
|
assert_u64_gt(nmalloc, 0,
|
||||||
"nmalloc should be greater than zero");
|
"nmalloc should be greater than zero");
|
||||||
assert_u64_ge(nmalloc, ndalloc,
|
assert_u64_ge(nmalloc, ndalloc,
|
||||||
"nmalloc should be at least as large as ndalloc");
|
"nmalloc should be at least as large as ndalloc");
|
||||||
assert_u64_gt(curhchunks, 0,
|
assert_u64_gt(curlextents, 0,
|
||||||
"At least one chunk should be currently allocated");
|
"At least one chunk should be currently allocated");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -349,10 +349,10 @@ main(void)
|
|||||||
|
|
||||||
return (test(
|
return (test(
|
||||||
test_stats_summary,
|
test_stats_summary,
|
||||||
test_stats_huge,
|
test_stats_large,
|
||||||
test_stats_arenas_summary,
|
test_stats_arenas_summary,
|
||||||
test_stats_arenas_small,
|
test_stats_arenas_small,
|
||||||
test_stats_arenas_huge,
|
test_stats_arenas_large,
|
||||||
test_stats_arenas_bins,
|
test_stats_arenas_bins,
|
||||||
test_stats_arenas_hchunks));
|
test_stats_arenas_lextents));
|
||||||
}
|
}
|
||||||
|
@ -53,7 +53,7 @@ TEST_BEGIN(test_zero_small)
|
|||||||
}
|
}
|
||||||
TEST_END
|
TEST_END
|
||||||
|
|
||||||
TEST_BEGIN(test_zero_huge)
|
TEST_BEGIN(test_zero_large)
|
||||||
{
|
{
|
||||||
|
|
||||||
test_skip_if(!config_fill);
|
test_skip_if(!config_fill);
|
||||||
@ -67,5 +67,5 @@ main(void)
|
|||||||
|
|
||||||
return (test(
|
return (test(
|
||||||
test_zero_small,
|
test_zero_small,
|
||||||
test_zero_huge));
|
test_zero_large));
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user