project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rename huge to large.

Browse Source
This commit is contained in:
Jason Evans 2016-05-31 14:50:21 -07:00
parent 714d1640f3
commit 7d63fed0fd
37 changed files with 587 additions and 626 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile.in
View File

@ -88,7 +88,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/ctl.c \ $(srcroot)src/ctl.c \
$(srcroot)src/extent.c \ $(srcroot)src/extent.c \
$(srcroot)src/hash.c \ $(srcroot)src/hash.c \
$(srcroot)src/huge.c \ $(srcroot)src/large.c \
$(srcroot)src/mb.c \ $(srcroot)src/mb.c \
$(srcroot)src/mutex.c \ $(srcroot)src/mutex.c \
$(srcroot)src/nstime.c \ $(srcroot)src/nstime.c \

74
doc/jemalloc.xml.in
View File

@ -1872,22 +1872,22 @@ typedef struct {
<listitem><para>Number of bytes per slab.</para></listitem> <listitem><para>Number of bytes per slab.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arenas.nhchunks"> <varlistentry id="arenas.nlextents">
<term> <term>
<mallctl>arenas.nhchunks</mallctl> <mallctl>arenas.nlextents</mallctl>
(<type>unsigned</type>) (<type>unsigned</type>)
<literal>r-</literal> <literal>r-</literal>
</term> </term>
<listitem><para>Total number of huge size classes.</para></listitem> <listitem><para>Total number of large size classes.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arenas.hchunk.i.size"> <varlistentry id="arenas.lextent.i.size">
<term> <term>
<mallctl>arenas.hchunk.&lt;i&gt;.size</mallctl> <mallctl>arenas.lextent.&lt;i&gt;.size</mallctl>
(<type>size_t</type>) (<type>size_t</type>)
<literal>r-</literal> <literal>r-</literal>
</term> </term>
<listitem><para>Maximum size supported by this huge size <listitem><para>Maximum size supported by this large size
class.</para></listitem> class.</para></listitem>
</varlistentry> </varlistentry>
@ -2361,50 +2361,6 @@ typedef struct {
</para></listitem> </para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.huge.allocated">
<term>
<mallctl>stats.arenas.&lt;i&gt;.huge.allocated</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Number of bytes currently allocated by huge objects.
</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.huge.nmalloc">
<term>
<mallctl>stats.arenas.&lt;i&gt;.huge.nmalloc</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Cumulative number of huge allocation requests served
directly by the arena.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.huge.ndalloc">
<term>
<mallctl>stats.arenas.&lt;i&gt;.huge.ndalloc</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Cumulative number of huge deallocation requests served
directly by the arena.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.huge.nrequests">
<term>
<mallctl>stats.arenas.&lt;i&gt;.huge.nrequests</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Cumulative number of huge allocation requests.
</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.bins.j.nmalloc"> <varlistentry id="stats.arenas.i.bins.j.nmalloc">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nmalloc</mallctl> <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nmalloc</mallctl>
@ -2500,9 +2456,9 @@ typedef struct {
<listitem><para>Current number of slabs.</para></listitem> <listitem><para>Current number of slabs.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.hchunks.j.nmalloc"> <varlistentry id="stats.arenas.i.lextents.j.nmalloc">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.hchunks.&lt;j&gt;.nmalloc</mallctl> <mallctl>stats.arenas.&lt;i&gt;.lextents.&lt;j&gt;.nmalloc</mallctl>
(<type>uint64_t</type>) (<type>uint64_t</type>)
<literal>r-</literal> <literal>r-</literal>
[<option>--enable-stats</option>] [<option>--enable-stats</option>]
@ -2511,9 +2467,9 @@ typedef struct {
class served directly by the arena.</para></listitem> class served directly by the arena.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.hchunks.j.ndalloc"> <varlistentry id="stats.arenas.i.lextents.j.ndalloc">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.hchunks.&lt;j&gt;.ndalloc</mallctl> <mallctl>stats.arenas.&lt;i&gt;.lextents.&lt;j&gt;.ndalloc</mallctl>
(<type>uint64_t</type>) (<type>uint64_t</type>)
<literal>r-</literal> <literal>r-</literal>
[<option>--enable-stats</option>] [<option>--enable-stats</option>]
@ -2522,9 +2478,9 @@ typedef struct {
size class served directly by the arena.</para></listitem> size class served directly by the arena.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.hchunks.j.nrequests"> <varlistentry id="stats.arenas.i.lextents.j.nrequests">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.hchunks.&lt;j&gt;.nrequests</mallctl> <mallctl>stats.arenas.&lt;i&gt;.lextents.&lt;j&gt;.nrequests</mallctl>
(<type>uint64_t</type>) (<type>uint64_t</type>)
<literal>r-</literal> <literal>r-</literal>
[<option>--enable-stats</option>] [<option>--enable-stats</option>]
@ -2533,14 +2489,14 @@ typedef struct {
class.</para></listitem> class.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.hchunks.j.curhchunks"> <varlistentry id="stats.arenas.i.lextents.j.curlextents">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.hchunks.&lt;j&gt;.curhchunks</mallctl> <mallctl>stats.arenas.&lt;i&gt;.lextents.&lt;j&gt;.curlextents</mallctl>
(<type>size_t</type>) (<type>size_t</type>)
<literal>r-</literal> <literal>r-</literal>
[<option>--enable-stats</option>] [<option>--enable-stats</option>]
</term> </term>
<listitem><para>Current number of huge allocations for this size class. <listitem><para>Current number of large allocations for this size class.
</para></listitem> </para></listitem>
</varlistentry> </varlistentry>
</variablelist> </variablelist>

36
include/jemalloc/internal/arena.h
View File

@ -229,10 +229,10 @@ struct arena_s {
*/ */
size_t decay_backlog[SMOOTHSTEP_NSTEPS]; size_t decay_backlog[SMOOTHSTEP_NSTEPS];
/* Extant huge allocations. */ /* Extant large allocations. */
ql_head(extent_t) huge; ql_head(extent_t) large;
/* Synchronizes all huge allocation/update/deallocation. */ /* Synchronizes all large allocation/update/deallocation. */
malloc_mutex_t huge_mtx; malloc_mutex_t large_mtx;
/* /*
* Heaps of chunks that were previously allocated. These are used when * Heaps of chunks that were previously allocated. These are used when
@ -287,13 +287,13 @@ void arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent,
bool cache); bool cache);
void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent, void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent,
bool cache); bool cache);
extent_t *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena,
size_t usize, size_t alignment, bool *zero); size_t usize, size_t alignment, bool *zero);
void arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool locked); bool locked);
void arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void arena_chunk_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize); extent_t *extent, size_t oldsize);
void arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void arena_chunk_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize); extent_t *extent, size_t oldsize);
ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena); ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena);
bool arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena, bool arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena,
@ -341,7 +341,7 @@ void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena,
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time,
size_t *nactive, size_t *ndirty, arena_stats_t *astats, size_t *nactive, size_t *ndirty, arena_stats_t *astats,
malloc_bin_stats_t *bstats, malloc_huge_stats_t *hstats); malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats);
unsigned arena_nthreads_get(arena_t *arena, bool internal); unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal);
@ -470,7 +470,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
assert(ptr != NULL); assert(ptr != NULL);
if (unlikely(!extent_slab_get(extent))) if (unlikely(!extent_slab_get(extent)))
return (huge_prof_tctx_get(tsdn, extent)); return (large_prof_tctx_get(tsdn, extent));
return ((prof_tctx_t *)(uintptr_t)1U); return ((prof_tctx_t *)(uintptr_t)1U);
} }
@ -483,7 +483,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
assert(ptr != NULL); assert(ptr != NULL);
if (unlikely(!extent_slab_get(extent))) if (unlikely(!extent_slab_get(extent)))
huge_prof_tctx_set(tsdn, extent, tctx); large_prof_tctx_set(tsdn, extent, tctx);
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
@ -495,7 +495,7 @@ arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
assert(ptr != NULL); assert(ptr != NULL);
assert(!extent_slab_get(extent)); assert(!extent_slab_get(extent));
huge_prof_tctx_reset(tsdn, extent); large_prof_tctx_reset(tsdn, extent);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
@ -535,7 +535,7 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
tcache, size, ind, zero, slow_path)); tcache, size, ind, zero, slow_path));
} }
if (likely(size <= tcache_maxclass)) { if (likely(size <= tcache_maxclass)) {
return (tcache_alloc_huge(tsdn_tsd(tsdn), arena, return (tcache_alloc_large(tsdn_tsd(tsdn), arena,
tcache, size, ind, zero, slow_path)); tcache, size, ind, zero, slow_path));
} }
/* (size > tcache_maxclass) case falls through. */ /* (size > tcache_maxclass) case falls through. */
@ -563,7 +563,7 @@ arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
if (likely(extent_slab_get(extent))) if (likely(extent_slab_get(extent)))
ret = index2size(extent_slab_data_get_const(extent)->binind); ret = index2size(extent_slab_data_get_const(extent)->binind);
else else
ret = huge_salloc(tsdn, extent); ret = large_salloc(tsdn, extent);
return (ret); return (ret);
} }
@ -594,11 +594,11 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
arena_dalloc_promoted(tsdn, extent, ptr, arena_dalloc_promoted(tsdn, extent, ptr,
tcache, slow_path); tcache, slow_path);
} else { } else {
tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, tcache_dalloc_large(tsdn_tsd(tsdn), tcache,
ptr, usize, slow_path); ptr, usize, slow_path);
} }
} else } else
huge_dalloc(tsdn, extent); large_dalloc(tsdn, extent);
} }
} }
@ -627,11 +627,11 @@ arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
arena_dalloc_promoted(tsdn, extent, ptr, arena_dalloc_promoted(tsdn, extent, ptr,
tcache, slow_path); tcache, slow_path);
} else { } else {
tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, ptr, tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
size, slow_path); size, slow_path);
} }
} else } else
huge_dalloc(tsdn, extent); large_dalloc(tsdn, extent);
} }
} }
# endif /* JEMALLOC_ARENA_INLINE_B */ # endif /* JEMALLOC_ARENA_INLINE_B */

2
include/jemalloc/internal/ctl.h
View File

@ -51,7 +51,7 @@ struct ctl_arena_stats_s {
uint64_t nrequests_small; uint64_t nrequests_small;
malloc_bin_stats_t bstats[NBINS]; malloc_bin_stats_t bstats[NBINS];
malloc_huge_stats_t hstats[NSIZES - NBINS]; malloc_large_stats_t lstats[NSIZES - NBINS];
}; };
struct ctl_stats_s { struct ctl_stats_s {

4
include/jemalloc/internal/extent.h
View File

@ -51,7 +51,7 @@ struct extent_s {
/* Small region slab metadata. */ /* Small region slab metadata. */
arena_slab_data_t e_slab_data; arena_slab_data_t e_slab_data;
/* Profile counters, used for huge objects. */ /* Profile counters, used for large objects. */
union { union {
void *e_prof_tctx_pun; void *e_prof_tctx_pun;
prof_tctx_t *e_prof_tctx; prof_tctx_t *e_prof_tctx;
@ -67,7 +67,7 @@ struct extent_s {
/* Linkage for per size class address-ordered heaps. */ /* Linkage for per size class address-ordered heaps. */
phn(extent_t) ph_link; phn(extent_t) ph_link;
/* Linkage for arena's huge and extent_cache lists. */ /* Linkage for arena's large and extent_cache lists. */
ql_elm(extent_t) ql_link; ql_elm(extent_t) ql_link;
}; };
}; };

37
include/jemalloc/internal/huge.h
View File

@ -1,37 +0,0 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
void *huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero);
bool huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero);
void *huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
#ifdef JEMALLOC_JET
typedef void (huge_dalloc_junk_t)(void *, size_t);
extern huge_dalloc_junk_t *huge_dalloc_junk;
#else
void huge_dalloc_junk(void *ptr, size_t usize);
#endif
void huge_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent);
void huge_dalloc(tsdn_t *tsdn, extent_t *extent);
size_t huge_salloc(tsdn_t *tsdn, const extent_t *extent);
prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent);
void huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx);
void huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/

26
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -362,7 +362,7 @@ typedef unsigned szind_t;
#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/pages.h" #include "jemalloc/internal/pages.h"
#include "jemalloc/internal/chunk.h" #include "jemalloc/internal/chunk.h"
#include "jemalloc/internal/huge.h" #include "jemalloc/internal/large.h"
#include "jemalloc/internal/tcache.h" #include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h" #include "jemalloc/internal/hash.h"
#include "jemalloc/internal/prof.h" #include "jemalloc/internal/prof.h"
@ -396,7 +396,7 @@ typedef unsigned szind_t;
#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/pages.h" #include "jemalloc/internal/pages.h"
#include "jemalloc/internal/chunk.h" #include "jemalloc/internal/chunk.h"
#include "jemalloc/internal/huge.h" #include "jemalloc/internal/large.h"
#include "jemalloc/internal/tcache.h" #include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h" #include "jemalloc/internal/hash.h"
#include "jemalloc/internal/prof.h" #include "jemalloc/internal/prof.h"
@ -486,7 +486,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/pages.h" #include "jemalloc/internal/pages.h"
#include "jemalloc/internal/chunk.h" #include "jemalloc/internal/chunk.h"
#include "jemalloc/internal/huge.h" #include "jemalloc/internal/large.h"
#include "jemalloc/internal/tcache.h" #include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h" #include "jemalloc/internal/hash.h"
#include "jemalloc/internal/prof.h" #include "jemalloc/internal/prof.h"
@ -515,7 +515,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/pages.h" #include "jemalloc/internal/pages.h"
#include "jemalloc/internal/chunk.h" #include "jemalloc/internal/chunk.h"
#include "jemalloc/internal/huge.h" #include "jemalloc/internal/large.h"
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
pszind_t psz2ind(size_t psz); pszind_t psz2ind(size_t psz);
@ -547,7 +547,7 @@ JEMALLOC_INLINE pszind_t
psz2ind(size_t psz) psz2ind(size_t psz)
{ {
if (unlikely(psz > HUGE_MAXCLASS)) if (unlikely(psz > LARGE_MAXCLASS))
return (NPSIZES); return (NPSIZES);
{ {
pszind_t x = lg_floor((psz<<1)-1); pszind_t x = lg_floor((psz<<1)-1);
@ -608,7 +608,7 @@ JEMALLOC_INLINE size_t
psz2u(size_t psz) psz2u(size_t psz)
{ {
if (unlikely(psz > HUGE_MAXCLASS)) if (unlikely(psz > LARGE_MAXCLASS))
return (0); return (0);
{ {
size_t x = lg_floor((psz<<1)-1); size_t x = lg_floor((psz<<1)-1);
@ -625,7 +625,7 @@ JEMALLOC_INLINE szind_t
size2index_compute(size_t size) size2index_compute(size_t size)
{ {
if (unlikely(size > HUGE_MAXCLASS)) if (unlikely(size > LARGE_MAXCLASS))
return (NSIZES); return (NSIZES);
#if (NTBINS != 0) #if (NTBINS != 0)
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
@ -721,7 +721,7 @@ JEMALLOC_ALWAYS_INLINE size_t
s2u_compute(size_t size) s2u_compute(size_t size)
{ {
if (unlikely(size > HUGE_MAXCLASS)) if (unlikely(size > LARGE_MAXCLASS))
return (0); return (0);
#if (NTBINS > 0) #if (NTBINS > 0)
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
@ -797,9 +797,9 @@ sa2u(size_t size, size_t alignment)
return (usize); return (usize);
} }
/* Huge size class. Beware of overflow. */ /* Large size class. Beware of overflow. */
if (unlikely(alignment > HUGE_MAXCLASS)) if (unlikely(alignment > LARGE_MAXCLASS))
return (0); return (0);
/* Make sure result is a large size class. */ /* Make sure result is a large size class. */
@ -814,7 +814,7 @@ sa2u(size_t size, size_t alignment)
} }
/* /*
* Calculate the multi-page mapping that huge_palloc() would need in * Calculate the multi-page mapping that large_palloc() would need in
* order to guarantee the alignment. * order to guarantee the alignment.
*/ */
if (usize + large_pad + PAGE_CEILING(alignment) < usize) { if (usize + large_pad + PAGE_CEILING(alignment) < usize) {
@ -1113,7 +1113,7 @@ iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t usize, copysize; size_t usize, copysize;
usize = sa2u(size + extra, alignment); usize = sa2u(size + extra, alignment);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
return (NULL); return (NULL);
p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); p = ipalloct(tsdn, usize, alignment, zero, tcache, arena);
if (p == NULL) { if (p == NULL) {
@ -1121,7 +1121,7 @@ iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
return (NULL); return (NULL);
/* Try again, without extra this time. */ /* Try again, without extra this time. */
usize = sa2u(size, alignment); usize = sa2u(size, alignment);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
return (NULL); return (NULL);
p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); p = ipalloct(tsdn, usize, alignment, zero, tcache, arena);
if (p == NULL) if (p == NULL)

37
include/jemalloc/internal/large.h Normal file
View File

@ -0,0 +1,37 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero);
bool large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero);
void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
#ifdef JEMALLOC_JET
typedef void (large_dalloc_junk_t)(void *, size_t);
extern large_dalloc_junk_t *large_dalloc_junk;
#else
void large_dalloc_junk(void *ptr, size_t usize);
#endif
void large_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent);
void large_dalloc(tsdn_t *tsdn, extent_t *extent);
size_t large_salloc(tsdn_t *tsdn, const extent_t *extent);
prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent);
void large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx);
void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/

36
include/jemalloc/internal/private_symbols.txt
View File

@ -9,14 +9,14 @@ arena_boot
arena_choose arena_choose
arena_choose_hard arena_choose_hard
arena_choose_impl arena_choose_impl
arena_chunk_alloc_huge arena_chunk_alloc_large
arena_chunk_cache_alloc arena_chunk_cache_alloc
arena_chunk_cache_dalloc arena_chunk_cache_dalloc
arena_chunk_cache_maybe_insert arena_chunk_cache_maybe_insert
arena_chunk_cache_maybe_remove arena_chunk_cache_maybe_remove
arena_chunk_dalloc_huge arena_chunk_dalloc_large
arena_chunk_ralloc_huge_expand arena_chunk_ralloc_large_expand
arena_chunk_ralloc_huge_shrink arena_chunk_ralloc_large_shrink
arena_cleanup arena_cleanup
arena_dalloc arena_dalloc
arena_dalloc_bin_junked_locked arena_dalloc_bin_junked_locked
@ -222,17 +222,6 @@ hash_rotl_64
hash_x64_128 hash_x64_128
hash_x86_128 hash_x86_128
hash_x86_32 hash_x86_32
huge_dalloc
huge_dalloc_junk
huge_dalloc_junked_locked
huge_malloc
huge_palloc
huge_prof_tctx_get
huge_prof_tctx_reset
huge_prof_tctx_set
huge_ralloc
huge_ralloc_no_move
huge_salloc
iaalloc iaalloc
ialloc ialloc
iallocztm iallocztm
@ -258,6 +247,17 @@ ixalloc
jemalloc_postfork_child jemalloc_postfork_child
jemalloc_postfork_parent jemalloc_postfork_parent
jemalloc_prefork jemalloc_prefork
large_dalloc
large_dalloc_junk
large_dalloc_junked_locked
large_malloc
large_palloc
large_prof_tctx_get
large_prof_tctx_reset
large_prof_tctx_set
large_ralloc
large_ralloc_no_move
large_salloc
lg_floor lg_floor
lg_prof_sample lg_prof_sample
malloc_cprintf malloc_cprintf
@ -438,17 +438,17 @@ stats_cactive_get
stats_cactive_sub stats_cactive_sub
stats_print stats_print
tcache_alloc_easy tcache_alloc_easy
tcache_alloc_huge tcache_alloc_large
tcache_alloc_small tcache_alloc_small
tcache_alloc_small_hard tcache_alloc_small_hard
tcache_arena_reassociate tcache_arena_reassociate
tcache_bin_flush_huge tcache_bin_flush_large
tcache_bin_flush_small tcache_bin_flush_small
tcache_bin_info tcache_bin_info
tcache_boot tcache_boot
tcache_cleanup tcache_cleanup
tcache_create tcache_create
tcache_dalloc_huge tcache_dalloc_large
tcache_dalloc_small tcache_dalloc_small
tcache_enabled_cleanup tcache_enabled_cleanup
tcache_enabled_get tcache_enabled_get

8
include/jemalloc/internal/size_classes.sh
View File

@ -237,7 +237,7 @@ size_classes() {
fi fi
fi fi
# Final written value is correct: # Final written value is correct:
huge_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" large_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))"
index=$((${index} + 1)) index=$((${index} + 1))
ndelta=$((${ndelta} + 1)) ndelta=$((${ndelta} + 1))
done done
@ -257,7 +257,7 @@ size_classes() {
# - lookup_maxclass # - lookup_maxclass
# - small_maxclass # - small_maxclass
# - lg_large_minclass # - lg_large_minclass
# - huge_maxclass # - large_maxclass
} }
cat <<EOF cat <<EOF
@ -290,7 +290,7 @@ cat <<EOF
* LOOKUP_MAXCLASS: Maximum size class included in lookup table. * LOOKUP_MAXCLASS: Maximum size class included in lookup table.
* SMALL_MAXCLASS: Maximum small size class. * SMALL_MAXCLASS: Maximum small size class.
* LG_LARGE_MINCLASS: Lg of minimum large size class. * LG_LARGE_MINCLASS: Lg of minimum large size class.
* HUGE_MAXCLASS: Maximum (huge) size class. * LARGE_MAXCLASS: Maximum (large) size class.
*/ */
#define LG_SIZE_CLASS_GROUP ${lg_g} #define LG_SIZE_CLASS_GROUP ${lg_g}
@ -315,7 +315,7 @@ for lg_z in ${lg_zarr} ; do
echo "#define LOOKUP_MAXCLASS ${lookup_maxclass}" echo "#define LOOKUP_MAXCLASS ${lookup_maxclass}"
echo "#define SMALL_MAXCLASS ${small_maxclass}" echo "#define SMALL_MAXCLASS ${small_maxclass}"
echo "#define LG_LARGE_MINCLASS ${lg_large_minclass}" echo "#define LG_LARGE_MINCLASS ${lg_large_minclass}"
echo "#define HUGE_MAXCLASS ${huge_maxclass}" echo "#define LARGE_MAXCLASS ${large_maxclass}"
echo "#endif" echo "#endif"
echo echo
done done

18
include/jemalloc/internal/stats.h
View File

@ -3,7 +3,7 @@
typedef struct tcache_bin_stats_s tcache_bin_stats_t; typedef struct tcache_bin_stats_s tcache_bin_stats_t;
typedef struct malloc_bin_stats_s malloc_bin_stats_t; typedef struct malloc_bin_stats_s malloc_bin_stats_t;
typedef struct malloc_huge_stats_s malloc_huge_stats_t; typedef struct malloc_large_stats_s malloc_large_stats_t;
typedef struct arena_stats_s arena_stats_t; typedef struct arena_stats_s arena_stats_t;
typedef struct chunk_stats_s chunk_stats_t; typedef struct chunk_stats_s chunk_stats_t;
@ -61,7 +61,7 @@ struct malloc_bin_stats_s {
size_t curslabs; size_t curslabs;
}; };
struct malloc_huge_stats_s { struct malloc_large_stats_s {
/* /*
* Total number of allocation/deallocation requests served directly by * Total number of allocation/deallocation requests served directly by
* the arena. * the arena.
@ -77,7 +77,7 @@ struct malloc_huge_stats_s {
uint64_t nrequests; uint64_t nrequests;
/* Current number of (multi-)chunk allocations of this size class. */ /* Current number of (multi-)chunk allocations of this size class. */
size_t curhchunks; size_t curlextents;
}; };
struct arena_stats_s { struct arena_stats_s {
@ -108,13 +108,13 @@ struct arena_stats_s {
size_t metadata_mapped; size_t metadata_mapped;
size_t metadata_allocated; /* Protected via atomic_*_z(). */ size_t metadata_allocated; /* Protected via atomic_*_z(). */
size_t allocated_huge; size_t allocated_large;
uint64_t nmalloc_huge; uint64_t nmalloc_large;
uint64_t ndalloc_huge; uint64_t ndalloc_large;
uint64_t nrequests_huge; uint64_t nrequests_large;
/* One element for each huge size class. */ /* One element for each large size class. */
malloc_huge_stats_t hstats[NSIZES - NBINS]; malloc_large_stats_t lstats[NSIZES - NBINS];
}; };
#endif /* JEMALLOC_H_STRUCTS */ #endif /* JEMALLOC_H_STRUCTS */

24
include/jemalloc/internal/tcache.h
View File

@ -30,8 +30,8 @@ typedef struct tcaches_s tcaches_t;
*/ */
#define TCACHE_NSLOTS_SMALL_MAX 200 #define TCACHE_NSLOTS_SMALL_MAX 200
/* Number of cache slots for huge size classes. */ /* Number of cache slots for large size classes. */
#define TCACHE_NSLOTS_HUGE 20 #define TCACHE_NSLOTS_LARGE 20
/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */ /* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
#define LG_TCACHE_MAXCLASS_DEFAULT 15 #define LG_TCACHE_MAXCLASS_DEFAULT 15
@ -113,7 +113,7 @@ extern tcache_bin_info_t *tcache_bin_info;
/* /*
* Number of tcache bins. There are NBINS small-object bins, plus 0 or more * Number of tcache bins. There are NBINS small-object bins, plus 0 or more
* huge-object bins. * large-object bins.
*/ */
extern unsigned nhbins; extern unsigned nhbins;
@ -136,7 +136,7 @@ void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
tcache_bin_t *tbin, szind_t binind, bool *tcache_success); tcache_bin_t *tbin, szind_t binind, bool *tcache_success);
void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
szind_t binind, unsigned rem); szind_t binind, unsigned rem);
void tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
unsigned rem, tcache_t *tcache); unsigned rem, tcache_t *tcache);
void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
arena_t *oldarena, arena_t *newarena); arena_t *oldarena, arena_t *newarena);
@ -163,11 +163,11 @@ void tcache_enabled_set(bool enabled);
void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success); void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
size_t size, szind_t ind, bool zero, bool slow_path); size_t size, szind_t ind, bool zero, bool slow_path);
void *tcache_alloc_huge(tsd_t *tsd, arena_t *arena, tcache_t *tcache, void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
size_t size, szind_t ind, bool zero, bool slow_path); size_t size, szind_t ind, bool zero, bool slow_path);
void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
szind_t binind, bool slow_path); szind_t binind, bool slow_path);
void tcache_dalloc_huge(tsd_t *tsd, tcache_t *tcache, void *ptr, void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
size_t size, bool slow_path); size_t size, bool slow_path);
tcache_t *tcaches_get(tsd_t *tsd, unsigned ind); tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
#endif #endif
@ -336,7 +336,7 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_huge(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
szind_t binind, bool zero, bool slow_path) szind_t binind, bool zero, bool slow_path)
{ {
void *ret; void *ret;
@ -349,14 +349,14 @@ tcache_alloc_huge(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
assert(tcache_success == (ret != NULL)); assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) { if (unlikely(!tcache_success)) {
/* /*
* Only allocate one huge object at a time, because it's quite * Only allocate one large object at a time, because it's quite
* expensive to create one and not use it. * expensive to create one and not use it.
*/ */
arena = arena_choose(tsd, arena); arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL)) if (unlikely(arena == NULL))
return (NULL); return (NULL);
ret = huge_malloc(tsd_tsdn(tsd), arena, s2u(size), zero); ret = large_malloc(tsd_tsdn(tsd), arena, s2u(size), zero);
if (ret == NULL) if (ret == NULL)
return (NULL); return (NULL);
} else { } else {
@ -416,7 +416,7 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_huge(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size, tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
bool slow_path) bool slow_path)
{ {
szind_t binind; szind_t binind;
@ -429,12 +429,12 @@ tcache_dalloc_huge(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
binind = size2index(size); binind = size2index(size);
if (slow_path && config_fill && unlikely(opt_junk_free)) if (slow_path && config_fill && unlikely(opt_junk_free))
huge_dalloc_junk(ptr, size); large_dalloc_junk(ptr, size);
tbin = &tcache->tbins[binind]; tbin = &tcache->tbins[binind];
tbin_info = &tcache_bin_info[binind]; tbin_info = &tcache_bin_info[binind];
if (unlikely(tbin->ncached == tbin_info->ncached_max)) { if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
tcache_bin_flush_huge(tsd, tbin, binind, tcache_bin_flush_large(tsd, tbin, binind,
(tbin_info->ncached_max >> 1), tcache); (tbin_info->ncached_max >> 1), tcache);
} }
assert(tbin->ncached < tbin_info->ncached_max); assert(tbin->ncached < tbin_info->ncached_max);

2
include/jemalloc/internal/witness.h
View File

@ -32,7 +32,7 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
#define WITNESS_RANK_LEAF 0xffffffffU #define WITNESS_RANK_LEAF 0xffffffffU
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF #define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF
#define WITNESS_RANK_ARENA_HUGE WITNESS_RANK_LEAF #define WITNESS_RANK_ARENA_LARGE WITNESS_RANK_LEAF
#define WITNESS_RANK_DSS WITNESS_RANK_LEAF #define WITNESS_RANK_DSS WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF #define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF #define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF

4
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
View File

@ -47,11 +47,11 @@
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ctl.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\ctl.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\extent.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\huge.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_decls.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_decls.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_defs.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_defs.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\large.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mutex.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\mutex.h" />
<ClInclude Include="..\..\..\..\include\jemalloc\internal\nstime.h" /> <ClInclude Include="..\..\..\..\include\jemalloc\internal\nstime.h" />
@ -98,8 +98,8 @@
<ClCompile Include="..\..\..\..\src\ctl.c" /> <ClCompile Include="..\..\..\..\src\ctl.c" />
<ClCompile Include="..\..\..\..\src\extent.c" /> <ClCompile Include="..\..\..\..\src\extent.c" />
<ClCompile Include="..\..\..\..\src\hash.c" /> <ClCompile Include="..\..\..\..\src\hash.c" />
<ClCompile Include="..\..\..\..\src\huge.c" />
<ClCompile Include="..\..\..\..\src\jemalloc.c" /> <ClCompile Include="..\..\..\..\src\jemalloc.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\mb.c" /> <ClCompile Include="..\..\..\..\src\mb.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" /> <ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" /> <ClCompile Include="..\..\..\..\src\nstime.c" />

10
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
View File

@ -80,9 +80,6 @@
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h"> <ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h">
<Filter>Header Files\internal</Filter> <Filter>Header Files\internal</Filter>
</ClInclude> </ClInclude>
<ClInclude Include="..\..\..\..\include\jemalloc\internal\huge.h">
<Filter>Header Files\internal</Filter>
</ClInclude>
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h"> <ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h">
<Filter>Header Files\internal</Filter> <Filter>Header Files\internal</Filter>
</ClInclude> </ClInclude>
@ -95,6 +92,9 @@
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h"> <ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_macros.h">
<Filter>Header Files\internal</Filter> <Filter>Header Files\internal</Filter>
</ClInclude> </ClInclude>
<ClInclude Include="..\..\..\..\include\jemalloc\internal\large.h">
<Filter>Header Files\internal</Filter>
</ClInclude>
<ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h"> <ClInclude Include="..\..\..\..\include\jemalloc\internal\mb.h">
<Filter>Header Files\internal</Filter> <Filter>Header Files\internal</Filter>
</ClInclude> </ClInclude>
@ -205,10 +205,10 @@
<ClCompile Include="..\..\..\..\src\hash.c"> <ClCompile Include="..\..\..\..\src\hash.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\..\..\..\src\huge.c"> <ClCompile Include="..\..\..\..\src\jemalloc.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\..\..\..\src\jemalloc.c"> <ClCompile Include="..\..\..\..\src\large.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\..\..\..\src\mb.c"> <ClCompile Include="..\..\..\..\src\mb.c">

143
src/arena.c
View File

@ -256,71 +256,71 @@ arena_nactive_sub(arena_t *arena, size_t sub_pages)
} }
static void static void
arena_huge_malloc_stats_update(arena_t *arena, size_t usize) arena_large_malloc_stats_update(arena_t *arena, size_t usize)
{ {
szind_t index = size2index(usize); szind_t index = size2index(usize);
szind_t hindex = (index >= NBINS) ? index - NBINS : 0; szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
cassert(config_stats); cassert(config_stats);
arena->stats.nmalloc_huge++; arena->stats.nmalloc_large++;
arena->stats.allocated_huge += usize; arena->stats.allocated_large += usize;
arena->stats.hstats[hindex].nmalloc++; arena->stats.lstats[hindex].nmalloc++;
arena->stats.hstats[hindex].nrequests++; arena->stats.lstats[hindex].nrequests++;
arena->stats.hstats[hindex].curhchunks++; arena->stats.lstats[hindex].curlextents++;
} }
static void static void
arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) arena_large_malloc_stats_update_undo(arena_t *arena, size_t usize)
{ {
szind_t index = size2index(usize); szind_t index = size2index(usize);
szind_t hindex = (index >= NBINS) ? index - NBINS : 0; szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
cassert(config_stats); cassert(config_stats);
arena->stats.nmalloc_huge--; arena->stats.nmalloc_large--;
arena->stats.allocated_huge -= usize; arena->stats.allocated_large -= usize;
arena->stats.hstats[hindex].nmalloc--; arena->stats.lstats[hindex].nmalloc--;
arena->stats.hstats[hindex].nrequests--; arena->stats.lstats[hindex].nrequests--;
arena->stats.hstats[hindex].curhchunks--; arena->stats.lstats[hindex].curlextents--;
} }
static void static void
arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) arena_large_dalloc_stats_update(arena_t *arena, size_t usize)
{ {
szind_t index = size2index(usize); szind_t index = size2index(usize);
szind_t hindex = (index >= NBINS) ? index - NBINS : 0; szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
cassert(config_stats); cassert(config_stats);
arena->stats.ndalloc_huge++; arena->stats.ndalloc_large++;
arena->stats.allocated_huge -= usize; arena->stats.allocated_large -= usize;
arena->stats.hstats[hindex].ndalloc++; arena->stats.lstats[hindex].ndalloc++;
arena->stats.hstats[hindex].curhchunks--; arena->stats.lstats[hindex].curlextents--;
} }
static void static void
arena_huge_reset_stats_cancel(arena_t *arena, size_t usize) arena_large_reset_stats_cancel(arena_t *arena, size_t usize)
{ {
szind_t index = size2index(usize); szind_t index = size2index(usize);
szind_t hindex = (index >= NBINS) ? index - NBINS : 0; szind_t hindex = (index >= NBINS) ? index - NBINS : 0;
cassert(config_stats); cassert(config_stats);
arena->stats.ndalloc_huge++; arena->stats.ndalloc_large++;
arena->stats.hstats[hindex].ndalloc--; arena->stats.lstats[hindex].ndalloc--;
} }
static void static void
arena_huge_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize) arena_large_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize)
{ {
arena_huge_dalloc_stats_update(arena, oldusize); arena_large_dalloc_stats_update(arena, oldusize);
arena_huge_malloc_stats_update(arena, usize); arena_large_malloc_stats_update(arena, usize);
} }
static extent_t * static extent_t *
arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena, arena_chunk_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero) chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero)
{ {
extent_t *extent; extent_t *extent;
@ -332,7 +332,7 @@ arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
/* Revert optimistic stats updates. */ /* Revert optimistic stats updates. */
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
if (config_stats) { if (config_stats) {
arena_huge_malloc_stats_update_undo(arena, usize); arena_large_malloc_stats_update_undo(arena, usize);
arena->stats.mapped -= usize; arena->stats.mapped -= usize;
} }
arena_nactive_sub(arena, (usize + large_pad) >> LG_PAGE); arena_nactive_sub(arena, (usize + large_pad) >> LG_PAGE);
@ -343,7 +343,7 @@ arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
} }
extent_t * extent_t *
arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero) size_t alignment, bool *zero)
{ {
extent_t *extent; extent_t *extent;
@ -353,7 +353,7 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
/* Optimistically update stats. */ /* Optimistically update stats. */
if (config_stats) { if (config_stats) {
arena_huge_malloc_stats_update(arena, usize); arena_large_malloc_stats_update(arena, usize);
arena->stats.mapped += usize; arena->stats.mapped += usize;
} }
arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE); arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE);
@ -362,7 +362,7 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
usize, large_pad, alignment, zero, false); usize, large_pad, alignment, zero, false);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
if (extent == NULL) { if (extent == NULL) {
extent = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks, extent = arena_chunk_alloc_large_hard(tsdn, arena, &chunk_hooks,
usize, alignment, zero); usize, alignment, zero);
} }
@ -370,7 +370,7 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
} }
void void
arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent, arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool locked) bool locked)
{ {
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
@ -378,7 +378,8 @@ arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
if (!locked) if (!locked)
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
if (config_stats) { if (config_stats) {
arena_huge_dalloc_stats_update(arena, extent_usize_get(extent)); arena_large_dalloc_stats_update(arena,
extent_usize_get(extent));
arena->stats.mapped -= extent_size_get(extent); arena->stats.mapped -= extent_size_get(extent);
} }
arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE); arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE);
@ -389,7 +390,7 @@ arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
} }
void void
arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent, arena_chunk_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t oldusize) size_t oldusize)
{ {
size_t usize = extent_usize_get(extent); size_t usize = extent_usize_get(extent);
@ -397,7 +398,7 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
if (config_stats) { if (config_stats) {
arena_huge_ralloc_stats_update(arena, oldusize, usize); arena_large_ralloc_stats_update(arena, oldusize, usize);
arena->stats.mapped -= udiff; arena->stats.mapped -= udiff;
} }
arena_nactive_sub(arena, udiff >> LG_PAGE); arena_nactive_sub(arena, udiff >> LG_PAGE);
@ -405,7 +406,7 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
} }
void void
arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent, arena_chunk_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t oldusize) size_t oldusize)
{ {
size_t usize = extent_usize_get(extent); size_t usize = extent_usize_get(extent);
@ -413,7 +414,7 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
if (config_stats) { if (config_stats) {
arena_huge_ralloc_stats_update(arena, oldusize, usize); arena_large_ralloc_stats_update(arena, oldusize, usize);
arena->stats.mapped += udiff; arena->stats.mapped += udiff;
} }
arena_nactive_add(arena, udiff >> LG_PAGE); arena_nactive_add(arena, udiff >> LG_PAGE);
@ -891,26 +892,26 @@ arena_reset(tsd_t *tsd, arena_t *arena)
* stats refreshes would impose an inconvenient burden. * stats refreshes would impose an inconvenient burden.
*/ */
/* Huge allocations. */ /* Large allocations. */
malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
for (extent = ql_last(&arena->huge, ql_link); extent != NULL; extent = for (extent = ql_last(&arena->large, ql_link); extent != NULL; extent =
ql_last(&arena->huge, ql_link)) { ql_last(&arena->large, ql_link)) {
void *ptr = extent_base_get(extent); void *ptr = extent_base_get(extent);
size_t usize; size_t usize;
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
if (config_stats || (config_prof && opt_prof)) if (config_stats || (config_prof && opt_prof))
usize = isalloc(tsd_tsdn(tsd), extent, ptr); usize = isalloc(tsd_tsdn(tsd), extent, ptr);
/* Remove huge allocation from prof sample set. */ /* Remove large allocation from prof sample set. */
if (config_prof && opt_prof) if (config_prof && opt_prof)
prof_free(tsd, extent, ptr, usize); prof_free(tsd, extent, ptr, usize);
huge_dalloc(tsd_tsdn(tsd), extent); large_dalloc(tsd_tsdn(tsd), extent);
malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx);
/* Cancel out unwanted effects on stats. */ /* Cancel out unwanted effects on stats. */
if (config_stats) if (config_stats)
arena_huge_reset_stats_cancel(arena, usize); arena_large_reset_stats_cancel(arena, usize);
} }
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx);
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock); malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
@ -1283,7 +1284,7 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
if (likely(size <= SMALL_MAXCLASS)) if (likely(size <= SMALL_MAXCLASS))
return (arena_malloc_small(tsdn, arena, ind, zero)); return (arena_malloc_small(tsdn, arena, ind, zero));
return (huge_malloc(tsdn, arena, index2size(ind), zero)); return (large_malloc(tsdn, arena, index2size(ind), zero));
} }
void * void *
@ -1299,9 +1300,9 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
tcache, true); tcache, true);
} else { } else {
if (likely(alignment <= CACHELINE)) if (likely(alignment <= CACHELINE))
ret = huge_malloc(tsdn, arena, usize, zero); ret = large_malloc(tsdn, arena, usize, zero);
else else
ret = huge_palloc(tsdn, arena, usize, alignment, zero); ret = large_palloc(tsdn, arena, usize, alignment, zero);
} }
return (ret); return (ret);
} }
@ -1360,10 +1361,10 @@ arena_dalloc_promoted(tsdn_t *tsdn, extent_t *extent, void *ptr,
usize = arena_prof_demote(tsdn, extent, ptr); usize = arena_prof_demote(tsdn, extent, ptr);
if (usize <= tcache_maxclass) { if (usize <= tcache_maxclass) {
tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, ptr, usize, tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, usize,
slow_path); slow_path);
} else } else
huge_dalloc(tsdn, extent); large_dalloc(tsdn, extent);
} }
static void static void
@ -1493,9 +1494,9 @@ arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t usize_min, usize_max; size_t usize_min, usize_max;
/* Calls with non-zero extra had to clamp extra. */ /* Calls with non-zero extra had to clamp extra. */
assert(extra == 0 || size + extra <= HUGE_MAXCLASS); assert(extra == 0 || size + extra <= LARGE_MAXCLASS);
if (unlikely(size > HUGE_MAXCLASS)) if (unlikely(size > LARGE_MAXCLASS))
return (true); return (true);
usize_min = s2u(size); usize_min = s2u(size);
@ -1515,7 +1516,7 @@ arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
arena_decay_tick(tsdn, extent_arena_get(extent)); arena_decay_tick(tsdn, extent_arena_get(extent));
return (false); return (false);
} else if (oldsize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS) { } else if (oldsize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS) {
return (huge_ralloc_no_move(tsdn, extent, usize_min, usize_max, return (large_ralloc_no_move(tsdn, extent, usize_min, usize_max,
zero)); zero));
} }
@ -1531,7 +1532,7 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
return (arena_malloc(tsdn, arena, usize, size2index(usize), return (arena_malloc(tsdn, arena, usize, size2index(usize),
zero, tcache, true)); zero, tcache, true));
usize = sa2u(usize, alignment); usize = sa2u(usize, alignment);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
return (NULL); return (NULL);
return (ipalloct(tsdn, usize, alignment, zero, tcache, arena)); return (ipalloct(tsdn, usize, alignment, zero, tcache, arena));
} }
@ -1544,7 +1545,7 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t usize, copysize; size_t usize, copysize;
usize = s2u(size); usize = s2u(size);
if (unlikely(usize == 0 || size > HUGE_MAXCLASS)) if (unlikely(usize == 0 || size > LARGE_MAXCLASS))
return (NULL); return (NULL);
if (likely(usize <= SMALL_MAXCLASS)) { if (likely(usize <= SMALL_MAXCLASS)) {
@ -1555,8 +1556,8 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
} }
if (oldsize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS) { if (oldsize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS) {
return (huge_ralloc(tsdn, arena, extent, usize, alignment, zero, return (large_ralloc(tsdn, arena, extent, usize, alignment,
tcache)); zero, tcache));
} }
/* /*
@ -1670,7 +1671,7 @@ void
arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time,
size_t *nactive, size_t *ndirty, arena_stats_t *astats, size_t *nactive, size_t *ndirty, arena_stats_t *astats,
malloc_bin_stats_t *bstats, malloc_huge_stats_t *hstats) malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats)
{ {
unsigned i; unsigned i;
@ -1687,16 +1688,16 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
astats->purged += arena->stats.purged; astats->purged += arena->stats.purged;
astats->metadata_mapped += arena->stats.metadata_mapped; astats->metadata_mapped += arena->stats.metadata_mapped;
astats->metadata_allocated += arena_metadata_allocated_get(arena); astats->metadata_allocated += arena_metadata_allocated_get(arena);
astats->allocated_huge += arena->stats.allocated_huge; astats->allocated_large += arena->stats.allocated_large;
astats->nmalloc_huge += arena->stats.nmalloc_huge; astats->nmalloc_large += arena->stats.nmalloc_large;
astats->ndalloc_huge += arena->stats.ndalloc_huge; astats->ndalloc_large += arena->stats.ndalloc_large;
astats->nrequests_huge += arena->stats.nrequests_huge; astats->nrequests_large += arena->stats.nrequests_large;
for (i = 0; i < NSIZES - NBINS; i++) { for (i = 0; i < NSIZES - NBINS; i++) {
hstats[i].nmalloc += arena->stats.hstats[i].nmalloc; lstats[i].nmalloc += arena->stats.lstats[i].nmalloc;
hstats[i].ndalloc += arena->stats.hstats[i].ndalloc; lstats[i].ndalloc += arena->stats.lstats[i].ndalloc;
hstats[i].nrequests += arena->stats.hstats[i].nrequests; lstats[i].nrequests += arena->stats.lstats[i].nrequests;
hstats[i].curhchunks += arena->stats.hstats[i].curhchunks; lstats[i].curlextents += arena->stats.lstats[i].curlextents;
} }
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
@ -1786,9 +1787,9 @@ arena_new(tsdn_t *tsdn, unsigned ind)
if (opt_purge == purge_mode_decay) if (opt_purge == purge_mode_decay)
arena_decay_init(arena, arena_decay_time_default_get()); arena_decay_init(arena, arena_decay_time_default_get());
ql_new(&arena->huge); ql_new(&arena->large);
if (malloc_mutex_init(&arena->huge_mtx, "arena_huge", if (malloc_mutex_init(&arena->large_mtx, "arena_large",
WITNESS_RANK_ARENA_HUGE)) WITNESS_RANK_ARENA_LARGE))
return (NULL); return (NULL);
for (i = 0; i < NPSIZES; i++) { for (i = 0; i < NPSIZES; i++) {
@ -1859,7 +1860,7 @@ arena_prefork3(tsdn_t *tsdn, arena_t *arena)
for (i = 0; i < NBINS; i++) for (i = 0; i < NBINS; i++)
malloc_mutex_prefork(tsdn, &arena->bins[i].lock); malloc_mutex_prefork(tsdn, &arena->bins[i].lock);
malloc_mutex_prefork(tsdn, &arena->huge_mtx); malloc_mutex_prefork(tsdn, &arena->large_mtx);
} }
void void
@ -1867,7 +1868,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena)
{ {
unsigned i; unsigned i;
malloc_mutex_postfork_parent(tsdn, &arena->huge_mtx); malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
for (i = 0; i < NBINS; i++) for (i = 0; i < NBINS; i++)
malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock); malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock);
malloc_mutex_postfork_parent(tsdn, &arena->extent_cache_mtx); malloc_mutex_postfork_parent(tsdn, &arena->extent_cache_mtx);
@ -1880,7 +1881,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena)
{ {
unsigned i; unsigned i;
malloc_mutex_postfork_child(tsdn, &arena->huge_mtx); malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
for (i = 0; i < NBINS; i++) for (i = 0; i < NBINS; i++)
malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock); malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock);
malloc_mutex_postfork_child(tsdn, &arena->extent_cache_mtx); malloc_mutex_postfork_child(tsdn, &arena->extent_cache_mtx);

2
src/chunk_dss.c
View File

@ -78,7 +78,7 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
/* /*
* sbrk() uses a signed increment argument, so take care not to * sbrk() uses a signed increment argument, so take care not to
* interpret a huge allocation request as a negative increment. * interpret a large allocation request as a negative increment.
*/ */
if ((intptr_t)size < 0) if ((intptr_t)size < 0)
return (NULL); return (NULL);

6
src/ckh.c
View File

@ -267,7 +267,7 @@ ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
lg_curcells++; lg_curcells++;
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) { if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
ret = true; ret = true;
goto label_return; goto label_return;
} }
@ -315,7 +315,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
lg_prevbuckets = ckh->lg_curbuckets; lg_prevbuckets = ckh->lg_curbuckets;
lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1;
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
return; return;
tab = (ckhc_t *)ipallocztm(tsdn, usize, CACHELINE, true, NULL, true, tab = (ckhc_t *)ipallocztm(tsdn, usize, CACHELINE, true, NULL, true,
arena_ichoose(tsdn, NULL)); arena_ichoose(tsdn, NULL));
@ -390,7 +390,7 @@ ckh_new(tsdn_t *tsdn, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh->keycomp = keycomp; ckh->keycomp = keycomp;
usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) { if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
ret = true; ret = true;
goto label_return; goto label_return;
} }

146
src/ctl.c
View File

@ -126,8 +126,8 @@ CTL_PROTO(arenas_bin_i_size)
CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_nregs)
CTL_PROTO(arenas_bin_i_slab_size) CTL_PROTO(arenas_bin_i_slab_size)
INDEX_PROTO(arenas_bin_i) INDEX_PROTO(arenas_bin_i)
CTL_PROTO(arenas_hchunk_i_size) CTL_PROTO(arenas_lextent_i_size)
INDEX_PROTO(arenas_hchunk_i) INDEX_PROTO(arenas_lextent_i)
CTL_PROTO(arenas_narenas) CTL_PROTO(arenas_narenas)
CTL_PROTO(arenas_initialized) CTL_PROTO(arenas_initialized)
CTL_PROTO(arenas_lg_dirty_mult) CTL_PROTO(arenas_lg_dirty_mult)
@ -137,7 +137,7 @@ CTL_PROTO(arenas_page)
CTL_PROTO(arenas_tcache_max) CTL_PROTO(arenas_tcache_max)
CTL_PROTO(arenas_nbins) CTL_PROTO(arenas_nbins)
CTL_PROTO(arenas_nhbins) CTL_PROTO(arenas_nhbins)
CTL_PROTO(arenas_nhchunks) CTL_PROTO(arenas_nlextents)
CTL_PROTO(arenas_extend) CTL_PROTO(arenas_extend)
CTL_PROTO(prof_thread_active_init) CTL_PROTO(prof_thread_active_init)
CTL_PROTO(prof_active) CTL_PROTO(prof_active)
@ -150,10 +150,10 @@ CTL_PROTO(stats_arenas_i_small_allocated)
CTL_PROTO(stats_arenas_i_small_nmalloc) CTL_PROTO(stats_arenas_i_small_nmalloc)
CTL_PROTO(stats_arenas_i_small_ndalloc) CTL_PROTO(stats_arenas_i_small_ndalloc)
CTL_PROTO(stats_arenas_i_small_nrequests) CTL_PROTO(stats_arenas_i_small_nrequests)
CTL_PROTO(stats_arenas_i_huge_allocated) CTL_PROTO(stats_arenas_i_large_allocated)
CTL_PROTO(stats_arenas_i_huge_nmalloc) CTL_PROTO(stats_arenas_i_large_nmalloc)
CTL_PROTO(stats_arenas_i_huge_ndalloc) CTL_PROTO(stats_arenas_i_large_ndalloc)
CTL_PROTO(stats_arenas_i_huge_nrequests) CTL_PROTO(stats_arenas_i_large_nrequests)
CTL_PROTO(stats_arenas_i_bins_j_nmalloc) CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
CTL_PROTO(stats_arenas_i_bins_j_ndalloc) CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
CTL_PROTO(stats_arenas_i_bins_j_nrequests) CTL_PROTO(stats_arenas_i_bins_j_nrequests)
@ -164,11 +164,11 @@ CTL_PROTO(stats_arenas_i_bins_j_nslabs)
CTL_PROTO(stats_arenas_i_bins_j_nreslabs) CTL_PROTO(stats_arenas_i_bins_j_nreslabs)
CTL_PROTO(stats_arenas_i_bins_j_curslabs) CTL_PROTO(stats_arenas_i_bins_j_curslabs)
INDEX_PROTO(stats_arenas_i_bins_j) INDEX_PROTO(stats_arenas_i_bins_j)
CTL_PROTO(stats_arenas_i_hchunks_j_nmalloc) CTL_PROTO(stats_arenas_i_lextents_j_nmalloc)
CTL_PROTO(stats_arenas_i_hchunks_j_ndalloc) CTL_PROTO(stats_arenas_i_lextents_j_ndalloc)
CTL_PROTO(stats_arenas_i_hchunks_j_nrequests) CTL_PROTO(stats_arenas_i_lextents_j_nrequests)
CTL_PROTO(stats_arenas_i_hchunks_j_curhchunks) CTL_PROTO(stats_arenas_i_lextents_j_curlextents)
INDEX_PROTO(stats_arenas_i_hchunks_j) INDEX_PROTO(stats_arenas_i_lextents_j)
CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_nthreads)
CTL_PROTO(stats_arenas_i_dss) CTL_PROTO(stats_arenas_i_dss)
CTL_PROTO(stats_arenas_i_lg_dirty_mult) CTL_PROTO(stats_arenas_i_lg_dirty_mult)
@ -310,15 +310,15 @@ static const ctl_indexed_node_t arenas_bin_node[] = {
{INDEX(arenas_bin_i)} {INDEX(arenas_bin_i)}
}; };
static const ctl_named_node_t arenas_hchunk_i_node[] = { static const ctl_named_node_t arenas_lextent_i_node[] = {
{NAME("size"), CTL(arenas_hchunk_i_size)} {NAME("size"), CTL(arenas_lextent_i_size)}
}; };
static const ctl_named_node_t super_arenas_hchunk_i_node[] = { static const ctl_named_node_t super_arenas_lextent_i_node[] = {
{NAME(""), CHILD(named, arenas_hchunk_i)} {NAME(""), CHILD(named, arenas_lextent_i)}
}; };
static const ctl_indexed_node_t arenas_hchunk_node[] = { static const ctl_indexed_node_t arenas_lextent_node[] = {
{INDEX(arenas_hchunk_i)} {INDEX(arenas_lextent_i)}
}; };
static const ctl_named_node_t arenas_node[] = { static const ctl_named_node_t arenas_node[] = {
@ -332,8 +332,8 @@ static const ctl_named_node_t arenas_node[] = {
{NAME("nbins"), CTL(arenas_nbins)}, {NAME("nbins"), CTL(arenas_nbins)},
{NAME("nhbins"), CTL(arenas_nhbins)}, {NAME("nhbins"), CTL(arenas_nhbins)},
{NAME("bin"), CHILD(indexed, arenas_bin)}, {NAME("bin"), CHILD(indexed, arenas_bin)},
{NAME("nhchunks"), CTL(arenas_nhchunks)}, {NAME("nlextents"), CTL(arenas_nlextents)},
{NAME("hchunk"), CHILD(indexed, arenas_hchunk)}, {NAME("lextent"), CHILD(indexed, arenas_lextent)},
{NAME("extend"), CTL(arenas_extend)} {NAME("extend"), CTL(arenas_extend)}
}; };
@ -359,11 +359,11 @@ static const ctl_named_node_t stats_arenas_i_small_node[] = {
{NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}
}; };
static const ctl_named_node_t stats_arenas_i_huge_node[] = { static const ctl_named_node_t stats_arenas_i_large_node[] = {
{NAME("allocated"), CTL(stats_arenas_i_huge_allocated)}, {NAME("allocated"), CTL(stats_arenas_i_large_allocated)},
{NAME("nmalloc"), CTL(stats_arenas_i_huge_nmalloc)}, {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_huge_ndalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)},
{NAME("nrequests"), CTL(stats_arenas_i_huge_nrequests)} {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}
}; };
static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
@ -385,18 +385,18 @@ static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {
{INDEX(stats_arenas_i_bins_j)} {INDEX(stats_arenas_i_bins_j)}
}; };
static const ctl_named_node_t stats_arenas_i_hchunks_j_node[] = { static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = {
{NAME("nmalloc"), CTL(stats_arenas_i_hchunks_j_nmalloc)}, {NAME("nmalloc"), CTL(stats_arenas_i_lextents_j_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_hchunks_j_ndalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_lextents_j_ndalloc)},
{NAME("nrequests"), CTL(stats_arenas_i_hchunks_j_nrequests)}, {NAME("nrequests"), CTL(stats_arenas_i_lextents_j_nrequests)},
{NAME("curhchunks"), CTL(stats_arenas_i_hchunks_j_curhchunks)} {NAME("curlextents"), CTL(stats_arenas_i_lextents_j_curlextents)}
}; };
static const ctl_named_node_t super_stats_arenas_i_hchunks_j_node[] = { static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = {
{NAME(""), CHILD(named, stats_arenas_i_hchunks_j)} {NAME(""), CHILD(named, stats_arenas_i_lextents_j)}
}; };
static const ctl_indexed_node_t stats_arenas_i_hchunks_node[] = { static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
{INDEX(stats_arenas_i_hchunks_j)} {INDEX(stats_arenas_i_lextents_j)}
}; };
static const ctl_named_node_t stats_arenas_i_node[] = { static const ctl_named_node_t stats_arenas_i_node[] = {
@ -413,9 +413,9 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
{NAME("purged"), CTL(stats_arenas_i_purged)}, {NAME("purged"), CTL(stats_arenas_i_purged)},
{NAME("metadata"), CHILD(named, stats_arenas_i_metadata)}, {NAME("metadata"), CHILD(named, stats_arenas_i_metadata)},
{NAME("small"), CHILD(named, stats_arenas_i_small)}, {NAME("small"), CHILD(named, stats_arenas_i_small)},
{NAME("huge"), CHILD(named, stats_arenas_i_huge)}, {NAME("large"), CHILD(named, stats_arenas_i_large)},
{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
{NAME("hchunks"), CHILD(indexed, stats_arenas_i_hchunks)} {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}
}; };
static const ctl_named_node_t super_stats_arenas_i_node[] = { static const ctl_named_node_t super_stats_arenas_i_node[] = {
{NAME(""), CHILD(named, stats_arenas_i)} {NAME(""), CHILD(named, stats_arenas_i)}
@ -476,8 +476,8 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
astats->ndalloc_small = 0; astats->ndalloc_small = 0;
astats->nrequests_small = 0; astats->nrequests_small = 0;
memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t)); memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t));
memset(astats->hstats, 0, (NSIZES - NBINS) * memset(astats->lstats, 0, (NSIZES - NBINS) *
sizeof(malloc_huge_stats_t)); sizeof(malloc_large_stats_t));
} }
} }
@ -490,7 +490,7 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_stats_t *cstats, arena_t *arena)
arena_stats_merge(tsdn, arena, &cstats->nthreads, &cstats->dss, arena_stats_merge(tsdn, arena, &cstats->nthreads, &cstats->dss,
&cstats->lg_dirty_mult, &cstats->decay_time, &cstats->lg_dirty_mult, &cstats->decay_time,
&cstats->pactive, &cstats->pdirty, &cstats->astats, &cstats->pactive, &cstats->pdirty, &cstats->astats,
cstats->bstats, cstats->hstats); cstats->bstats, cstats->lstats);
for (i = 0; i < NBINS; i++) { for (i = 0; i < NBINS; i++) {
cstats->allocated_small += cstats->bstats[i].curregs * cstats->allocated_small += cstats->bstats[i].curregs *
@ -532,10 +532,12 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
sstats->ndalloc_small += astats->ndalloc_small; sstats->ndalloc_small += astats->ndalloc_small;
sstats->nrequests_small += astats->nrequests_small; sstats->nrequests_small += astats->nrequests_small;
sstats->astats.allocated_huge += astats->astats.allocated_huge; sstats->astats.allocated_large +=
sstats->astats.nmalloc_huge += astats->astats.nmalloc_huge; astats->astats.allocated_large;
sstats->astats.ndalloc_huge += astats->astats.ndalloc_huge; sstats->astats.nmalloc_large += astats->astats.nmalloc_large;
sstats->astats.nrequests_huge += astats->astats.nrequests_huge; sstats->astats.ndalloc_large += astats->astats.ndalloc_large;
sstats->astats.nrequests_large +=
astats->astats.nrequests_large;
for (i = 0; i < NBINS; i++) { for (i = 0; i < NBINS; i++) {
sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
@ -556,12 +558,12 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
} }
for (i = 0; i < NSIZES - NBINS; i++) { for (i = 0; i < NSIZES - NBINS; i++) {
sstats->hstats[i].nmalloc += astats->hstats[i].nmalloc; sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc;
sstats->hstats[i].ndalloc += astats->hstats[i].ndalloc; sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc;
sstats->hstats[i].nrequests += sstats->lstats[i].nrequests +=
astats->hstats[i].nrequests; astats->lstats[i].nrequests;
sstats->hstats[i].curhchunks += sstats->lstats[i].curlextents +=
astats->hstats[i].curhchunks; astats->lstats[i].curlextents;
} }
} }
} }
@ -643,7 +645,7 @@ ctl_refresh(tsdn_t *tsdn)
&base_mapped); &base_mapped);
ctl_stats.allocated = ctl_stats.allocated =
ctl_stats.arenas[ctl_stats.narenas].allocated_small + ctl_stats.arenas[ctl_stats.narenas].allocated_small +
ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge; ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large;
ctl_stats.active = ctl_stats.active =
(ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE); (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE);
ctl_stats.metadata = base_allocated + ctl_stats.metadata = base_allocated +
@ -1812,15 +1814,15 @@ arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
return (super_arenas_bin_i_node); return (super_arenas_bin_i_node);
} }
CTL_RO_NL_GEN(arenas_nhchunks, NSIZES - NBINS, unsigned) CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned)
CTL_RO_NL_GEN(arenas_hchunk_i_size, index2size(NBINS+(szind_t)mib[2]), size_t) CTL_RO_NL_GEN(arenas_lextent_i_size, index2size(NBINS+(szind_t)mib[2]), size_t)
static const ctl_named_node_t * static const ctl_named_node_t *
arenas_hchunk_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
{ {
if (i > NSIZES - NBINS) if (i > NSIZES - NBINS)
return (NULL); return (NULL);
return (super_arenas_hchunk_i_node); return (super_arenas_lextent_i_node);
} }
static int static int
@ -2012,14 +2014,14 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc,
ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t) ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests,
ctl_stats.arenas[mib[2]].nrequests_small, uint64_t) ctl_stats.arenas[mib[2]].nrequests_small, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_huge_allocated, CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated,
ctl_stats.arenas[mib[2]].astats.allocated_huge, size_t) ctl_stats.arenas[mib[2]].astats.allocated_large, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nmalloc, CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc,
ctl_stats.arenas[mib[2]].astats.nmalloc_huge, uint64_t) ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_huge_ndalloc, CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc,
ctl_stats.arenas[mib[2]].astats.ndalloc_huge, uint64_t) ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nrequests, CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests,
ctl_stats.arenas[mib[2]].astats.nmalloc_huge, uint64_t) /* Intentional. */ ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t) /* Intentional. */
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc,
ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t) ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t)
@ -2050,23 +2052,23 @@ stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
return (super_stats_arenas_i_bins_j_node); return (super_stats_arenas_i_bins_j_node);
} }
CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_nmalloc, CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc,
ctl_stats.arenas[mib[2]].hstats[mib[4]].nmalloc, uint64_t) ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_ndalloc, CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc,
ctl_stats.arenas[mib[2]].hstats[mib[4]].ndalloc, uint64_t) ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_nrequests, CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests,
ctl_stats.arenas[mib[2]].hstats[mib[4]].nrequests, uint64_t) ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_curhchunks, CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents,
ctl_stats.arenas[mib[2]].hstats[mib[4]].curhchunks, size_t) ctl_stats.arenas[mib[2]].lstats[mib[4]].curlextents, size_t)
static const ctl_named_node_t * static const ctl_named_node_t *
stats_arenas_i_hchunks_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen,
size_t j) size_t j)
{ {
if (j > NSIZES - NBINS) if (j > NSIZES - NBINS)
return (NULL); return (NULL);
return (super_stats_arenas_i_hchunks_j_node); return (super_stats_arenas_i_lextents_j_node);
} }
static const ctl_named_node_t * static const ctl_named_node_t *

4
src/extent.c
View File

@ -40,7 +40,7 @@ extent_size_quantize_floor(size_t size)
pszind_t pind; pszind_t pind;
assert(size > 0); assert(size > 0);
assert(size - large_pad <= HUGE_MAXCLASS); assert(size - large_pad <= LARGE_MAXCLASS);
assert((size & PAGE_MASK) == 0); assert((size & PAGE_MASK) == 0);
assert(size != 0); assert(size != 0);
@ -77,7 +77,7 @@ extent_size_quantize_ceil(size_t size)
size_t ret; size_t ret;
assert(size > 0); assert(size > 0);
assert(size - large_pad <= HUGE_MAXCLASS); assert(size - large_pad <= LARGE_MAXCLASS);
assert((size & PAGE_MASK) == 0); assert((size & PAGE_MASK) == 0);
ret = extent_size_quantize_floor(size); ret = extent_size_quantize_floor(size);

32
src/jemalloc.c
View File

@ -1457,7 +1457,7 @@ ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize,
if (config_stats || (config_prof && opt_prof)) { if (config_stats || (config_prof && opt_prof)) {
*usize = index2size(ind); *usize = index2size(ind);
assert(*usize > 0 && *usize <= HUGE_MAXCLASS); assert(*usize > 0 && *usize <= LARGE_MAXCLASS);
} }
if (config_prof && opt_prof) if (config_prof && opt_prof)
@ -1589,7 +1589,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
} }
usize = sa2u(size, alignment); usize = sa2u(size, alignment);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) { if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) {
result = NULL; result = NULL;
goto label_oom; goto label_oom;
} }
@ -1663,7 +1663,7 @@ je_calloc(size_t num, size_t size)
if (num == 0 || size == 0) if (num == 0 || size == 0)
num_size = 1; num_size = 1;
else else
num_size = HUGE_MAXCLASS + 1; /* Trigger OOM. */ num_size = LARGE_MAXCLASS + 1; /* Trigger OOM. */
/* /*
* Try to avoid division here. We know that it isn't possible to * Try to avoid division here. We know that it isn't possible to
* overflow during multiplication if neither operand uses any of the * overflow during multiplication if neither operand uses any of the
@ -1671,7 +1671,7 @@ je_calloc(size_t num, size_t size)
*/ */
} else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) << } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) <<
2))) && (num_size / size != num))) 2))) && (num_size / size != num)))
num_size = HUGE_MAXCLASS + 1; /* size_t overflow. */ num_size = LARGE_MAXCLASS + 1; /* size_t overflow. */
if (likely(!malloc_slow)) { if (likely(!malloc_slow)) {
ret = ialloc_body(num_size, true, &tsdn, &usize, false); ret = ialloc_body(num_size, true, &tsdn, &usize, false);
@ -1819,7 +1819,7 @@ je_realloc(void *ptr, size_t size)
old_usize = isalloc(tsd_tsdn(tsd), extent, ptr); old_usize = isalloc(tsd_tsdn(tsd), extent, ptr);
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = s2u(size); usize = s2u(size);
ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ? ret = unlikely(usize == 0 || usize > LARGE_MAXCLASS) ?
NULL : irealloc_prof(tsd, extent, ptr, old_usize, NULL : irealloc_prof(tsd, extent, ptr, old_usize,
usize); usize);
} else { } else {
@ -1956,7 +1956,7 @@ imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize,
*alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
*usize = sa2u(size, *alignment); *usize = sa2u(size, *alignment);
} }
if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS)) if (unlikely(*usize == 0 || *usize > LARGE_MAXCLASS))
return (true); return (true);
*zero = MALLOCX_ZERO_GET(flags); *zero = MALLOCX_ZERO_GET(flags);
if ((flags & MALLOCX_TCACHE_MASK) != 0) { if ((flags & MALLOCX_TCACHE_MASK) != 0) {
@ -2084,7 +2084,7 @@ imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize,
return (NULL); return (NULL);
if (config_stats || (config_prof && opt_prof)) { if (config_stats || (config_prof && opt_prof)) {
*usize = index2size(ind); *usize = index2size(ind);
assert(*usize > 0 && *usize <= HUGE_MAXCLASS); assert(*usize > 0 && *usize <= LARGE_MAXCLASS);
} }
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
@ -2233,7 +2233,7 @@ je_rallocx(void *ptr, size_t size, int flags)
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) if (unlikely(usize == 0 || usize > LARGE_MAXCLASS))
goto label_oom; goto label_oom;
p = irallocx_prof(tsd, extent, ptr, old_usize, size, alignment, p = irallocx_prof(tsd, extent, ptr, old_usize, size, alignment,
&usize, zero, tcache, arena); &usize, zero, tcache, arena);
@ -2314,17 +2314,17 @@ ixallocx_prof(tsd_t *tsd, extent_t *extent, void *ptr, size_t old_usize,
*/ */
if (alignment == 0) { if (alignment == 0) {
usize_max = s2u(size+extra); usize_max = s2u(size+extra);
assert(usize_max > 0 && usize_max <= HUGE_MAXCLASS); assert(usize_max > 0 && usize_max <= LARGE_MAXCLASS);
} else { } else {
usize_max = sa2u(size+extra, alignment); usize_max = sa2u(size+extra, alignment);
if (unlikely(usize_max == 0 || usize_max > HUGE_MAXCLASS)) { if (unlikely(usize_max == 0 || usize_max > LARGE_MAXCLASS)) {
/* /*
* usize_max is out of range, and chances are that * usize_max is out of range, and chances are that
* allocation will fail, but use the maximum possible * allocation will fail, but use the maximum possible
* value and carry on with prof_alloc_prep(), just in * value and carry on with prof_alloc_prep(), just in
* case allocation succeeds. * case allocation succeeds.
*/ */
usize_max = HUGE_MAXCLASS; usize_max = LARGE_MAXCLASS;
} }
} }
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
@ -2368,18 +2368,18 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
/* /*
* The API explicitly absolves itself of protecting against (size + * The API explicitly absolves itself of protecting against (size +
* extra) numerical overflow, but we may need to clamp extra to avoid * extra) numerical overflow, but we may need to clamp extra to avoid
* exceeding HUGE_MAXCLASS. * exceeding LARGE_MAXCLASS.
* *
* Ordinarily, size limit checking is handled deeper down, but here we * Ordinarily, size limit checking is handled deeper down, but here we
* have to check as part of (size + extra) clamping, since we need the * have to check as part of (size + extra) clamping, since we need the
* clamped value in the above helper functions. * clamped value in the above helper functions.
*/ */
if (unlikely(size > HUGE_MAXCLASS)) { if (unlikely(size > LARGE_MAXCLASS)) {
usize = old_usize; usize = old_usize;
goto label_not_resized; goto label_not_resized;
} }
if (unlikely(HUGE_MAXCLASS - size < extra)) if (unlikely(LARGE_MAXCLASS - size < extra))
extra = HUGE_MAXCLASS - size; extra = LARGE_MAXCLASS - size;
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = ixallocx_prof(tsd, extent, ptr, old_usize, size, extra, usize = ixallocx_prof(tsd, extent, ptr, old_usize, size, extra,
@ -2512,7 +2512,7 @@ je_nallocx(size_t size, int flags)
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn);
usize = inallocx(tsdn, size, flags); usize = inallocx(tsdn, size, flags);
if (unlikely(usize > HUGE_MAXCLASS)) if (unlikely(usize > LARGE_MAXCLASS))
return (0); return (0);
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn);

119
src/huge.c → src/large.c
View File

@ -1,19 +1,19 @@
#define JEMALLOC_HUGE_C_ #define JEMALLOC_LARGE_C_
#include "jemalloc/internal/jemalloc_internal.h" #include "jemalloc/internal/jemalloc_internal.h"
/******************************************************************************/ /******************************************************************************/
void * void *
huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
{ {
assert(usize == s2u(usize)); assert(usize == s2u(usize));
return (huge_palloc(tsdn, arena, usize, CACHELINE, zero)); return (large_palloc(tsdn, arena, usize, CACHELINE, zero));
} }
void * void *
huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero) bool zero)
{ {
size_t ausize; size_t ausize;
@ -24,7 +24,7 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
assert(!tsdn_null(tsdn) || arena != NULL); assert(!tsdn_null(tsdn) || arena != NULL);
ausize = sa2u(usize, alignment); ausize = sa2u(usize, alignment);
if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS)) if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS))
return (NULL); return (NULL);
/* /*
@ -34,15 +34,15 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
is_zeroed = zero; is_zeroed = zero;
if (likely(!tsdn_null(tsdn))) if (likely(!tsdn_null(tsdn)))
arena = arena_choose(tsdn_tsd(tsdn), arena); arena = arena_choose(tsdn_tsd(tsdn), arena);
if (unlikely(arena == NULL) || (extent = arena_chunk_alloc_huge(tsdn, if (unlikely(arena == NULL) || (extent = arena_chunk_alloc_large(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL) arena, usize, alignment, &is_zeroed)) == NULL)
return (NULL); return (NULL);
/* Insert extent into huge. */ /* Insert extent into large. */
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->large_mtx);
ql_elm_new(extent, ql_link); ql_elm_new(extent, ql_link);
ql_tail_insert(&arena->huge, extent, ql_link); ql_tail_insert(&arena->large, extent, ql_link);
malloc_mutex_unlock(tsdn, &arena->huge_mtx); malloc_mutex_unlock(tsdn, &arena->large_mtx);
if (config_prof && arena_prof_accum(tsdn, arena, usize)) if (config_prof && arena_prof_accum(tsdn, arena, usize))
prof_idump(tsdn); prof_idump(tsdn);
@ -61,23 +61,23 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
} }
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
#undef huge_dalloc_junk #undef large_dalloc_junk
#define huge_dalloc_junk JEMALLOC_N(n_huge_dalloc_junk) #define large_dalloc_junk JEMALLOC_N(n_large_dalloc_junk)
#endif #endif
void void
huge_dalloc_junk(void *ptr, size_t usize) large_dalloc_junk(void *ptr, size_t usize)
{ {
memset(ptr, JEMALLOC_FREE_JUNK, usize); memset(ptr, JEMALLOC_FREE_JUNK, usize);
} }
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
#undef huge_dalloc_junk #undef large_dalloc_junk
#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) #define large_dalloc_junk JEMALLOC_N(large_dalloc_junk)
huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(n_huge_dalloc_junk); large_dalloc_junk_t *large_dalloc_junk = JEMALLOC_N(n_large_dalloc_junk);
#endif #endif
static void static void
huge_dalloc_maybe_junk(tsdn_t *tsdn, void *ptr, size_t usize) large_dalloc_maybe_junk(tsdn_t *tsdn, void *ptr, size_t usize)
{ {
if (config_fill && have_dss && unlikely(opt_junk_free)) { if (config_fill && have_dss && unlikely(opt_junk_free)) {
@ -86,13 +86,13 @@ huge_dalloc_maybe_junk(tsdn_t *tsdn, void *ptr, size_t usize)
* unmapped. * unmapped.
*/ */
if (!config_munmap || (have_dss && chunk_in_dss(tsdn, ptr))) if (!config_munmap || (have_dss && chunk_in_dss(tsdn, ptr)))
huge_dalloc_junk(ptr, usize); large_dalloc_junk(ptr, usize);
memset(ptr, JEMALLOC_FREE_JUNK, usize); memset(ptr, JEMALLOC_FREE_JUNK, usize);
} }
} }
static bool static bool
huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
{ {
arena_t *arena = extent_arena_get(extent); arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent); size_t oldusize = extent_usize_get(extent);
@ -109,20 +109,20 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
return (true); return (true);
if (config_fill && unlikely(opt_junk_free)) { if (config_fill && unlikely(opt_junk_free)) {
huge_dalloc_maybe_junk(tsdn, extent_addr_get(trail), large_dalloc_maybe_junk(tsdn, extent_addr_get(trail),
extent_usize_get(trail)); extent_usize_get(trail));
} }
arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks, trail); arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks, trail);
} }
arena_chunk_ralloc_huge_shrink(tsdn, arena, extent, oldusize); arena_chunk_ralloc_large_shrink(tsdn, arena, extent, oldusize);
return (false); return (false);
} }
static bool static bool
huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool zero) bool zero)
{ {
arena_t *arena = extent_arena_get(extent); arena_t *arena = extent_arena_get(extent);
@ -173,34 +173,35 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
JEMALLOC_ALLOC_JUNK, usize - oldusize); JEMALLOC_ALLOC_JUNK, usize - oldusize);
} }
arena_chunk_ralloc_huge_expand(tsdn, arena, extent, oldusize); arena_chunk_ralloc_large_expand(tsdn, arena, extent, oldusize);
return (false); return (false);
} }
bool bool
huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero) size_t usize_max, bool zero)
{ {
assert(s2u(extent_usize_get(extent)) == extent_usize_get(extent)); assert(s2u(extent_usize_get(extent)) == extent_usize_get(extent));
/* The following should have been caught by callers. */ /* The following should have been caught by callers. */
assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS); assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS);
/* Both allocation sizes must be huge to avoid a move. */ /* Both allocation sizes must be large to avoid a move. */
assert(extent_usize_get(extent) >= LARGE_MINCLASS && usize_max >= assert(extent_usize_get(extent) >= LARGE_MINCLASS && usize_max >=
LARGE_MINCLASS); LARGE_MINCLASS);
if (usize_max > extent_usize_get(extent)) { if (usize_max > extent_usize_get(extent)) {
/* Attempt to expand the allocation in-place. */ /* Attempt to expand the allocation in-place. */
if (!huge_ralloc_no_move_expand(tsdn, extent, usize_max, if (!large_ralloc_no_move_expand(tsdn, extent, usize_max,
zero)) { zero)) {
arena_decay_tick(tsdn, extent_arena_get(extent)); arena_decay_tick(tsdn, extent_arena_get(extent));
return (false); return (false);
} }
/* Try again, this time with usize_min. */ /* Try again, this time with usize_min. */
if (usize_min < usize_max && usize_min > if (usize_min < usize_max && usize_min >
extent_usize_get(extent) && huge_ralloc_no_move_expand(tsdn, extent_usize_get(extent) &&
extent, usize_min, zero)) { large_ralloc_no_move_expand(tsdn, extent, usize_min,
zero)) {
arena_decay_tick(tsdn, extent_arena_get(extent)); arena_decay_tick(tsdn, extent_arena_get(extent));
return (false); return (false);
} }
@ -218,7 +219,7 @@ huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
/* Attempt to shrink the allocation in-place. */ /* Attempt to shrink the allocation in-place. */
if (extent_usize_get(extent) > usize_max) { if (extent_usize_get(extent) > usize_max) {
if (!huge_ralloc_no_move_shrink(tsdn, extent, usize_max)) { if (!large_ralloc_no_move_shrink(tsdn, extent, usize_max)) {
arena_decay_tick(tsdn, extent_arena_get(extent)); arena_decay_tick(tsdn, extent_arena_get(extent));
return (false); return (false);
} }
@ -227,30 +228,30 @@ huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
} }
static void * static void *
huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero) size_t alignment, bool zero)
{ {
if (alignment <= CACHELINE) if (alignment <= CACHELINE)
return (huge_malloc(tsdn, arena, usize, zero)); return (large_malloc(tsdn, arena, usize, zero));
return (huge_palloc(tsdn, arena, usize, alignment, zero)); return (large_palloc(tsdn, arena, usize, alignment, zero));
} }
void * void *
huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
size_t alignment, bool zero, tcache_t *tcache) size_t alignment, bool zero, tcache_t *tcache)
{ {
void *ret; void *ret;
size_t copysize; size_t copysize;
/* The following should have been caught by callers. */ /* The following should have been caught by callers. */
assert(usize > 0 && usize <= HUGE_MAXCLASS); assert(usize > 0 && usize <= LARGE_MAXCLASS);
/* Both allocation sizes must be huge to avoid a move. */ /* Both allocation sizes must be large to avoid a move. */
assert(extent_usize_get(extent) >= LARGE_MINCLASS && usize >= assert(extent_usize_get(extent) >= LARGE_MINCLASS && usize >=
LARGE_MINCLASS); LARGE_MINCLASS);
/* Try to avoid moving the allocation. */ /* Try to avoid moving the allocation. */
if (!huge_ralloc_no_move(tsdn, extent, usize, usize, zero)) if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero))
return (extent_addr_get(extent)); return (extent_addr_get(extent));
/* /*
@ -258,7 +259,7 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
* different size class. In that case, fall back to allocating new * different size class. In that case, fall back to allocating new
* space and copying. * space and copying.
*/ */
ret = huge_ralloc_move_helper(tsdn, arena, usize, alignment, zero); ret = large_ralloc_move_helper(tsdn, arena, usize, alignment, zero);
if (ret == NULL) if (ret == NULL)
return (NULL); return (NULL);
@ -271,82 +272,82 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
} }
static void static void
huge_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked) large_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked)
{ {
arena_t *arena; arena_t *arena;
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
if (!junked_locked) if (!junked_locked)
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->large_mtx);
ql_remove(&arena->huge, extent, ql_link); ql_remove(&arena->large, extent, ql_link);
if (!junked_locked) { if (!junked_locked) {
malloc_mutex_unlock(tsdn, &arena->huge_mtx); malloc_mutex_unlock(tsdn, &arena->large_mtx);
huge_dalloc_maybe_junk(tsdn, extent_addr_get(extent), large_dalloc_maybe_junk(tsdn, extent_addr_get(extent),
extent_usize_get(extent)); extent_usize_get(extent));
} }
arena_chunk_dalloc_huge(tsdn, arena, extent, junked_locked); arena_chunk_dalloc_large(tsdn, arena, extent, junked_locked);
if (!junked_locked) if (!junked_locked)
arena_decay_tick(tsdn, arena); arena_decay_tick(tsdn, arena);
} }
void void
huge_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent) large_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent)
{ {
huge_dalloc_impl(tsdn, extent, true); large_dalloc_impl(tsdn, extent, true);
} }
void void
huge_dalloc(tsdn_t *tsdn, extent_t *extent) large_dalloc(tsdn_t *tsdn, extent_t *extent)
{ {
huge_dalloc_impl(tsdn, extent, false); large_dalloc_impl(tsdn, extent, false);
} }
size_t size_t
huge_salloc(tsdn_t *tsdn, const extent_t *extent) large_salloc(tsdn_t *tsdn, const extent_t *extent)
{ {
size_t usize; size_t usize;
arena_t *arena; arena_t *arena;
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->large_mtx);
usize = extent_usize_get(extent); usize = extent_usize_get(extent);
malloc_mutex_unlock(tsdn, &arena->huge_mtx); malloc_mutex_unlock(tsdn, &arena->large_mtx);
return (usize); return (usize);
} }
prof_tctx_t * prof_tctx_t *
huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent) large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent)
{ {
prof_tctx_t *tctx; prof_tctx_t *tctx;
arena_t *arena; arena_t *arena;
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->large_mtx);
tctx = extent_prof_tctx_get(extent); tctx = extent_prof_tctx_get(extent);
malloc_mutex_unlock(tsdn, &arena->huge_mtx); malloc_mutex_unlock(tsdn, &arena->large_mtx);
return (tctx); return (tctx);
} }
void void
huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx) large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx)
{ {
arena_t *arena; arena_t *arena;
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->large_mtx);
extent_prof_tctx_set(extent, tctx); extent_prof_tctx_set(extent, tctx);
malloc_mutex_unlock(tsdn, &arena->huge_mtx); malloc_mutex_unlock(tsdn, &arena->large_mtx);
} }
void void
huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent)
{ {
huge_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U); large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
} }

71
src/stats.c
View File

@ -37,10 +37,10 @@ size_t stats_cactive = 0;
static void stats_arena_bins_print(void (*write_cb)(void *, const char *), static void stats_arena_bins_print(void (*write_cb)(void *, const char *),
void *cbopaque, unsigned i); void *cbopaque, unsigned i);
static void stats_arena_hchunks_print( static void stats_arena_lextents_print(
void (*write_cb)(void *, const char *), void *cbopaque, unsigned i); void (*write_cb)(void *, const char *), void *cbopaque, unsigned i);
static void stats_arena_print(void (*write_cb)(void *, const char *), static void stats_arena_print(void (*write_cb)(void *, const char *),
void *cbopaque, unsigned i, bool bins, bool huge); void *cbopaque, unsigned i, bool bins, bool large);
/******************************************************************************/ /******************************************************************************/
@ -157,34 +157,34 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
} }
static void static void
stats_arena_hchunks_print(void (*write_cb)(void *, const char *), stats_arena_lextents_print(void (*write_cb)(void *, const char *),
void *cbopaque, unsigned i) void *cbopaque, unsigned i)
{ {
unsigned nbins, nhchunks, j; unsigned nbins, nlextents, j;
bool in_gap; bool in_gap;
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"huge: size ind allocated nmalloc ndalloc" "large: size ind allocated nmalloc ndalloc"
" nrequests curhchunks\n"); " nrequests curlextents\n");
CTL_GET("arenas.nbins", &nbins, unsigned); CTL_GET("arenas.nbins", &nbins, unsigned);
CTL_GET("arenas.nhchunks", &nhchunks, unsigned); CTL_GET("arenas.nlextents", &nlextents, unsigned);
for (j = 0, in_gap = false; j < nhchunks; j++) { for (j = 0, in_gap = false; j < nlextents; j++) {
uint64_t nmalloc, ndalloc, nrequests; uint64_t nmalloc, ndalloc, nrequests;
size_t hchunk_size, curhchunks; size_t lextent_size, curlextents;
CTL_M2_M4_GET("stats.arenas.0.hchunks.0.nmalloc", i, j, CTL_M2_M4_GET("stats.arenas.0.lextents.0.nmalloc", i, j,
&nmalloc, uint64_t); &nmalloc, uint64_t);
CTL_M2_M4_GET("stats.arenas.0.hchunks.0.ndalloc", i, j, CTL_M2_M4_GET("stats.arenas.0.lextents.0.ndalloc", i, j,
&ndalloc, uint64_t); &ndalloc, uint64_t);
CTL_M2_M4_GET("stats.arenas.0.hchunks.0.nrequests", i, j, CTL_M2_M4_GET("stats.arenas.0.lextents.0.nrequests", i, j,
&nrequests, uint64_t); &nrequests, uint64_t);
if (nrequests == 0) if (nrequests == 0)
in_gap = true; in_gap = true;
else { else {
CTL_M2_GET("arenas.hchunk.0.size", j, &hchunk_size, CTL_M2_GET("arenas.lextent.0.size", j, &lextent_size,
size_t); size_t);
CTL_M2_M4_GET("stats.arenas.0.hchunks.0.curhchunks", i, CTL_M2_M4_GET("stats.arenas.0.lextents.0.curlextents",
j, &curhchunks, size_t); i, j, &curlextents, size_t);
if (in_gap) { if (in_gap) {
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
" ---\n"); " ---\n");
@ -193,9 +193,9 @@ stats_arena_hchunks_print(void (*write_cb)(void *, const char *),
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"%20zu %3u %12zu %12"FMTu64" %12"FMTu64 "%20zu %3u %12zu %12"FMTu64" %12"FMTu64
" %12"FMTu64" %12zu\n", " %12"FMTu64" %12zu\n",
hchunk_size, nbins + j, lextent_size, nbins + j,
curhchunks * hchunk_size, nmalloc, ndalloc, curlextents * lextent_size, nmalloc, ndalloc,
nrequests, curhchunks); nrequests, curlextents);
} }
} }
if (in_gap) { if (in_gap) {
@ -206,7 +206,7 @@ stats_arena_hchunks_print(void (*write_cb)(void *, const char *),
static void static void
stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
unsigned i, bool bins, bool huge) unsigned i, bool bins, bool large)
{ {
unsigned nthreads; unsigned nthreads;
const char *dss; const char *dss;
@ -216,8 +216,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
uint64_t npurge, nmadvise, purged; uint64_t npurge, nmadvise, purged;
size_t small_allocated; size_t small_allocated;
uint64_t small_nmalloc, small_ndalloc, small_nrequests; uint64_t small_nmalloc, small_ndalloc, small_nrequests;
size_t huge_allocated; size_t large_allocated;
uint64_t huge_nmalloc, huge_ndalloc, huge_nrequests; uint64_t large_nmalloc, large_ndalloc, large_nrequests;
CTL_GET("arenas.page", &page, size_t); CTL_GET("arenas.page", &page, size_t);
@ -268,20 +268,21 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
"small: %12zu %12"FMTu64" %12"FMTu64 "small: %12zu %12"FMTu64" %12"FMTu64
" %12"FMTu64"\n", " %12"FMTu64"\n",
small_allocated, small_nmalloc, small_ndalloc, small_nrequests); small_allocated, small_nmalloc, small_ndalloc, small_nrequests);
CTL_M2_GET("stats.arenas.0.huge.allocated", i, &huge_allocated, size_t); CTL_M2_GET("stats.arenas.0.large.allocated", i, &large_allocated,
CTL_M2_GET("stats.arenas.0.huge.nmalloc", i, &huge_nmalloc, uint64_t); size_t);
CTL_M2_GET("stats.arenas.0.huge.ndalloc", i, &huge_ndalloc, uint64_t); CTL_M2_GET("stats.arenas.0.large.nmalloc", i, &large_nmalloc, uint64_t);
CTL_M2_GET("stats.arenas.0.huge.nrequests", i, &huge_nrequests, CTL_M2_GET("stats.arenas.0.large.ndalloc", i, &large_ndalloc, uint64_t);
CTL_M2_GET("stats.arenas.0.large.nrequests", i, &large_nrequests,
uint64_t); uint64_t);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"huge: %12zu %12"FMTu64" %12"FMTu64 "large: %12zu %12"FMTu64" %12"FMTu64
" %12"FMTu64"\n", " %12"FMTu64"\n",
huge_allocated, huge_nmalloc, huge_ndalloc, huge_nrequests); large_allocated, large_nmalloc, large_ndalloc, large_nrequests);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"total: %12zu %12"FMTu64" %12"FMTu64 "total: %12zu %12"FMTu64" %12"FMTu64
" %12"FMTu64"\n", " %12"FMTu64"\n",
small_allocated + huge_allocated, small_nmalloc + huge_nmalloc, small_allocated + large_allocated, small_nmalloc + large_nmalloc,
small_ndalloc + huge_ndalloc, small_nrequests + huge_nrequests); small_ndalloc + large_ndalloc, small_nrequests + large_nrequests);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"active: %12zu\n", pactive * page); "active: %12zu\n", pactive * page);
CTL_M2_GET("stats.arenas.0.mapped", i, &mapped, size_t); CTL_M2_GET("stats.arenas.0.mapped", i, &mapped, size_t);
@ -300,8 +301,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
if (bins) if (bins)
stats_arena_bins_print(write_cb, cbopaque, i); stats_arena_bins_print(write_cb, cbopaque, i);
if (huge) if (large)
stats_arena_hchunks_print(write_cb, cbopaque, i); stats_arena_lextents_print(write_cb, cbopaque, i);
} }
void void
@ -315,7 +316,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool merged = true; bool merged = true;
bool unmerged = true; bool unmerged = true;
bool bins = true; bool bins = true;
bool huge = true; bool large = true;
/* /*
* Refresh stats, in case mallctl() was called by the application. * Refresh stats, in case mallctl() was called by the application.
@ -356,7 +357,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
bins = false; bins = false;
break; break;
case 'l': case 'l':
huge = false; large = false;
break; break;
default:; default:;
} }
@ -568,7 +569,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\nMerged arenas stats:\n"); "\nMerged arenas stats:\n");
stats_arena_print(write_cb, cbopaque, stats_arena_print(write_cb, cbopaque,
narenas, bins, huge); narenas, bins, large);
} }
} }
} }
@ -594,7 +595,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
cbopaque, cbopaque,
"\narenas[%u]:\n", i); "\narenas[%u]:\n", i);
stats_arena_print(write_cb, stats_arena_print(write_cb,
cbopaque, i, bins, huge); cbopaque, i, bins, large);
} }
} }
} }

28
src/tcache.c
View File

@ -46,7 +46,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache)
tbin->ncached - tbin->low_water + (tbin->low_water tbin->ncached - tbin->low_water + (tbin->low_water
>> 2)); >> 2));
} else { } else {
tcache_bin_flush_huge(tsd, tbin, binind, tbin->ncached tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached
- tbin->low_water + (tbin->low_water >> 2), tcache); - tbin->low_water + (tbin->low_water >> 2), tcache);
} }
/* /*
@ -164,7 +164,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
} }
void void
tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
unsigned rem, tcache_t *tcache) unsigned rem, tcache_t *tcache)
{ {
arena_t *arena; arena_t *arena;
@ -194,9 +194,9 @@ tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
} }
if (config_stats) { if (config_stats) {
merged_stats = true; merged_stats = true;
arena->stats.nrequests_huge += arena->stats.nrequests_large +=
tbin->tstats.nrequests; tbin->tstats.nrequests;
arena->stats.hstats[binind - NBINS].nrequests += arena->stats.lstats[binind - NBINS].nrequests +=
tbin->tstats.nrequests; tbin->tstats.nrequests;
tbin->tstats.nrequests = 0; tbin->tstats.nrequests = 0;
} }
@ -207,7 +207,7 @@ tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
assert(ptr != NULL); assert(ptr != NULL);
extent = iealloc(tsd_tsdn(tsd), ptr); extent = iealloc(tsd_tsdn(tsd), ptr);
if (extent_arena_get(extent) == locked_arena) { if (extent_arena_get(extent) == locked_arena) {
huge_dalloc_junked_locked(tsd_tsdn(tsd), large_dalloc_junked_locked(tsd_tsdn(tsd),
extent); extent);
} else { } else {
/* /*
@ -232,8 +232,8 @@ tcache_bin_flush_huge(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
* arena, so the stats didn't get merged. Manually do so now. * arena, so the stats didn't get merged. Manually do so now.
*/ */
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock); malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
arena->stats.nrequests_huge += tbin->tstats.nrequests; arena->stats.nrequests_large += tbin->tstats.nrequests;
arena->stats.hstats[binind - NBINS].nrequests += arena->stats.lstats[binind - NBINS].nrequests +=
tbin->tstats.nrequests; tbin->tstats.nrequests;
tbin->tstats.nrequests = 0; tbin->tstats.nrequests = 0;
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock); malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
@ -371,12 +371,12 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
for (; i < nhbins; i++) { for (; i < nhbins; i++) {
tcache_bin_t *tbin = &tcache->tbins[i]; tcache_bin_t *tbin = &tcache->tbins[i];
tcache_bin_flush_huge(tsd, tbin, i, 0, tcache); tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
if (config_stats && tbin->tstats.nrequests != 0) { if (config_stats && tbin->tstats.nrequests != 0) {
malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock); malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
arena->stats.nrequests_huge += tbin->tstats.nrequests; arena->stats.nrequests_large += tbin->tstats.nrequests;
arena->stats.hstats[i - NBINS].nrequests += arena->stats.lstats[i - NBINS].nrequests +=
tbin->tstats.nrequests; tbin->tstats.nrequests;
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock); malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
} }
@ -431,10 +431,10 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
} }
for (; i < nhbins; i++) { for (; i < nhbins; i++) {
malloc_huge_stats_t *hstats = &arena->stats.hstats[i - NBINS]; malloc_large_stats_t *lstats = &arena->stats.lstats[i - NBINS];
tcache_bin_t *tbin = &tcache->tbins[i]; tcache_bin_t *tbin = &tcache->tbins[i];
arena->stats.nrequests_huge += tbin->tstats.nrequests; arena->stats.nrequests_large += tbin->tstats.nrequests;
hstats->nrequests += tbin->tstats.nrequests; lstats->nrequests += tbin->tstats.nrequests;
tbin->tstats.nrequests = 0; tbin->tstats.nrequests = 0;
} }
} }
@ -537,7 +537,7 @@ tcache_boot(tsdn_t *tsdn)
stack_nelms += tcache_bin_info[i].ncached_max; stack_nelms += tcache_bin_info[i].ncached_max;
} }
for (; i < nhbins; i++) { for (; i < nhbins; i++) {
tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_HUGE; tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
stack_nelms += tcache_bin_info[i].ncached_max; stack_nelms += tcache_bin_info[i].ncached_max;
} }

28
test/integration/chunk.c
View File

@ -120,7 +120,7 @@ chunk_merge(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
TEST_BEGIN(test_chunk) TEST_BEGIN(test_chunk)
{ {
void *p; void *p;
size_t old_size, new_size, huge0, huge1, huge2, sz; size_t old_size, new_size, large0, large1, large2, sz;
unsigned arena_ind; unsigned arena_ind;
int flags; int flags;
size_t hooks_mib[3], purge_mib[3]; size_t hooks_mib[3], purge_mib[3];
@ -162,14 +162,14 @@ TEST_BEGIN(test_chunk)
assert_ptr_ne(old_hooks.split, chunk_split, "Unexpected split error"); assert_ptr_ne(old_hooks.split, chunk_split, "Unexpected split error");
assert_ptr_ne(old_hooks.merge, chunk_merge, "Unexpected merge error"); assert_ptr_ne(old_hooks.merge, chunk_merge, "Unexpected merge error");
/* Get huge size classes. */ /* Get large size classes. */
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.lextent.0.size", &large0, &sz, NULL, 0), 0,
"Unexpected arenas.hchunk.0.size failure"); "Unexpected arenas.lextent.0.size failure");
assert_d_eq(mallctl("arenas.hchunk.1.size", &huge1, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.lextent.1.size", &large1, &sz, NULL, 0), 0,
"Unexpected arenas.hchunk.1.size failure"); "Unexpected arenas.lextent.1.size failure");
assert_d_eq(mallctl("arenas.hchunk.2.size", &huge2, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.lextent.2.size", &large2, &sz, NULL, 0), 0,
"Unexpected arenas.hchunk.2.size failure"); "Unexpected arenas.lextent.2.size failure");
/* Test dalloc/decommit/purge cascade. */ /* Test dalloc/decommit/purge cascade. */
purge_miblen = sizeof(purge_mib)/sizeof(size_t); purge_miblen = sizeof(purge_mib)/sizeof(size_t);
@ -178,13 +178,13 @@ TEST_BEGIN(test_chunk)
purge_mib[1] = (size_t)arena_ind; purge_mib[1] = (size_t)arena_ind;
do_dalloc = false; do_dalloc = false;
do_decommit = false; do_decommit = false;
p = mallocx(huge0 * 2, flags); p = mallocx(large0 * 2, flags);
assert_ptr_not_null(p, "Unexpected mallocx() error"); assert_ptr_not_null(p, "Unexpected mallocx() error");
did_dalloc = false; did_dalloc = false;
did_decommit = false; did_decommit = false;
did_purge = false; did_purge = false;
did_split = false; did_split = false;
xallocx_success_a = (xallocx(p, huge0, 0, flags) == huge0); xallocx_success_a = (xallocx(p, large0, 0, flags) == large0);
assert_d_eq(mallctlbymib(purge_mib, purge_miblen, NULL, NULL, NULL, 0), assert_d_eq(mallctlbymib(purge_mib, purge_miblen, NULL, NULL, NULL, 0),
0, "Unexpected arena.%u.purge error", arena_ind); 0, "Unexpected arena.%u.purge error", arena_ind);
if (xallocx_success_a) { if (xallocx_success_a) {
@ -199,18 +199,18 @@ TEST_BEGIN(test_chunk)
/* Test decommit/commit and observe split/merge. */ /* Test decommit/commit and observe split/merge. */
do_dalloc = false; do_dalloc = false;
do_decommit = true; do_decommit = true;
p = mallocx(huge0 * 2, flags); p = mallocx(large0 * 2, flags);
assert_ptr_not_null(p, "Unexpected mallocx() error"); assert_ptr_not_null(p, "Unexpected mallocx() error");
did_decommit = false; did_decommit = false;
did_commit = false; did_commit = false;
did_split = false; did_split = false;
did_merge = false; did_merge = false;
xallocx_success_b = (xallocx(p, huge0, 0, flags) == huge0); xallocx_success_b = (xallocx(p, large0, 0, flags) == large0);
assert_d_eq(mallctlbymib(purge_mib, purge_miblen, NULL, NULL, NULL, 0), assert_d_eq(mallctlbymib(purge_mib, purge_miblen, NULL, NULL, NULL, 0),
0, "Unexpected arena.%u.purge error", arena_ind); 0, "Unexpected arena.%u.purge error", arena_ind);
if (xallocx_success_b) if (xallocx_success_b)
assert_true(did_split, "Expected split"); assert_true(did_split, "Expected split");
xallocx_success_c = (xallocx(p, huge0 * 2, 0, flags) == huge0 * 2); xallocx_success_c = (xallocx(p, large0 * 2, 0, flags) == large0 * 2);
assert_b_eq(did_decommit, did_commit, "Expected decommit/commit match"); assert_b_eq(did_decommit, did_commit, "Expected decommit/commit match");
if (xallocx_success_b && xallocx_success_c) if (xallocx_success_b && xallocx_success_c)
assert_true(did_merge, "Expected merge"); assert_true(did_merge, "Expected merge");
@ -218,7 +218,7 @@ TEST_BEGIN(test_chunk)
do_dalloc = true; do_dalloc = true;
do_decommit = false; do_decommit = false;
/* Make sure non-huge allocation succeeds. */ /* Make sure non-large allocation succeeds. */
p = mallocx(42, flags); p = mallocx(42, flags);
assert_ptr_not_null(p, "Unexpected mallocx() error"); assert_ptr_not_null(p, "Unexpected mallocx() error");
dallocx(p, flags); dallocx(p, flags);

24
test/integration/mallocx.c
View File

@ -18,10 +18,10 @@ get_nsizes_impl(const char *cmd)
} }
static unsigned static unsigned
get_nhuge(void) get_nlarge(void)
{ {
return (get_nsizes_impl("arenas.nhchunks")); return (get_nsizes_impl("arenas.nlextents"));
} }
static size_t static size_t
@ -44,20 +44,20 @@ get_size_impl(const char *cmd, size_t ind)
} }
static size_t static size_t
get_huge_size(size_t ind) get_large_size(size_t ind)
{ {
return (get_size_impl("arenas.hchunk.0.size", ind)); return (get_size_impl("arenas.lextent.0.size", ind));
} }
TEST_BEGIN(test_overflow) TEST_BEGIN(test_overflow)
{ {
size_t hugemax; size_t largemax;
hugemax = get_huge_size(get_nhuge()-1); largemax = get_large_size(get_nlarge()-1);
assert_ptr_null(mallocx(hugemax+1, 0), assert_ptr_null(mallocx(largemax+1, 0),
"Expected OOM for mallocx(size=%#zx, 0)", hugemax+1); "Expected OOM for mallocx(size=%#zx, 0)", largemax+1);
assert_ptr_null(mallocx(ZU(PTRDIFF_MAX)+1, 0), assert_ptr_null(mallocx(ZU(PTRDIFF_MAX)+1, 0),
"Expected OOM for mallocx(size=%#zx, 0)", ZU(PTRDIFF_MAX)+1); "Expected OOM for mallocx(size=%#zx, 0)", ZU(PTRDIFF_MAX)+1);
@ -73,7 +73,7 @@ TEST_END
TEST_BEGIN(test_oom) TEST_BEGIN(test_oom)
{ {
size_t hugemax; size_t largemax;
bool oom; bool oom;
void *ptrs[3]; void *ptrs[3];
unsigned i; unsigned i;
@ -82,16 +82,16 @@ TEST_BEGIN(test_oom)
* It should be impossible to allocate three objects that each consume * It should be impossible to allocate three objects that each consume
* nearly half the virtual address space. * nearly half the virtual address space.
*/ */
hugemax = get_huge_size(get_nhuge()-1); largemax = get_large_size(get_nlarge()-1);
oom = false; oom = false;
for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) {
ptrs[i] = mallocx(hugemax, 0); ptrs[i] = mallocx(largemax, 0);
if (ptrs[i] == NULL) if (ptrs[i] == NULL)
oom = true; oom = true;
} }
assert_true(oom, assert_true(oom,
"Expected OOM during series of calls to mallocx(size=%zu, 0)", "Expected OOM during series of calls to mallocx(size=%zu, 0)",
hugemax); largemax);
for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) {
if (ptrs[i] != NULL) if (ptrs[i] != NULL)
dallocx(ptrs[i], 0); dallocx(ptrs[i], 0);

8
test/integration/overflow.c
View File

@ -2,19 +2,19 @@
TEST_BEGIN(test_overflow) TEST_BEGIN(test_overflow)
{ {
unsigned nhchunks; unsigned nlextents;
size_t mib[4]; size_t mib[4];
size_t sz, miblen, max_size_class; size_t sz, miblen, max_size_class;
void *p; void *p;
sz = sizeof(unsigned); sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.nhchunks", &nhchunks, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.nlextents", &nlextents, &sz, NULL, 0), 0,
"Unexpected mallctl() error"); "Unexpected mallctl() error");
miblen = sizeof(mib) / sizeof(size_t); miblen = sizeof(mib) / sizeof(size_t);
assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0, assert_d_eq(mallctlnametomib("arenas.lextent.0.size", mib, &miblen), 0,
"Unexpected mallctlnametomib() error"); "Unexpected mallctlnametomib() error");
mib[2] = nhchunks - 1; mib[2] = nlextents - 1;
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0, assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0,

16
test/integration/rallocx.c
View File

@ -14,10 +14,10 @@ get_nsizes_impl(const char *cmd)
} }
static unsigned static unsigned
get_nhuge(void) get_nlarge(void)
{ {
return (get_nsizes_impl("arenas.nhchunks")); return (get_nsizes_impl("arenas.nlextents"));
} }
static size_t static size_t
@ -40,10 +40,10 @@ get_size_impl(const char *cmd, size_t ind)
} }
static size_t static size_t
get_huge_size(size_t ind) get_large_size(size_t ind)
{ {
return (get_size_impl("arenas.hchunk.0.size", ind)); return (get_size_impl("arenas.lextent.0.size", ind));
} }
TEST_BEGIN(test_grow_and_shrink) TEST_BEGIN(test_grow_and_shrink)
@ -221,16 +221,16 @@ TEST_END
TEST_BEGIN(test_overflow) TEST_BEGIN(test_overflow)
{ {
size_t hugemax; size_t largemax;
void *p; void *p;
hugemax = get_huge_size(get_nhuge()-1); largemax = get_large_size(get_nlarge()-1);
p = mallocx(1, 0); p = mallocx(1, 0);
assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_ptr_not_null(p, "Unexpected mallocx() failure");
assert_ptr_null(rallocx(p, hugemax+1, 0), assert_ptr_null(rallocx(p, largemax+1, 0),
"Expected OOM for rallocx(p, size=%#zx, 0)", hugemax+1); "Expected OOM for rallocx(p, size=%#zx, 0)", largemax+1);
assert_ptr_null(rallocx(p, ZU(PTRDIFF_MAX)+1, 0), assert_ptr_null(rallocx(p, ZU(PTRDIFF_MAX)+1, 0),
"Expected OOM for rallocx(p, size=%#zx, 0)", ZU(PTRDIFF_MAX)+1); "Expected OOM for rallocx(p, size=%#zx, 0)", ZU(PTRDIFF_MAX)+1);

104
test/integration/xallocx.c
View File

@ -92,10 +92,10 @@ get_nsmall(void)
} }
static unsigned static unsigned
get_nhuge(void) get_nlarge(void)
{ {
return (get_nsizes_impl("arenas.nhchunks")); return (get_nsizes_impl("arenas.nlextents"));
} }
static size_t static size_t
@ -125,20 +125,20 @@ get_small_size(size_t ind)
} }
static size_t static size_t
get_huge_size(size_t ind) get_large_size(size_t ind)
{ {
return (get_size_impl("arenas.hchunk.0.size", ind)); return (get_size_impl("arenas.lextent.0.size", ind));
} }
TEST_BEGIN(test_size) TEST_BEGIN(test_size)
{ {
size_t small0, hugemax; size_t small0, largemax;
void *p; void *p;
/* Get size classes. */ /* Get size classes. */
small0 = get_small_size(0); small0 = get_small_size(0);
hugemax = get_huge_size(get_nhuge()-1); largemax = get_large_size(get_nlarge()-1);
p = mallocx(small0, 0); p = mallocx(small0, 0);
assert_ptr_not_null(p, "Unexpected mallocx() error"); assert_ptr_not_null(p, "Unexpected mallocx() error");
@ -148,13 +148,13 @@ TEST_BEGIN(test_size)
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test largest supported size. */ /* Test largest supported size. */
assert_zu_le(xallocx(p, hugemax, 0, 0), hugemax, assert_zu_le(xallocx(p, largemax, 0, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test size overflow. */ /* Test size overflow. */
assert_zu_le(xallocx(p, hugemax+1, 0, 0), hugemax, assert_zu_le(xallocx(p, largemax+1, 0, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_le(xallocx(p, SIZE_T_MAX, 0, 0), hugemax, assert_zu_le(xallocx(p, SIZE_T_MAX, 0, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
dallocx(p, 0); dallocx(p, 0);
@ -163,30 +163,30 @@ TEST_END
TEST_BEGIN(test_size_extra_overflow) TEST_BEGIN(test_size_extra_overflow)
{ {
size_t small0, hugemax; size_t small0, largemax;
void *p; void *p;
/* Get size classes. */ /* Get size classes. */
small0 = get_small_size(0); small0 = get_small_size(0);
hugemax = get_huge_size(get_nhuge()-1); largemax = get_large_size(get_nlarge()-1);
p = mallocx(small0, 0); p = mallocx(small0, 0);
assert_ptr_not_null(p, "Unexpected mallocx() error"); assert_ptr_not_null(p, "Unexpected mallocx() error");
/* Test overflows that can be resolved by clamping extra. */ /* Test overflows that can be resolved by clamping extra. */
assert_zu_le(xallocx(p, hugemax-1, 2, 0), hugemax, assert_zu_le(xallocx(p, largemax-1, 2, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_le(xallocx(p, hugemax, 1, 0), hugemax, assert_zu_le(xallocx(p, largemax, 1, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test overflow such that hugemax-size underflows. */ /* Test overflow such that largemax-size underflows. */
assert_zu_le(xallocx(p, hugemax+1, 2, 0), hugemax, assert_zu_le(xallocx(p, largemax+1, 2, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_le(xallocx(p, hugemax+2, 3, 0), hugemax, assert_zu_le(xallocx(p, largemax+2, 3, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_le(xallocx(p, SIZE_T_MAX-2, 2, 0), hugemax, assert_zu_le(xallocx(p, SIZE_T_MAX-2, 2, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_le(xallocx(p, SIZE_T_MAX-1, 1, 0), hugemax, assert_zu_le(xallocx(p, SIZE_T_MAX-1, 1, 0), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
dallocx(p, 0); dallocx(p, 0);
@ -195,13 +195,13 @@ TEST_END
TEST_BEGIN(test_extra_small) TEST_BEGIN(test_extra_small)
{ {
size_t small0, small1, hugemax; size_t small0, small1, largemax;
void *p; void *p;
/* Get size classes. */ /* Get size classes. */
small0 = get_small_size(0); small0 = get_small_size(0);
small1 = get_small_size(1); small1 = get_small_size(1);
hugemax = get_huge_size(get_nhuge()-1); largemax = get_large_size(get_nlarge()-1);
p = mallocx(small0, 0); p = mallocx(small0, 0);
assert_ptr_not_null(p, "Unexpected mallocx() error"); assert_ptr_not_null(p, "Unexpected mallocx() error");
@ -216,7 +216,7 @@ TEST_BEGIN(test_extra_small)
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test size+extra overflow. */ /* Test size+extra overflow. */
assert_zu_eq(xallocx(p, small0, hugemax - small0 + 1, 0), small0, assert_zu_eq(xallocx(p, small0, largemax - small0 + 1, 0), small0,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, small0, SIZE_T_MAX - small0, 0), small0, assert_zu_eq(xallocx(p, small0, SIZE_T_MAX - small0, 0), small0,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
@ -225,66 +225,66 @@ TEST_BEGIN(test_extra_small)
} }
TEST_END TEST_END
TEST_BEGIN(test_extra_huge) TEST_BEGIN(test_extra_large)
{ {
int flags = MALLOCX_ARENA(arena_ind()); int flags = MALLOCX_ARENA(arena_ind());
size_t smallmax, huge1, huge2, huge3, hugemax; size_t smallmax, large1, large2, large3, largemax;
void *p; void *p;
/* Get size classes. */ /* Get size classes. */
smallmax = get_small_size(get_nsmall()-1); smallmax = get_small_size(get_nsmall()-1);
huge1 = get_huge_size(1); large1 = get_large_size(1);
huge2 = get_huge_size(2); large2 = get_large_size(2);
huge3 = get_huge_size(3); large3 = get_large_size(3);
hugemax = get_huge_size(get_nhuge()-1); largemax = get_large_size(get_nlarge()-1);
p = mallocx(huge3, flags); p = mallocx(large3, flags);
assert_ptr_not_null(p, "Unexpected mallocx() error"); assert_ptr_not_null(p, "Unexpected mallocx() error");
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3, assert_zu_eq(xallocx(p, large3, 0, flags), large3,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test size decrease with zero extra. */ /* Test size decrease with zero extra. */
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1, assert_zu_ge(xallocx(p, large1, 0, flags), large1,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, smallmax, 0, flags), huge1, assert_zu_ge(xallocx(p, smallmax, 0, flags), large1,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3, assert_zu_eq(xallocx(p, large3, 0, flags), large3,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test size decrease with non-zero extra. */ /* Test size decrease with non-zero extra. */
assert_zu_eq(xallocx(p, huge1, huge3 - huge1, flags), huge3, assert_zu_eq(xallocx(p, large1, large3 - large1, flags), large3,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge2, huge3 - huge2, flags), huge3, assert_zu_eq(xallocx(p, large2, large3 - large2, flags), large3,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge1, huge2 - huge1, flags), huge2, assert_zu_eq(xallocx(p, large1, large2 - large1, flags), large2,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, smallmax, huge1 - smallmax, flags), huge1, assert_zu_ge(xallocx(p, smallmax, large1 - smallmax, flags), large1,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1, assert_zu_ge(xallocx(p, large1, 0, flags), large1,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test size increase with zero extra. */ /* Test size increase with zero extra. */
assert_zu_le(xallocx(p, huge3, 0, flags), huge3, assert_zu_le(xallocx(p, large3, 0, flags), large3,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_le(xallocx(p, hugemax+1, 0, flags), huge3, assert_zu_le(xallocx(p, largemax+1, 0, flags), large3,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1, assert_zu_ge(xallocx(p, large1, 0, flags), large1,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test size increase with non-zero extra. */ /* Test size increase with non-zero extra. */
assert_zu_le(xallocx(p, huge1, SIZE_T_MAX - huge1, flags), hugemax, assert_zu_le(xallocx(p, large1, SIZE_T_MAX - large1, flags), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_ge(xallocx(p, huge1, 0, flags), huge1, assert_zu_ge(xallocx(p, large1, 0, flags), large1,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test size increase with non-zero extra. */ /* Test size increase with non-zero extra. */
assert_zu_le(xallocx(p, huge1, huge3 - huge1, flags), huge3, assert_zu_le(xallocx(p, large1, large3 - large1, flags), large3,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
assert_zu_eq(xallocx(p, huge3, 0, flags), huge3, assert_zu_eq(xallocx(p, large3, 0, flags), large3,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
/* Test size+extra overflow. */ /* Test size+extra overflow. */
assert_zu_le(xallocx(p, huge3, hugemax - huge3 + 1, flags), hugemax, assert_zu_le(xallocx(p, large3, largemax - large3 + 1, flags), largemax,
"Unexpected xallocx() behavior"); "Unexpected xallocx() behavior");
dallocx(p, flags); dallocx(p, flags);
@ -374,15 +374,15 @@ test_zero(size_t szmin, size_t szmax)
dallocx(p, flags); dallocx(p, flags);
} }
TEST_BEGIN(test_zero_huge) TEST_BEGIN(test_zero_large)
{ {
size_t huge0, huge1; size_t large0, large1;
/* Get size classes. */ /* Get size classes. */
huge0 = get_huge_size(0); large0 = get_large_size(0);
huge1 = get_huge_size(1); large1 = get_large_size(1);
test_zero(huge1, huge0 * 2); test_zero(large1, large0 * 2);
} }
TEST_END TEST_END
@ -397,6 +397,6 @@ main(void)
test_size, test_size,
test_size_extra_overflow, test_size_extra_overflow,
test_extra_small, test_extra_small,
test_extra_huge, test_extra_large,
test_zero_huge)); test_zero_large));
} }

20
test/unit/arena_reset.c
View File

@ -25,10 +25,10 @@ get_nsmall(void)
} }
static unsigned static unsigned
get_nhuge(void) get_nlarge(void)
{ {
return (get_nsizes_impl("arenas.nhchunks")); return (get_nsizes_impl("arenas.nlextents"));
} }
static size_t static size_t
@ -58,10 +58,10 @@ get_small_size(size_t ind)
} }
static size_t static size_t
get_huge_size(size_t ind) get_large_size(size_t ind)
{ {
return (get_size_impl("arenas.hchunk.0.size", ind)); return (get_size_impl("arenas.lextent.0.size", ind));
} }
/* Like ivsalloc(), but safe to call on discarded allocations. */ /* Like ivsalloc(), but safe to call on discarded allocations. */
@ -81,8 +81,8 @@ vsalloc(tsdn_t *tsdn, const void *ptr)
TEST_BEGIN(test_arena_reset) TEST_BEGIN(test_arena_reset)
{ {
#define NHUGE 32 #define NLARGE 32
unsigned arena_ind, nsmall, nhuge, nptrs, i; unsigned arena_ind, nsmall, nlarge, nptrs, i;
size_t sz, miblen; size_t sz, miblen;
void **ptrs; void **ptrs;
int flags; int flags;
@ -96,8 +96,8 @@ TEST_BEGIN(test_arena_reset)
flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
nsmall = get_nsmall(); nsmall = get_nsmall();
nhuge = get_nhuge() > NHUGE ? NHUGE : get_nhuge(); nlarge = get_nlarge() > NLARGE ? NLARGE : get_nlarge();
nptrs = nsmall + nhuge; nptrs = nsmall + nlarge;
ptrs = (void **)malloc(nptrs * sizeof(void *)); ptrs = (void **)malloc(nptrs * sizeof(void *));
assert_ptr_not_null(ptrs, "Unexpected malloc() failure"); assert_ptr_not_null(ptrs, "Unexpected malloc() failure");
@ -108,8 +108,8 @@ TEST_BEGIN(test_arena_reset)
assert_ptr_not_null(ptrs[i], assert_ptr_not_null(ptrs[i],
"Unexpected mallocx(%zu, %#x) failure", sz, flags); "Unexpected mallocx(%zu, %#x) failure", sz, flags);
} }
for (i = 0; i < nhuge; i++) { for (i = 0; i < nlarge; i++) {
sz = get_huge_size(i); sz = get_large_size(i);
ptrs[nsmall + i] = mallocx(sz, flags); ptrs[nsmall + i] = mallocx(sz, flags);
assert_ptr_not_null(ptrs[i], assert_ptr_not_null(ptrs[i],
"Unexpected mallocx(%zu, %#x) failure", sz, flags); "Unexpected mallocx(%zu, %#x) failure", sz, flags);

42
test/unit/decay.c
View File

@ -22,7 +22,7 @@ TEST_BEGIN(test_decay_ticks)
{ {
ticker_t *decay_ticker; ticker_t *decay_ticker;
unsigned tick0, tick1; unsigned tick0, tick1;
size_t sz, huge0; size_t sz, large0;
void *p; void *p;
test_skip_if(opt_purge != purge_mode_decay); test_skip_if(opt_purge != purge_mode_decay);
@ -32,18 +32,18 @@ TEST_BEGIN(test_decay_ticks)
"Unexpected failure getting decay ticker"); "Unexpected failure getting decay ticker");
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.lextent.0.size", &large0, &sz, NULL, 0), 0,
"Unexpected mallctl failure"); "Unexpected mallctl failure");
/* /*
* Test the standard APIs using a huge size class, since we can't * Test the standard APIs using a large size class, since we can't
* control tcache interactions for small size classes (except by * control tcache interactions for small size classes (except by
* completely disabling tcache for the entire test program). * completely disabling tcache for the entire test program).
*/ */
/* malloc(). */ /* malloc(). */
tick0 = ticker_read(decay_ticker); tick0 = ticker_read(decay_ticker);
p = malloc(huge0); p = malloc(large0);
assert_ptr_not_null(p, "Unexpected malloc() failure"); assert_ptr_not_null(p, "Unexpected malloc() failure");
tick1 = ticker_read(decay_ticker); tick1 = ticker_read(decay_ticker);
assert_u32_ne(tick1, tick0, "Expected ticker to tick during malloc()"); assert_u32_ne(tick1, tick0, "Expected ticker to tick during malloc()");
@ -55,7 +55,7 @@ TEST_BEGIN(test_decay_ticks)
/* calloc(). */ /* calloc(). */
tick0 = ticker_read(decay_ticker); tick0 = ticker_read(decay_ticker);
p = calloc(1, huge0); p = calloc(1, large0);
assert_ptr_not_null(p, "Unexpected calloc() failure"); assert_ptr_not_null(p, "Unexpected calloc() failure");
tick1 = ticker_read(decay_ticker); tick1 = ticker_read(decay_ticker);
assert_u32_ne(tick1, tick0, "Expected ticker to tick during calloc()"); assert_u32_ne(tick1, tick0, "Expected ticker to tick during calloc()");
@ -63,7 +63,7 @@ TEST_BEGIN(test_decay_ticks)
/* posix_memalign(). */ /* posix_memalign(). */
tick0 = ticker_read(decay_ticker); tick0 = ticker_read(decay_ticker);
assert_d_eq(posix_memalign(&p, sizeof(size_t), huge0), 0, assert_d_eq(posix_memalign(&p, sizeof(size_t), large0), 0,
"Unexpected posix_memalign() failure"); "Unexpected posix_memalign() failure");
tick1 = ticker_read(decay_ticker); tick1 = ticker_read(decay_ticker);
assert_u32_ne(tick1, tick0, assert_u32_ne(tick1, tick0,
@ -72,7 +72,7 @@ TEST_BEGIN(test_decay_ticks)
/* aligned_alloc(). */ /* aligned_alloc(). */
tick0 = ticker_read(decay_ticker); tick0 = ticker_read(decay_ticker);
p = aligned_alloc(sizeof(size_t), huge0); p = aligned_alloc(sizeof(size_t), large0);
assert_ptr_not_null(p, "Unexpected aligned_alloc() failure"); assert_ptr_not_null(p, "Unexpected aligned_alloc() failure");
tick1 = ticker_read(decay_ticker); tick1 = ticker_read(decay_ticker);
assert_u32_ne(tick1, tick0, assert_u32_ne(tick1, tick0,
@ -82,13 +82,13 @@ TEST_BEGIN(test_decay_ticks)
/* realloc(). */ /* realloc(). */
/* Allocate. */ /* Allocate. */
tick0 = ticker_read(decay_ticker); tick0 = ticker_read(decay_ticker);
p = realloc(NULL, huge0); p = realloc(NULL, large0);
assert_ptr_not_null(p, "Unexpected realloc() failure"); assert_ptr_not_null(p, "Unexpected realloc() failure");
tick1 = ticker_read(decay_ticker); tick1 = ticker_read(decay_ticker);
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
/* Reallocate. */ /* Reallocate. */
tick0 = ticker_read(decay_ticker); tick0 = ticker_read(decay_ticker);
p = realloc(p, huge0); p = realloc(p, large0);
assert_ptr_not_null(p, "Unexpected realloc() failure"); assert_ptr_not_null(p, "Unexpected realloc() failure");
tick1 = ticker_read(decay_ticker); tick1 = ticker_read(decay_ticker);
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
@ -99,13 +99,13 @@ TEST_BEGIN(test_decay_ticks)
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
/* /*
* Test the *allocx() APIs using huge and small size classes, with * Test the *allocx() APIs using large and small size classes, with
* tcache explicitly disabled. * tcache explicitly disabled.
*/ */
{ {
unsigned i; unsigned i;
size_t allocx_sizes[2]; size_t allocx_sizes[2];
allocx_sizes[0] = huge0; allocx_sizes[0] = large0;
allocx_sizes[1] = 1; allocx_sizes[1] = 1;
for (i = 0; i < sizeof(allocx_sizes) / sizeof(size_t); i++) { for (i = 0; i < sizeof(allocx_sizes) / sizeof(size_t); i++) {
@ -154,13 +154,13 @@ TEST_BEGIN(test_decay_ticks)
} }
/* /*
* Test tcache fill/flush interactions for huge and small size classes, * Test tcache fill/flush interactions for large and small size classes,
* using an explicit tcache. * using an explicit tcache.
*/ */
if (config_tcache) { if (config_tcache) {
unsigned tcache_ind, i; unsigned tcache_ind, i;
size_t tcache_sizes[2]; size_t tcache_sizes[2];
tcache_sizes[0] = huge0; tcache_sizes[0] = large0;
tcache_sizes[1] = 1; tcache_sizes[1] = 1;
sz = sizeof(unsigned); sz = sizeof(unsigned);
@ -201,14 +201,14 @@ TEST_BEGIN(test_decay_ticker)
uint64_t epoch; uint64_t epoch;
uint64_t npurge0 = 0; uint64_t npurge0 = 0;
uint64_t npurge1 = 0; uint64_t npurge1 = 0;
size_t sz, huge; size_t sz, large;
unsigned i, nupdates0; unsigned i, nupdates0;
nstime_t time, decay_time, deadline; nstime_t time, decay_time, deadline;
test_skip_if(opt_purge != purge_mode_decay); test_skip_if(opt_purge != purge_mode_decay);
/* /*
* Allocate a bunch of huge objects, pause the clock, deallocate the * Allocate a bunch of large objects, pause the clock, deallocate the
* objects, restore the clock, then [md]allocx() in a tight loop to * objects, restore the clock, then [md]allocx() in a tight loop to
* verify the ticker triggers purging. * verify the ticker triggers purging.
*/ */
@ -219,10 +219,10 @@ TEST_BEGIN(test_decay_ticker)
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.tcache_max", &tcache_max, &sz, NULL, assert_d_eq(mallctl("arenas.tcache_max", &tcache_max, &sz, NULL,
0), 0, "Unexpected mallctl failure"); 0), 0, "Unexpected mallctl failure");
huge = nallocx(tcache_max + 1, flags); large = nallocx(tcache_max + 1, flags);
} else { } else {
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge, &sz, NULL, assert_d_eq(mallctl("arenas.lextent.0.size", &large, &sz, NULL,
0), 0, "Unexpected mallctl failure"); 0), 0, "Unexpected mallctl failure");
} }
@ -235,7 +235,7 @@ TEST_BEGIN(test_decay_ticker)
config_stats ? 0 : ENOENT, "Unexpected mallctl result"); config_stats ? 0 : ENOENT, "Unexpected mallctl result");
for (i = 0; i < NPS; i++) { for (i = 0; i < NPS; i++) {
ps[i] = mallocx(huge, flags); ps[i] = mallocx(large, flags);
assert_ptr_not_null(ps[i], "Unexpected mallocx() failure"); assert_ptr_not_null(ps[i], "Unexpected mallocx() failure");
} }
@ -293,13 +293,13 @@ TEST_BEGIN(test_decay_nonmonotonic)
uint64_t epoch; uint64_t epoch;
uint64_t npurge0 = 0; uint64_t npurge0 = 0;
uint64_t npurge1 = 0; uint64_t npurge1 = 0;
size_t sz, huge0; size_t sz, large0;
unsigned i, nupdates0; unsigned i, nupdates0;
test_skip_if(opt_purge != purge_mode_decay); test_skip_if(opt_purge != purge_mode_decay);
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.lextent.0.size", &large0, &sz, NULL, 0), 0,
"Unexpected mallctl failure"); "Unexpected mallctl failure");
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
@ -319,7 +319,7 @@ TEST_BEGIN(test_decay_nonmonotonic)
nstime_update = nstime_update_mock; nstime_update = nstime_update_mock;
for (i = 0; i < NPS; i++) { for (i = 0; i < NPS; i++) {
ps[i] = mallocx(huge0, flags); ps[i] = mallocx(large0, flags);
assert_ptr_not_null(ps[i], "Unexpected mallocx() failure"); assert_ptr_not_null(ps[i], "Unexpected mallocx() failure");
} }

16
test/unit/extent_quantize.c
View File

@ -35,16 +35,16 @@ TEST_BEGIN(test_small_extent_size)
} }
TEST_END TEST_END
TEST_BEGIN(test_huge_extent_size) TEST_BEGIN(test_large_extent_size)
{ {
bool cache_oblivious; bool cache_oblivious;
unsigned nhchunks, i; unsigned nlextents, i;
size_t sz, extent_size_prev, ceil_prev; size_t sz, extent_size_prev, ceil_prev;
size_t mib[4]; size_t mib[4];
size_t miblen = sizeof(mib) / sizeof(size_t); size_t miblen = sizeof(mib) / sizeof(size_t);
/* /*
* Iterate over all huge size classes, get their extent sizes, and * Iterate over all large size classes, get their extent sizes, and
* verify that the quantized size is the same as the extent size. * verify that the quantized size is the same as the extent size.
*/ */
@ -53,12 +53,12 @@ TEST_BEGIN(test_huge_extent_size)
NULL, 0), 0, "Unexpected mallctl failure"); NULL, 0), 0, "Unexpected mallctl failure");
sz = sizeof(unsigned); sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.nhchunks", &nhchunks, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.nlextents", &nlextents, &sz, NULL, 0), 0,
"Unexpected mallctl failure"); "Unexpected mallctl failure");
assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0, assert_d_eq(mallctlnametomib("arenas.lextent.0.size", mib, &miblen), 0,
"Unexpected mallctlnametomib failure"); "Unexpected mallctlnametomib failure");
for (i = 0; i < nhchunks; i++) { for (i = 0; i < nlextents; i++) {
size_t lextent_size, extent_size, floor, ceil; size_t lextent_size, extent_size, floor, ceil;
mib[2] = i; mib[2] = i;
@ -91,7 +91,7 @@ TEST_BEGIN(test_huge_extent_size)
ceil_prev, extent_size); ceil_prev, extent_size);
} }
} }
if (i + 1 < nhchunks) { if (i + 1 < nlextents) {
extent_size_prev = floor; extent_size_prev = floor;
ceil_prev = extent_size_quantize_ceil(extent_size + ceil_prev = extent_size_quantize_ceil(extent_size +
PAGE); PAGE);
@ -141,6 +141,6 @@ main(void)
return (test( return (test(
test_small_extent_size, test_small_extent_size,
test_huge_extent_size, test_large_extent_size,
test_monotonic)); test_monotonic));
} }

16
test/unit/junk.c
View File

@ -9,7 +9,7 @@ const char *malloc_conf =
#endif #endif
static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig; static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig;
static huge_dalloc_junk_t *huge_dalloc_junk_orig; static large_dalloc_junk_t *large_dalloc_junk_orig;
static void *watch_for_junking; static void *watch_for_junking;
static bool saw_junking; static bool saw_junking;
@ -37,10 +37,10 @@ arena_dalloc_junk_small_intercept(void *ptr, const arena_bin_info_t *bin_info)
} }
static void static void
huge_dalloc_junk_intercept(void *ptr, size_t usize) large_dalloc_junk_intercept(void *ptr, size_t usize)
{ {
huge_dalloc_junk_orig(ptr, usize); large_dalloc_junk_orig(ptr, usize);
/* /*
* The conditions under which junk filling actually occurs are nuanced * The conditions under which junk filling actually occurs are nuanced
* enough that it doesn't make sense to duplicate the decision logic in * enough that it doesn't make sense to duplicate the decision logic in
@ -59,8 +59,8 @@ test_junk(size_t sz_min, size_t sz_max)
if (opt_junk_free) { if (opt_junk_free) {
arena_dalloc_junk_small_orig = arena_dalloc_junk_small; arena_dalloc_junk_small_orig = arena_dalloc_junk_small;
arena_dalloc_junk_small = arena_dalloc_junk_small_intercept; arena_dalloc_junk_small = arena_dalloc_junk_small_intercept;
huge_dalloc_junk_orig = huge_dalloc_junk; large_dalloc_junk_orig = large_dalloc_junk;
huge_dalloc_junk = huge_dalloc_junk_intercept; large_dalloc_junk = large_dalloc_junk_intercept;
} }
sz_prev = 0; sz_prev = 0;
@ -110,7 +110,7 @@ test_junk(size_t sz_min, size_t sz_max)
if (opt_junk_free) { if (opt_junk_free) {
arena_dalloc_junk_small = arena_dalloc_junk_small_orig; arena_dalloc_junk_small = arena_dalloc_junk_small_orig;
huge_dalloc_junk = huge_dalloc_junk_orig; large_dalloc_junk = large_dalloc_junk_orig;
} }
} }
@ -122,7 +122,7 @@ TEST_BEGIN(test_junk_small)
} }
TEST_END TEST_END
TEST_BEGIN(test_junk_huge) TEST_BEGIN(test_junk_large)
{ {
test_skip_if(!config_fill); test_skip_if(!config_fill);
@ -136,5 +136,5 @@ main(void)
return (test( return (test(
test_junk_small, test_junk_small,
test_junk_huge)); test_junk_large));
} }

8
test/unit/mallctl.c
View File

@ -596,7 +596,7 @@ TEST_BEGIN(test_arenas_constants)
TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM); TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM);
TEST_ARENAS_CONSTANT(size_t, page, PAGE); TEST_ARENAS_CONSTANT(size_t, page, PAGE);
TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS); TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS);
TEST_ARENAS_CONSTANT(unsigned, nhchunks, NSIZES - NBINS); TEST_ARENAS_CONSTANT(unsigned, nlextents, NSIZES - NBINS);
#undef TEST_ARENAS_CONSTANT #undef TEST_ARENAS_CONSTANT
} }
@ -622,13 +622,13 @@ TEST_BEGIN(test_arenas_bin_constants)
} }
TEST_END TEST_END
TEST_BEGIN(test_arenas_hchunk_constants) TEST_BEGIN(test_arenas_lextent_constants)
{ {
#define TEST_ARENAS_HCHUNK_CONSTANT(t, name, expected) do { \ #define TEST_ARENAS_HCHUNK_CONSTANT(t, name, expected) do { \
t name; \ t name; \
size_t sz = sizeof(t); \ size_t sz = sizeof(t); \
assert_d_eq(mallctl("arenas.hchunk.0."#name, &name, &sz, NULL, \ assert_d_eq(mallctl("arenas.lextent.0."#name, &name, &sz, NULL, \
0), 0, "Unexpected mallctl() failure"); \ 0), 0, "Unexpected mallctl() failure"); \
assert_zu_eq(name, expected, "Incorrect "#name" size"); \ assert_zu_eq(name, expected, "Incorrect "#name" size"); \
} while (0) } while (0)
@ -704,7 +704,7 @@ main(void)
test_arenas_decay_time, test_arenas_decay_time,
test_arenas_constants, test_arenas_constants,
test_arenas_bin_constants, test_arenas_bin_constants,
test_arenas_hchunk_constants, test_arenas_lextent_constants,
test_arenas_extend, test_arenas_extend,
test_stats_arenas)); test_stats_arenas));
} }

8
test/unit/size_classes.c
View File

@ -3,18 +3,18 @@
static size_t static size_t
get_max_size_class(void) get_max_size_class(void)
{ {
unsigned nhchunks; unsigned nlextents;
size_t mib[4]; size_t mib[4];
size_t sz, miblen, max_size_class; size_t sz, miblen, max_size_class;
sz = sizeof(unsigned); sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.nhchunks", &nhchunks, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.nlextents", &nlextents, &sz, NULL, 0), 0,
"Unexpected mallctl() error"); "Unexpected mallctl() error");
miblen = sizeof(mib) / sizeof(size_t); miblen = sizeof(mib) / sizeof(size_t);
assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0, assert_d_eq(mallctlnametomib("arenas.lextent.0.size", mib, &miblen), 0,
"Unexpected mallctlnametomib() error"); "Unexpected mallctlnametomib() error");
mib[2] = nhchunks - 1; mib[2] = nlextents - 1;
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0, assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0,

48
test/unit/stats.c
View File

@ -33,7 +33,7 @@ TEST_BEGIN(test_stats_summary)
} }
TEST_END TEST_END
TEST_BEGIN(test_stats_huge) TEST_BEGIN(test_stats_large)
{ {
void *p; void *p;
uint64_t epoch; uint64_t epoch;
@ -49,14 +49,14 @@ TEST_BEGIN(test_stats_huge)
"Unexpected mallctl() failure"); "Unexpected mallctl() failure");
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("stats.arenas.0.huge.allocated", &allocated, &sz, assert_d_eq(mallctl("stats.arenas.0.large.allocated", &allocated, &sz,
NULL, 0), expected, "Unexpected mallctl() result"); NULL, 0), expected, "Unexpected mallctl() result");
sz = sizeof(uint64_t); sz = sizeof(uint64_t);
assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", &nmalloc, &sz, NULL, assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", &nmalloc, &sz, NULL,
0), expected, "Unexpected mallctl() result"); 0), expected, "Unexpected mallctl() result");
assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", &ndalloc, &sz, NULL, assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", &ndalloc, &sz, NULL,
0), expected, "Unexpected mallctl() result"); 0), expected, "Unexpected mallctl() result");
assert_d_eq(mallctl("stats.arenas.0.huge.nrequests", &nrequests, &sz, assert_d_eq(mallctl("stats.arenas.0.large.nrequests", &nrequests, &sz,
NULL, 0), expected, "Unexpected mallctl() result"); NULL, 0), expected, "Unexpected mallctl() result");
if (config_stats) { if (config_stats) {
@ -75,7 +75,7 @@ TEST_END
TEST_BEGIN(test_stats_arenas_summary) TEST_BEGIN(test_stats_arenas_summary)
{ {
unsigned arena; unsigned arena;
void *little, *huge; void *little, *large;
uint64_t epoch; uint64_t epoch;
size_t sz; size_t sz;
int expected = config_stats ? 0 : ENOENT; int expected = config_stats ? 0 : ENOENT;
@ -88,11 +88,11 @@ TEST_BEGIN(test_stats_arenas_summary)
little = mallocx(SMALL_MAXCLASS, 0); little = mallocx(SMALL_MAXCLASS, 0);
assert_ptr_not_null(little, "Unexpected mallocx() failure"); assert_ptr_not_null(little, "Unexpected mallocx() failure");
huge = mallocx(chunksize, 0); large = mallocx(chunksize, 0);
assert_ptr_not_null(huge, "Unexpected mallocx() failure"); assert_ptr_not_null(large, "Unexpected mallocx() failure");
dallocx(little, 0); dallocx(little, 0);
dallocx(huge, 0); dallocx(large, 0);
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
"Unexpected mallctl() failure"); "Unexpected mallctl() failure");
@ -185,7 +185,7 @@ TEST_BEGIN(test_stats_arenas_small)
} }
TEST_END TEST_END
TEST_BEGIN(test_stats_arenas_huge) TEST_BEGIN(test_stats_arenas_large)
{ {
unsigned arena; unsigned arena;
void *p; void *p;
@ -204,12 +204,12 @@ TEST_BEGIN(test_stats_arenas_huge)
"Unexpected mallctl() failure"); "Unexpected mallctl() failure");
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("stats.arenas.0.huge.allocated", &allocated, &sz, assert_d_eq(mallctl("stats.arenas.0.large.allocated", &allocated, &sz,
NULL, 0), expected, "Unexpected mallctl() result"); NULL, 0), expected, "Unexpected mallctl() result");
sz = sizeof(uint64_t); sz = sizeof(uint64_t);
assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", &nmalloc, &sz, assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", &nmalloc, &sz,
NULL, 0), expected, "Unexpected mallctl() result"); NULL, 0), expected, "Unexpected mallctl() result");
assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", &ndalloc, &sz, assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", &ndalloc, &sz,
NULL, 0), expected, "Unexpected mallctl() result"); NULL, 0), expected, "Unexpected mallctl() result");
if (config_stats) { if (config_stats) {
@ -299,12 +299,12 @@ TEST_BEGIN(test_stats_arenas_bins)
} }
TEST_END TEST_END
TEST_BEGIN(test_stats_arenas_hchunks) TEST_BEGIN(test_stats_arenas_lextents)
{ {
unsigned arena; unsigned arena;
void *p; void *p;
uint64_t epoch, nmalloc, ndalloc; uint64_t epoch, nmalloc, ndalloc;
size_t curhchunks, sz, hsize; size_t curlextents, sz, hsize;
int expected = config_stats ? 0 : ENOENT; int expected = config_stats ? 0 : ENOENT;
arena = 0; arena = 0;
@ -312,7 +312,7 @@ TEST_BEGIN(test_stats_arenas_hchunks)
0, "Unexpected mallctl() failure"); 0, "Unexpected mallctl() failure");
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.hchunk.0.size", &hsize, &sz, NULL, 0), 0, assert_d_eq(mallctl("arenas.lextent.0.size", &hsize, &sz, NULL, 0), 0,
"Unexpected mallctl() failure"); "Unexpected mallctl() failure");
p = mallocx(hsize, 0); p = mallocx(hsize, 0);
@ -322,20 +322,20 @@ TEST_BEGIN(test_stats_arenas_hchunks)
"Unexpected mallctl() failure"); "Unexpected mallctl() failure");
sz = sizeof(uint64_t); sz = sizeof(uint64_t);
assert_d_eq(mallctl("stats.arenas.0.hchunks.0.nmalloc", &nmalloc, &sz, assert_d_eq(mallctl("stats.arenas.0.lextents.0.nmalloc", &nmalloc, &sz,
NULL, 0), expected, "Unexpected mallctl() result"); NULL, 0), expected, "Unexpected mallctl() result");
assert_d_eq(mallctl("stats.arenas.0.hchunks.0.ndalloc", &ndalloc, &sz, assert_d_eq(mallctl("stats.arenas.0.lextents.0.ndalloc", &ndalloc, &sz,
NULL, 0), expected, "Unexpected mallctl() result"); NULL, 0), expected, "Unexpected mallctl() result");
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("stats.arenas.0.hchunks.0.curhchunks", &curhchunks, assert_d_eq(mallctl("stats.arenas.0.lextents.0.curlextents",
&sz, NULL, 0), expected, "Unexpected mallctl() result"); &curlextents, &sz, NULL, 0), expected, "Unexpected mallctl() result");
if (config_stats) { if (config_stats) {
assert_u64_gt(nmalloc, 0, assert_u64_gt(nmalloc, 0,
"nmalloc should be greater than zero"); "nmalloc should be greater than zero");
assert_u64_ge(nmalloc, ndalloc, assert_u64_ge(nmalloc, ndalloc,
"nmalloc should be at least as large as ndalloc"); "nmalloc should be at least as large as ndalloc");
assert_u64_gt(curhchunks, 0, assert_u64_gt(curlextents, 0,
"At least one chunk should be currently allocated"); "At least one chunk should be currently allocated");
} }
@ -349,10 +349,10 @@ main(void)
return (test( return (test(
test_stats_summary, test_stats_summary,
test_stats_huge, test_stats_large,
test_stats_arenas_summary, test_stats_arenas_summary,
test_stats_arenas_small, test_stats_arenas_small,
test_stats_arenas_huge, test_stats_arenas_large,
test_stats_arenas_bins, test_stats_arenas_bins,
test_stats_arenas_hchunks)); test_stats_arenas_lextents));
} }

4
test/unit/zero.c
View File

@ -53,7 +53,7 @@ TEST_BEGIN(test_zero_small)
} }
TEST_END TEST_END
TEST_BEGIN(test_zero_huge) TEST_BEGIN(test_zero_large)
{ {
test_skip_if(!config_fill); test_skip_if(!config_fill);
@ -67,5 +67,5 @@ main(void)
return (test( return (test(
test_zero_small, test_zero_small,
test_zero_huge)); test_zero_large));
} }