Add size class computation capability.
Add size class computation capability, currently used only as validation of the size class lookup tables. Generalize the size class spacing used for bins, for eventual use throughout the full range of allocation sizes.
This commit is contained in:
Jason Evans
parent
99118622ff
commit
d04047cc29
23
configure.ac
23
configure.ac
@ -1200,6 +1200,29 @@ if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then
|
||||
JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8)
|
||||
fi
|
||||
|
||||
dnl ============================================================================
|
||||
dnl Check for __builtin_clz() and __builtin_clzl().
|
||||
|
||||
AC_CACHE_CHECK([for __builtin_clz],
|
||||
[je_cv_builtin_clz],
|
||||
[AC_LINK_IFELSE([AC_LANG_PROGRAM([],
|
||||
[
|
||||
{
|
||||
unsigned x = 0;
|
||||
int y = __builtin_clz(x);
|
||||
}
|
||||
{
|
||||
unsigned long x = 0;
|
||||
int y = __builtin_clzl(x);
|
||||
}
|
||||
])],
|
||||
[je_cv_builtin_clz=yes],
|
||||
[je_cv_builtin_clz=no])])
|
||||
|
||||
if test "x${je_cv_builtin_clz}" = "xyes" ; then
|
||||
AC_DEFINE([JEMALLOC_HAVE_BUILTIN_CLZ], [ ])
|
||||
fi
|
||||
|
||||
dnl ============================================================================
|
||||
dnl Check for spinlock(3) operations as provided on Darwin.
|
||||
|
||||
|
||||
@ -463,8 +463,15 @@ void arena_postfork_child(arena_t *arena);
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
size_t small_size2bin_compute(size_t size);
|
||||
size_t small_size2bin_lookup(size_t size);
|
||||
size_t small_size2bin(size_t size);
|
||||
size_t small_bin2size_compute(size_t binind);
|
||||
size_t small_bin2size_lookup(size_t binind);
|
||||
size_t small_bin2size(size_t binind);
|
||||
size_t small_s2u_compute(size_t size);
|
||||
size_t small_s2u_lookup(size_t size);
|
||||
size_t small_s2u(size_t size);
|
||||
arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind);
|
||||
size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind);
|
||||
size_t arena_mapbitsp_read(size_t *mapbitsp);
|
||||
@ -507,18 +514,144 @@ void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache);
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
|
||||
# ifdef JEMALLOC_ARENA_INLINE_A
|
||||
JEMALLOC_INLINE size_t
|
||||
small_size2bin_compute(size_t size)
|
||||
{
|
||||
#if (NTBINS != 0)
|
||||
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
|
||||
size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
|
||||
size_t lg_ceil = lg_floor(pow2_ceil(size));
|
||||
return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
size_t x = lg_floor((size<<1)-1);
|
||||
size_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 :
|
||||
x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM);
|
||||
size_t grp = shift << LG_SIZE_CLASS_GROUP;
|
||||
|
||||
size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1)
|
||||
? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1;
|
||||
|
||||
size_t delta_inverse_mask = ZI(-1) << lg_delta;
|
||||
size_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) &
|
||||
((ZU(1) << LG_SIZE_CLASS_GROUP) - 1);
|
||||
|
||||
size_t bin = NTBINS + grp + mod;
|
||||
return (bin);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
small_size2bin_lookup(size_t size)
|
||||
{
|
||||
|
||||
assert(size <= LOOKUP_MAXCLASS);
|
||||
{
|
||||
size_t ret = ((size_t)(small_size2bin_tab[(size-1) >>
|
||||
LG_TINY_MIN]));
|
||||
assert(ret == small_size2bin_compute(size));
|
||||
return (ret);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
small_size2bin(size_t size)
|
||||
{
|
||||
|
||||
return ((size_t)(small_size2bin_tab[(size-1) >> LG_TINY_MIN]));
|
||||
assert(size > 0);
|
||||
if (size <= LOOKUP_MAXCLASS)
|
||||
return (small_size2bin_lookup(size));
|
||||
else
|
||||
return (small_size2bin_compute(size));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE size_t
|
||||
small_bin2size_compute(size_t binind)
|
||||
{
|
||||
#if (NTBINS > 0)
|
||||
if (binind < NTBINS)
|
||||
return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + binind));
|
||||
else
|
||||
#endif
|
||||
{
|
||||
size_t reduced_binind = binind - NTBINS;
|
||||
size_t grp = reduced_binind >> LG_SIZE_CLASS_GROUP;
|
||||
size_t mod = reduced_binind & ((ZU(1) << LG_SIZE_CLASS_GROUP) -
|
||||
1);
|
||||
|
||||
size_t grp_size_mask = ~((!!grp)-1);
|
||||
size_t grp_size = ((ZU(1) << (LG_QUANTUM +
|
||||
(LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask;
|
||||
|
||||
size_t shift = (grp == 0) ? 1 : grp;
|
||||
size_t lg_delta = shift + (LG_QUANTUM-1);
|
||||
size_t mod_size = (mod+1) << lg_delta;
|
||||
|
||||
size_t usize = grp_size + mod_size;
|
||||
return (usize);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
small_bin2size_lookup(size_t binind)
|
||||
{
|
||||
|
||||
assert(binind < NBINS);
|
||||
{
|
||||
size_t ret = ((size_t)(small_bin2size_tab[binind]));
|
||||
assert(ret == small_bin2size_compute(binind));
|
||||
return (ret);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
small_bin2size(size_t binind)
|
||||
{
|
||||
|
||||
return ((size_t)(small_bin2size_tab[binind]));
|
||||
return (small_bin2size_lookup(binind));
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
small_s2u_compute(size_t size)
|
||||
{
|
||||
#if (NTBINS > 0)
|
||||
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
|
||||
size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
|
||||
size_t lg_ceil = lg_floor(pow2_ceil(size));
|
||||
return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) :
|
||||
(ZU(1) << lg_ceil));
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
size_t x = lg_floor((size<<1)-1);
|
||||
size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1)
|
||||
? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1;
|
||||
size_t delta = ZU(1) << lg_delta;
|
||||
size_t delta_mask = delta - 1;
|
||||
size_t usize = (size + delta_mask) & ~delta_mask;
|
||||
return (usize);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
small_s2u_lookup(size_t size)
|
||||
{
|
||||
size_t ret = (small_bin2size(small_size2bin(size)));
|
||||
|
||||
assert(ret == small_s2u_compute(size));
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
small_s2u(size_t size)
|
||||
{
|
||||
|
||||
assert(size > 0);
|
||||
if (size <= LOOKUP_MAXCLASS)
|
||||
return (small_s2u_lookup(size));
|
||||
else
|
||||
return (small_s2u_compute(size));
|
||||
}
|
||||
# endif /* JEMALLOC_ARENA_INLINE_A */
|
||||
|
||||
|
||||
@ -475,7 +475,7 @@ s2u(size_t size)
|
||||
{
|
||||
|
||||
if (size <= SMALL_MAXCLASS)
|
||||
return (small_bin2size(small_size2bin(size)));
|
||||
return (small_s2u(size));
|
||||
if (size <= arena_maxclass)
|
||||
return (PAGE_CEILING(size));
|
||||
return (CHUNK_CEILING(size));
|
||||
@ -518,7 +518,7 @@ sa2u(size_t size, size_t alignment)
|
||||
|
||||
if (usize <= arena_maxclass && alignment <= PAGE) {
|
||||
if (usize <= SMALL_MAXCLASS)
|
||||
return (small_bin2size(small_size2bin(usize)));
|
||||
return (small_s2u(usize));
|
||||
return (PAGE_CEILING(usize));
|
||||
} else {
|
||||
size_t run_size;
|
||||
|
||||
@ -47,6 +47,11 @@
|
||||
*/
|
||||
#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8
|
||||
|
||||
/*
|
||||
* Defined if __builtin_clz() and __builtin_clzl() are available.
|
||||
*/
|
||||
#undef JEMALLOC_HAVE_BUILTIN_CLZ
|
||||
|
||||
/*
|
||||
* Defined if OSSpin*() functions are available, as provided by Darwin, and
|
||||
* documented in the spinlock(3) manual page.
|
||||
|
||||
@ -39,9 +39,15 @@
|
||||
#endif
|
||||
|
||||
#define ZU(z) ((size_t)z)
|
||||
#define ZI(z) ((ssize_t)z)
|
||||
#define QU(q) ((uint64_t)q)
|
||||
#define QI(q) ((int64_t)q)
|
||||
|
||||
#define KZU(z) ZU(z##ULL)
|
||||
#define KZI(z) ZI(z##ULL)
|
||||
#define KQU(q) QU(q##ULL)
|
||||
#define KQI(q) QI(q##ULL)
|
||||
|
||||
#ifndef __DECONST
|
||||
# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
|
||||
#endif
|
||||
|
||||
@ -234,6 +234,7 @@ ixalloc
|
||||
jemalloc_postfork_child
|
||||
jemalloc_postfork_parent
|
||||
jemalloc_prefork
|
||||
lg_floor
|
||||
malloc_cprintf
|
||||
malloc_mutex_init
|
||||
malloc_mutex_lock
|
||||
@ -348,8 +349,15 @@ s2u
|
||||
sa2u
|
||||
set_errno
|
||||
small_bin2size
|
||||
small_bin2size_compute
|
||||
small_bin2size_lookup
|
||||
small_bin2size_tab
|
||||
small_s2u
|
||||
small_s2u_compute
|
||||
small_s2u_lookup
|
||||
small_size2bin
|
||||
small_size2bin_compute
|
||||
small_size2bin_lookup
|
||||
small_size2bin_tab
|
||||
stats_cactive
|
||||
stats_cactive_add
|
||||
|
||||
@ -2,16 +2,23 @@
|
||||
|
||||
# The following limits are chosen such that they cover all supported platforms.
|
||||
|
||||
# Range of quanta.
|
||||
lg_qmin=3
|
||||
lg_qmax=4
|
||||
# Pointer sizes.
|
||||
lg_zarr="2 3"
|
||||
|
||||
# Quanta.
|
||||
lg_qarr="3 4"
|
||||
|
||||
# The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)].
|
||||
lg_tmin=3
|
||||
|
||||
# Range of page sizes.
|
||||
lg_pmin=12
|
||||
lg_pmax=16
|
||||
# Maximum lookup size.
|
||||
lg_kmax=12
|
||||
|
||||
# Page sizes.
|
||||
lg_parr="12 13 16"
|
||||
|
||||
# Size class group size (number of size classes for each size doubling).
|
||||
lg_g=2
|
||||
|
||||
pow2() {
|
||||
e=$1
|
||||
@ -22,68 +29,206 @@ pow2() {
|
||||
done
|
||||
}
|
||||
|
||||
lg() {
|
||||
x=$1
|
||||
lg_result=0
|
||||
while [ ${x} -gt 1 ] ; do
|
||||
lg_result=$((${lg_result} + 1))
|
||||
x=$((${x} / 2))
|
||||
done
|
||||
}
|
||||
|
||||
size_class() {
|
||||
index=$1
|
||||
lg_grp=$2
|
||||
lg_delta=$3
|
||||
ndelta=$4
|
||||
lg_p=$5
|
||||
lg_kmax=$6
|
||||
|
||||
lg ${ndelta}; lg_ndelta=${lg_result}; pow2 ${lg_ndelta}
|
||||
if [ ${pow2_result} -lt ${ndelta} ] ; then
|
||||
rem="yes"
|
||||
else
|
||||
rem="no"
|
||||
fi
|
||||
|
||||
lg_size=${lg_grp}
|
||||
if [ $((${lg_delta} + ${lg_ndelta})) -eq ${lg_grp} ] ; then
|
||||
lg_size=$((${lg_grp} + 1))
|
||||
else
|
||||
lg_size=${lg_grp}
|
||||
rem="yes"
|
||||
fi
|
||||
|
||||
if [ ${lg_size} -lt ${lg_p} ] ; then
|
||||
bin="yes"
|
||||
else
|
||||
bin="no"
|
||||
fi
|
||||
if [ ${lg_size} -lt ${lg_kmax} \
|
||||
-o ${lg_size} -eq ${lg_kmax} -a ${rem} = "no" ] ; then
|
||||
lg_delta_lookup=${lg_delta}
|
||||
else
|
||||
lg_delta_lookup="no"
|
||||
fi
|
||||
printf ' SC(%3d, %6d, %8d, %6d, %3s, %2s) \\\n' ${index} ${lg_grp} ${lg_delta} ${ndelta} ${bin} ${lg_delta_lookup}
|
||||
# Defined upon return:
|
||||
# - lg_delta_lookup (${lg_delta} or "no")
|
||||
# - bin ("yes" or "no")
|
||||
}
|
||||
|
||||
sep_line() {
|
||||
echo " \\"
|
||||
}
|
||||
|
||||
size_classes() {
|
||||
lg_z=$1
|
||||
lg_q=$2
|
||||
lg_t=$3
|
||||
lg_p=$4
|
||||
lg_g=$5
|
||||
|
||||
pow2 $((${lg_z} + 3)); ptr_bits=${pow2_result}
|
||||
pow2 ${lg_g}; g=${pow2_result}
|
||||
|
||||
echo "#define SIZE_CLASSES \\"
|
||||
echo " /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \\"
|
||||
|
||||
ntbins=0
|
||||
nlbins=0
|
||||
lg_tiny_maxclass='"NA"'
|
||||
nbins=0
|
||||
|
||||
# Tiny size classes.
|
||||
ndelta=0
|
||||
index=0
|
||||
lg_grp=${lg_t}
|
||||
lg_delta=${lg_grp}
|
||||
while [ ${lg_grp} -lt ${lg_q} ] ; do
|
||||
size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax}
|
||||
if [ ${lg_delta_lookup} != "no" ] ; then
|
||||
nlbins=$((${index} + 1))
|
||||
fi
|
||||
if [ ${bin} != "no" ] ; then
|
||||
nbins=$((${index} + 1))
|
||||
fi
|
||||
ntbins=$((${ntbins} + 1))
|
||||
lg_tiny_maxclass=${lg_grp} # Final written value is correct.
|
||||
index=$((${index} + 1))
|
||||
lg_delta=${lg_grp}
|
||||
lg_grp=$((${lg_grp} + 1))
|
||||
done
|
||||
|
||||
# First non-tiny group.
|
||||
if [ ${ntbins} -gt 0 ] ; then
|
||||
sep_line
|
||||
# The first size class has an unusual encoding, because the size has to be
|
||||
# split between grp and delta*ndelta.
|
||||
lg_grp=$((${lg_grp} - 1))
|
||||
ndelta=1
|
||||
size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax}
|
||||
index=$((${index} + 1))
|
||||
lg_grp=$((${lg_grp} + 1))
|
||||
lg_delta=$((${lg_delta} + 1))
|
||||
fi
|
||||
while [ ${ndelta} -lt ${g} ] ; do
|
||||
size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax}
|
||||
index=$((${index} + 1))
|
||||
ndelta=$((${ndelta} + 1))
|
||||
done
|
||||
|
||||
# All remaining groups.
|
||||
lg_grp=$((${lg_grp} + ${lg_g}))
|
||||
while [ ${lg_grp} -lt ${ptr_bits} ] ; do
|
||||
sep_line
|
||||
ndelta=1
|
||||
if [ ${lg_grp} -eq $((${ptr_bits} - 1)) ] ; then
|
||||
ndelta_limit=$((${g} - 1))
|
||||
else
|
||||
ndelta_limit=${g}
|
||||
fi
|
||||
while [ ${ndelta} -le ${ndelta_limit} ] ; do
|
||||
size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax}
|
||||
if [ ${lg_delta_lookup} != "no" ] ; then
|
||||
nlbins=$((${index} + 1))
|
||||
# Final written value is correct:
|
||||
lookup_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))"
|
||||
fi
|
||||
if [ ${bin} != "no" ] ; then
|
||||
nbins=$((${index} + 1))
|
||||
# Final written value is correct:
|
||||
small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))"
|
||||
fi
|
||||
index=$((${index} + 1))
|
||||
ndelta=$((${ndelta} + 1))
|
||||
done
|
||||
lg_grp=$((${lg_grp} + 1))
|
||||
lg_delta=$((${lg_delta} + 1))
|
||||
done
|
||||
echo
|
||||
|
||||
# Defined upon completion:
|
||||
# - ntbins
|
||||
# - nlbins
|
||||
# - nbins
|
||||
# - lg_tiny_maxclass
|
||||
# - lookup_maxclass
|
||||
# - small_maxclass
|
||||
}
|
||||
|
||||
cat <<EOF
|
||||
/* This file was automatically generated by size_classes.sh. */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
/*
|
||||
* This header requires LG_SIZEOF_PTR, LG_TINY_MIN, LG_QUANTUM, and LG_PAGE to
|
||||
* be defined prior to inclusion, and it in turn defines:
|
||||
*
|
||||
* LG_SIZE_CLASS_GROUP: Lg of size class count for each size doubling.
|
||||
* SIZE_CLASSES: Complete table of
|
||||
* SC(index, lg_delta, size, bin, lg_delta_lookup) tuples.
|
||||
* index: Size class index.
|
||||
* lg_grp: Lg group base size (no deltas added).
|
||||
* lg_delta: Lg delta to previous size class.
|
||||
* ndelta: Delta multiplier. size == 1<<lg_grp + ndelta<<lg_delta
|
||||
* bin: 'yes' if a small bin size class, 'no' otherwise.
|
||||
* lg_delta_lookup: Same as lg_delta if a lookup table size class, 'no'
|
||||
* otherwise.
|
||||
* NTBINS: Number of tiny bins.
|
||||
* NLBINS: Number of bins supported by the lookup table.
|
||||
* NBINS: Number of small size class bins.
|
||||
* LG_TINY_MAXCLASS: Lg of maximum tiny size class.
|
||||
* LOOKUP_MAXCLASS: Maximum size class included in lookup table.
|
||||
* SMALL_MAXCLASS: Maximum small size class.
|
||||
*/
|
||||
|
||||
#define LG_SIZE_CLASS_GROUP ${lg_g}
|
||||
|
||||
EOF
|
||||
|
||||
lg_q=${lg_qmin}
|
||||
while [ ${lg_q} -le ${lg_qmax} ] ; do
|
||||
lg_t=${lg_tmin}
|
||||
while [ ${lg_t} -le ${lg_q} ] ; do
|
||||
lg_p=${lg_pmin}
|
||||
while [ ${lg_p} -le ${lg_pmax} ] ; do
|
||||
echo "#if (LG_TINY_MIN == ${lg_t} && LG_QUANTUM == ${lg_q} && LG_PAGE == ${lg_p})"
|
||||
echo "#define SIZE_CLASSES_DEFINED"
|
||||
pow2 ${lg_q}; q=${pow2_result}
|
||||
pow2 ${lg_t}; t=${pow2_result}
|
||||
pow2 ${lg_p}; p=${pow2_result}
|
||||
bin=0
|
||||
psz=0
|
||||
sz=${t}
|
||||
delta=$((${sz} - ${psz}))
|
||||
echo "/* SIZE_CLASS(bin, delta, sz) */"
|
||||
echo "#define SIZE_CLASSES \\"
|
||||
|
||||
# Tiny size classes.
|
||||
while [ ${sz} -lt ${q} ] ; do
|
||||
echo " SIZE_CLASS(${bin}, ${delta}, ${sz}) \\"
|
||||
bin=$((${bin} + 1))
|
||||
psz=${sz}
|
||||
sz=$((${sz} + ${sz}))
|
||||
delta=$((${sz} - ${psz}))
|
||||
for lg_z in ${lg_zarr} ; do
|
||||
for lg_q in ${lg_qarr} ; do
|
||||
lg_t=${lg_tmin}
|
||||
while [ ${lg_t} -le ${lg_q} ] ; do
|
||||
# Iterate through page sizes and compute how many bins there are.
|
||||
for lg_p in ${lg_parr} ; do
|
||||
echo "#if (LG_SIZEOF_PTR == ${lg_z} && LG_TINY_MIN == ${lg_t} && LG_QUANTUM == ${lg_q} && LG_PAGE == ${lg_p})"
|
||||
size_classes ${lg_z} ${lg_q} ${lg_t} ${lg_p} ${lg_g}
|
||||
echo "#define SIZE_CLASSES_DEFINED"
|
||||
echo "#define NTBINS ${ntbins}"
|
||||
echo "#define NLBINS ${nlbins}"
|
||||
echo "#define NBINS ${nbins}"
|
||||
echo "#define LG_TINY_MAXCLASS ${lg_tiny_maxclass}"
|
||||
echo "#define LOOKUP_MAXCLASS ${lookup_maxclass}"
|
||||
echo "#define SMALL_MAXCLASS ${small_maxclass}"
|
||||
echo "#endif"
|
||||
echo
|
||||
done
|
||||
# Quantum-multiple size classes. For each doubling of sz, as many as 4
|
||||
# size classes exist. Their spacing is the greater of:
|
||||
# - q
|
||||
# - sz/4, where sz is a power of 2
|
||||
while [ ${sz} -lt ${p} ] ; do
|
||||
if [ ${sz} -ge $((${q} * 4)) ] ; then
|
||||
i=$((${sz} / 4))
|
||||
else
|
||||
i=${q}
|
||||
fi
|
||||
next_2pow=$((${sz} * 2))
|
||||
while [ ${sz} -lt $next_2pow ] ; do
|
||||
echo " SIZE_CLASS(${bin}, ${delta}, ${sz}) \\"
|
||||
bin=$((${bin} + 1))
|
||||
psz=${sz}
|
||||
sz=$((${sz} + ${i}))
|
||||
delta=$((${sz} - ${psz}))
|
||||
done
|
||||
done
|
||||
echo
|
||||
echo "#define NBINS ${bin}"
|
||||
echo "#define SMALL_MAXCLASS ${psz}"
|
||||
echo "#endif"
|
||||
echo
|
||||
lg_p=$((${lg_p} + 1))
|
||||
lg_t=$((${lg_t} + 1))
|
||||
done
|
||||
lg_t=$((${lg_t} + 1))
|
||||
done
|
||||
lg_q=$((${lg_q} + 1))
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
|
||||
@ -110,6 +110,7 @@ void malloc_printf(const char *format, ...)
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
size_t pow2_ceil(size_t x);
|
||||
size_t lg_floor(size_t x);
|
||||
void set_errno(int errnum);
|
||||
int get_errno(void);
|
||||
#endif
|
||||
@ -133,6 +134,52 @@ pow2_ceil(size_t x)
|
||||
return (x);
|
||||
}
|
||||
|
||||
#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
|
||||
JEMALLOC_INLINE size_t
|
||||
lg_floor(size_t x)
|
||||
{
|
||||
size_t ret;
|
||||
|
||||
asm ("bsr %1, %0"
|
||||
: "=r"(ret) // Outputs.
|
||||
: "r"(x) // Inputs.
|
||||
);
|
||||
return (ret);
|
||||
}
|
||||
#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
|
||||
JEMALLOC_INLINE size_t
|
||||
lg_floor(size_t x)
|
||||
{
|
||||
|
||||
#if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
|
||||
return ((8 << LG_SIZEOF_PTR - 1) - __builtin_clz(x));
|
||||
#elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
|
||||
return ((8 << LG_SIZEOF_PTR - 1) - __builtin_clzl(x));
|
||||
#else
|
||||
# error "Unsupported type sizes for lg_floor()"
|
||||
#endif
|
||||
}
|
||||
#else
|
||||
JEMALLOC_INLINE size_t
|
||||
lg_floor(size_t x)
|
||||
{
|
||||
|
||||
x |= (x >> 1);
|
||||
x |= (x >> 2);
|
||||
x |= (x >> 4);
|
||||
x |= (x >> 8);
|
||||
x |= (x >> 16);
|
||||
#if (LG_SIZEOF_PTR == 3 && LG_SIZEOF_PTR == LG_SIZEOF_LONG)
|
||||
x |= (x >> 32);
|
||||
return (65 - ffsl(~x));
|
||||
#elif (LG_SIZEOF_PTR == 2)
|
||||
return (33 - ffs(~x));
|
||||
#else
|
||||
# error "Unsupported type sizes for lg_floor()"
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Sets error code */
|
||||
JEMALLOC_INLINE void
|
||||
set_errno(int errnum)
|
||||
|
||||
62
src/arena.c
62
src/arena.c
@ -9,40 +9,39 @@ arena_bin_info_t arena_bin_info[NBINS];
|
||||
|
||||
JEMALLOC_ALIGNED(CACHELINE)
|
||||
const uint32_t small_bin2size_tab[NBINS] = {
|
||||
#define SIZE_CLASS(bin, delta, size) \
|
||||
#define B2S_bin_yes(size) \
|
||||
size,
|
||||
#define B2S_bin_no(size)
|
||||
#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
|
||||
B2S_bin_##bin((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
|
||||
SIZE_CLASSES
|
||||
#undef SIZE_CLASS
|
||||
#undef B2S_bin_yes
|
||||
#undef B2S_bin_no
|
||||
#undef SC
|
||||
};
|
||||
|
||||
JEMALLOC_ALIGNED(CACHELINE)
|
||||
const uint8_t small_size2bin_tab[] = {
|
||||
#define S2B_8(i) i,
|
||||
#define S2B_16(i) S2B_8(i) S2B_8(i)
|
||||
#define S2B_32(i) S2B_16(i) S2B_16(i)
|
||||
#define S2B_64(i) S2B_32(i) S2B_32(i)
|
||||
#define S2B_128(i) S2B_64(i) S2B_64(i)
|
||||
#define S2B_256(i) S2B_128(i) S2B_128(i)
|
||||
#define S2B_512(i) S2B_256(i) S2B_256(i)
|
||||
#define S2B_1024(i) S2B_512(i) S2B_512(i)
|
||||
#define S2B_2048(i) S2B_1024(i) S2B_1024(i)
|
||||
#define S2B_4096(i) S2B_2048(i) S2B_2048(i)
|
||||
#define S2B_8192(i) S2B_4096(i) S2B_4096(i)
|
||||
#define SIZE_CLASS(bin, delta, size) \
|
||||
S2B_##delta(bin)
|
||||
#define S2B_3(i) i,
|
||||
#define S2B_4(i) S2B_3(i) S2B_3(i)
|
||||
#define S2B_5(i) S2B_4(i) S2B_4(i)
|
||||
#define S2B_6(i) S2B_5(i) S2B_5(i)
|
||||
#define S2B_7(i) S2B_6(i) S2B_6(i)
|
||||
#define S2B_8(i) S2B_7(i) S2B_7(i)
|
||||
#define S2B_9(i) S2B_8(i) S2B_8(i)
|
||||
#define S2B_no(i)
|
||||
#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
|
||||
S2B_##lg_delta_lookup(index)
|
||||
SIZE_CLASSES
|
||||
#undef S2B_3
|
||||
#undef S2B_4
|
||||
#undef S2B_5
|
||||
#undef S2B_6
|
||||
#undef S2B_7
|
||||
#undef S2B_8
|
||||
#undef S2B_16
|
||||
#undef S2B_32
|
||||
#undef S2B_64
|
||||
#undef S2B_128
|
||||
#undef S2B_256
|
||||
#undef S2B_512
|
||||
#undef S2B_1024
|
||||
#undef S2B_2048
|
||||
#undef S2B_4096
|
||||
#undef S2B_8192
|
||||
#undef SIZE_CLASS
|
||||
#undef S2B_9
|
||||
#undef S2B_no
|
||||
#undef SC
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
@ -2586,13 +2585,18 @@ bin_info_init(void)
|
||||
arena_bin_info_t *bin_info;
|
||||
size_t prev_run_size = PAGE;
|
||||
|
||||
#define SIZE_CLASS(bin, delta, size) \
|
||||
bin_info = &arena_bin_info[bin]; \
|
||||
#define BIN_INFO_INIT_bin_yes(index, size) \
|
||||
bin_info = &arena_bin_info[index]; \
|
||||
bin_info->reg_size = size; \
|
||||
prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\
|
||||
bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs);
|
||||
#define BIN_INFO_INIT_bin_no(index, size)
|
||||
#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
|
||||
BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
|
||||
SIZE_CLASSES
|
||||
#undef SIZE_CLASS
|
||||
#undef BIN_INFO_INIT_bin_yes
|
||||
#undef BIN_INFO_INIT_bin_no
|
||||
#undef SC
|
||||
}
|
||||
|
||||
void
|
||||
|
||||
Loading…
Reference in New Issue
Block a user