project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement opt.cache_oblivious.

Browse Source 
Keep config.cache_oblivious for now to remain backward-compatible.
This commit is contained in:
Qi Wang 2021-02-09 22:24:35 -08:00 committed by Qi Wang
parent 8c5e5f50a2
commit a11be50332
12 changed files with 53 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
INSTALL.md
View File

@ -185,13 +185,13 @@ any of the following arguments (not a definitive list) to 'configure':
* `--disable-cache-oblivious`
Disable cache-oblivious large allocation alignment for large allocation
requests with no alignment constraints. If this feature is disabled, all
large allocations are page-aligned as an implementation artifact, which can
severely harm CPU cache utilization. However, the cache-oblivious layout
comes at the cost of one extra page per large allocation, which in the
most extreme case increases physical memory usage for the 16 KiB size class
to 20 KiB.
Disable cache-oblivious large allocation alignment by default, for large
allocation requests with no alignment constraints. If this feature is
disabled, all large allocations are page-aligned as an implementation
artifact, which can severely harm CPU cache utilization. However, the
cache-oblivious layout comes at the cost of one extra page per large
allocation, which in the most extreme case increases physical memory usage
for the 16 KiB size class to 20 KiB.
* `--disable-syscall`

16
doc/jemalloc.xml.in
View File

@ -936,6 +936,22 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
</para></listitem>
</varlistentry>
<varlistentry id="opt.cache_oblivious">
<term>
<mallctl>opt.cache_oblivious</mallctl>
(<type>bool</type>)
<literal>r-</literal>
</term>
<listitem><para>Enable / Disable cache-oblivious large allocation
alignment, for large requests with no alignment constraints. If this
feature is disabled, all large allocations are page-aligned as an
implementation artifact, which can severely harm CPU cache utilization.
However, the cache-oblivious layout comes at the cost of one extra page
per large allocation, which in the most extreme case increases physical
memory usage for the 16 KiB size class to 20 KiB. This option is enabled
by default.</para></listitem>
</varlistentry>
<varlistentry id="opt.metadata_thp">
<term>
<mallctl>opt.metadata_thp</mallctl>

1
include/jemalloc/internal/jemalloc_internal_externs.h
View File

@ -33,6 +33,7 @@ extern zero_realloc_action_t opt_zero_realloc_action;
extern malloc_init_t malloc_init_state;
extern const char *zero_realloc_mode_names[];
extern atomic_zu_t zero_realloc_count;
extern bool opt_cache_oblivious;
/* Number of CPUs. */
extern unsigned ncpus;

2
include/jemalloc/internal/prof_inlines.h
View File

@ -223,7 +223,7 @@ prof_sample_align(size_t orig_align) {
* w/o metadata lookup.
*/
assert(opt_prof);
return (config_cache_oblivious && orig_align < PAGE) ? PAGE :
return (opt_cache_oblivious && orig_align < PAGE) ? PAGE :
orig_align;
}

14
include/jemalloc/internal/sz.h
View File

@ -45,15 +45,13 @@ extern size_t sz_index2size_tab[SC_NSIZES];
*/
extern uint8_t sz_size2index_tab[];
static const size_t sz_large_pad =
#ifdef JEMALLOC_CACHE_OBLIVIOUS
PAGE
#else
0
#endif
;
/*
* Padding for large allocations: PAGE when opt_cache_oblivious == true (to
* enable cache index randomization); 0 otherwise.
*/
extern size_t sz_large_pad;
extern void sz_boot(const sc_data_t *sc_data);
extern void sz_boot(const sc_data_t *sc_data, bool cache_oblivious);
JEMALLOC_ALWAYS_INLINE pszind_t
sz_psz2ind(size_t psz) {

3
src/ctl.c
View File

@ -90,6 +90,7 @@ CTL_PROTO(config_utrace)
CTL_PROTO(config_xmalloc)
CTL_PROTO(opt_abort)
CTL_PROTO(opt_abort_conf)
CTL_PROTO(opt_cache_oblivious)
CTL_PROTO(opt_trust_madvise)
CTL_PROTO(opt_confirm_conf)
CTL_PROTO(opt_hpa)
@ -395,6 +396,7 @@ static const ctl_named_node_t config_node[] = {
static const ctl_named_node_t opt_node[] = {
{NAME("abort"), CTL(opt_abort)},
{NAME("abort_conf"), CTL(opt_abort_conf)},
{NAME("cache_oblivious"), CTL(opt_cache_oblivious)},
{NAME("trust_madvise"), CTL(opt_trust_madvise)},
{NAME("confirm_conf"), CTL(opt_confirm_conf)},
{NAME("hpa"), CTL(opt_hpa)},
@ -2095,6 +2097,7 @@ CTL_RO_CONFIG_GEN(config_xmalloc, bool)
CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool)
CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool)
CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool)
CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool)
CTL_RO_NL_GEN(opt_hpa, opt_hpa, bool)

15
src/jemalloc.c
View File

@ -102,6 +102,14 @@ bool opt_trust_madvise =
#endif
;
bool opt_cache_oblivious =
#ifdef JEMALLOC_CACHE_OBLIVIOUS
true
#else
false
#endif
;
zero_realloc_action_t opt_zero_realloc_action =
zero_realloc_action_strict;
@ -1697,7 +1705,7 @@ malloc_init_hard_a0_locked() {
prof_boot0();
}
malloc_conf_init(&sc_data, bin_shard_sizes);
sz_boot(&sc_data);
sz_boot(&sc_data, opt_cache_oblivious);
bin_info_boot(&sc_data, bin_shard_sizes);
if (opt_stats_print) {
@ -2790,12 +2798,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
* usize can be trusted to determine szind and slab.
*/
alloc_ctx.szind = sz_size2index(usize);
if (config_cache_oblivious) {
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
} else {
/* Non page aligned must be slab allocated. */
alloc_ctx.slab = true;
}
} else if (opt_prof) {
emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global,
ptr, &alloc_ctx);

3
src/large.c
View File

@ -95,7 +95,8 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
}
if (zero) {
if (config_cache_oblivious) {
if (opt_cache_oblivious) {
assert(sz_large_pad == PAGE);
/*
* Zero the trailing bytes of the original allocation's
* last page, since they are in an indeterminate state.

1
src/stats.c
View File

@ -1458,6 +1458,7 @@ stats_general_print(emitter_t *emitter) {
OPT_WRITE_BOOL("abort")
OPT_WRITE_BOOL("abort_conf")
OPT_WRITE_BOOL("cache_oblivious")
OPT_WRITE_BOOL("confirm_conf")
OPT_WRITE_BOOL("retain")
OPT_WRITE_CHAR_P("dss")

6
src/sz.c
View File

@ -1,9 +1,10 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/sz.h"
JEMALLOC_ALIGNED(CACHELINE)
size_t sz_pind2sz_tab[SC_NPSIZES+1];
size_t sz_large_pad;
size_t
sz_psz_quantize_floor(size_t size) {
@ -105,7 +106,8 @@ sz_boot_size2index_tab(const sc_data_t *sc_data) {
}
void
sz_boot(const sc_data_t *sc_data) {
sz_boot(const sc_data_t *sc_data, bool cache_oblivious) {
sz_large_pad = cache_oblivious ? PAGE : 0;
sz_boot_pind2sz_tab(sc_data);
sz_boot_index2size_tab(sc_data);
sz_boot_size2index_tab(sc_data);

2
test/unit/extent_quantize.c
View File

@ -47,7 +47,7 @@ TEST_BEGIN(test_large_extent_size) {
*/
sz = sizeof(bool);
expect_d_eq(mallctl("config.cache_oblivious", (void *)&cache_oblivious,
expect_d_eq(mallctl("opt.cache_oblivious", (void *)&cache_oblivious,
&sz, NULL, 0), 0, "Unexpected mallctl failure");
sz = sizeof(unsigned);

1
test/unit/mallctl.c
View File

@ -279,6 +279,7 @@ TEST_BEGIN(test_mallctl_opt) {
TEST_MALLCTL_OPT(bool, abort, always);
TEST_MALLCTL_OPT(bool, abort_conf, always);
TEST_MALLCTL_OPT(bool, cache_oblivious, always);
TEST_MALLCTL_OPT(bool, trust_madvise, always);
TEST_MALLCTL_OPT(bool, confirm_conf, always);
TEST_MALLCTL_OPT(const char *, metadata_thp, always);