HPA: Introduce a redesigned hpa_central_t.
For now, this only handles allocating virtual address space to shards, with no reuse. This is framework, though; it will change over time.
This commit is contained in:
parent
e09eac1d4e
commit
d93eef2f40
@ -99,7 +99,7 @@ bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
|
||||
unsigned *binshard);
|
||||
size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
|
||||
void **ptrs, size_t nfill, bool zero);
|
||||
void arena_boot(sc_data_t *sc_data);
|
||||
bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
|
||||
void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
|
||||
void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
|
||||
void arena_prefork2(tsdn_t *tsdn, arena_t *arena);
|
||||
|
@ -7,6 +7,37 @@
|
||||
#include "jemalloc/internal/pai.h"
|
||||
#include "jemalloc/internal/psset.h"
|
||||
|
||||
typedef struct hpa_central_s hpa_central_t;
|
||||
struct hpa_central_s {
|
||||
/*
|
||||
* The mutex guarding most of the operations on the central data
|
||||
* structure.
|
||||
*/
|
||||
malloc_mutex_t mtx;
|
||||
/*
|
||||
* Guards expansion of eden. We separate this from the regular mutex so
|
||||
* that cheaper operations can still continue while we're doing the OS
|
||||
* call.
|
||||
*/
|
||||
malloc_mutex_t grow_mtx;
|
||||
/*
|
||||
* Either NULL (if empty), or some integer multiple of a
|
||||
* hugepage-aligned number of hugepages. We carve them off one at a
|
||||
* time to satisfy new pageslab requests.
|
||||
*
|
||||
* Guarded by grow_mtx.
|
||||
*/
|
||||
void *eden;
|
||||
size_t eden_len;
|
||||
/* Source for metadata. */
|
||||
base_t *base;
|
||||
/* Number of grow operations done on this hpa_central_t. */
|
||||
uint64_t age_counter;
|
||||
|
||||
/* The HPA hooks. */
|
||||
hpa_hooks_t hooks;
|
||||
};
|
||||
|
||||
typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
|
||||
struct hpa_shard_nonderived_stats_s {
|
||||
/*
|
||||
@ -52,19 +83,20 @@ struct hpa_shard_s {
|
||||
* pointer to the hpa_shard_t.
|
||||
*/
|
||||
pai_t pai;
|
||||
malloc_mutex_t grow_mtx;
|
||||
|
||||
/* The central allocator we get our hugepages from. */
|
||||
hpa_central_t *central;
|
||||
/* Protects most of this shard's state. */
|
||||
malloc_mutex_t mtx;
|
||||
/*
|
||||
* Guards the shard's access to the central allocator (preventing
|
||||
* multiple threads operating on this shard from accessing the central
|
||||
* allocator).
|
||||
*/
|
||||
malloc_mutex_t grow_mtx;
|
||||
/* The base metadata allocator. */
|
||||
base_t *base;
|
||||
|
||||
/*
|
||||
* The HPA hooks for this shard. Eventually, once we have the
|
||||
* hpa_central_t back, these should live there (since it doesn't make
|
||||
* sense for different shards on the same hpa_central_t to have
|
||||
* different hooks).
|
||||
*/
|
||||
hpa_hooks_t hooks;
|
||||
|
||||
/*
|
||||
* This edata cache is the one we use when allocating a small extent
|
||||
* from a pageslab. The pageslab itself comes from the centralized
|
||||
@ -81,18 +113,13 @@ struct hpa_shard_s {
|
||||
*/
|
||||
uint64_t age_counter;
|
||||
|
||||
/*
|
||||
* Either NULL (if empty), or some integer multiple of a
|
||||
* hugepage-aligned number of hugepages. We carve them off one at a
|
||||
* time to satisfy new pageslab requests.
|
||||
*
|
||||
* Guarded by grow_mtx.
|
||||
*/
|
||||
void *eden;
|
||||
size_t eden_len;
|
||||
|
||||
/* The arena ind we're associated with. */
|
||||
unsigned ind;
|
||||
|
||||
/*
|
||||
* Our emap. This is just a cache of the emap pointer in the associated
|
||||
* hpa_central.
|
||||
*/
|
||||
emap_t *emap;
|
||||
|
||||
/* The configuration choices for this hpa shard. */
|
||||
@ -117,8 +144,9 @@ struct hpa_shard_s {
|
||||
* just that it can function properly given the system it's running on.
|
||||
*/
|
||||
bool hpa_supported();
|
||||
bool hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
|
||||
edata_cache_t *edata_cache, unsigned ind, const hpa_hooks_t *hooks,
|
||||
bool hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
|
||||
bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
|
||||
base_t *base, edata_cache_t *edata_cache, unsigned ind,
|
||||
const hpa_shard_opts_t *opts);
|
||||
|
||||
void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
|
||||
|
@ -20,6 +20,11 @@
|
||||
* others will be coming soon.
|
||||
*/
|
||||
|
||||
typedef struct pa_central_s pa_central_t;
|
||||
struct pa_central_s {
|
||||
hpa_central_t hpa;
|
||||
};
|
||||
|
||||
/*
|
||||
* The stats for a particular pa_shard. Because of the way the ctl module
|
||||
* handles stats epoch data collection (it has its own arena_stats, and merges
|
||||
@ -61,6 +66,9 @@ struct pa_shard_stats_s {
|
||||
*/
|
||||
typedef struct pa_shard_s pa_shard_t;
|
||||
struct pa_shard_s {
|
||||
/* The central PA this shard is associated with. */
|
||||
pa_central_t *central;
|
||||
|
||||
/*
|
||||
* Number of pages in active extents.
|
||||
*
|
||||
@ -76,6 +84,7 @@ struct pa_shard_s {
|
||||
* for those allocations.
|
||||
*/
|
||||
atomic_b_t use_hpa;
|
||||
|
||||
/*
|
||||
* If we never used the HPA to begin with, it wasn't initialized, and so
|
||||
* we shouldn't try to e.g. acquire its mutexes during fork. This
|
||||
@ -121,18 +130,21 @@ pa_shard_ehooks_get(pa_shard_t *shard) {
|
||||
}
|
||||
|
||||
/* Returns true on error. */
|
||||
bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
||||
unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
|
||||
nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
|
||||
ssize_t muzzy_decay_ms);
|
||||
bool pa_central_init(pa_central_t *central, base_t *base, bool hpa,
|
||||
hpa_hooks_t *hpa_hooks);
|
||||
|
||||
/* Returns true on error. */
|
||||
bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
|
||||
emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats,
|
||||
malloc_mutex_t *stats_mtx, nstime_t *cur_time, size_t oversize_threshold,
|
||||
ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms);
|
||||
|
||||
/*
|
||||
* This isn't exposed to users; we allow late enablement of the HPA shard so
|
||||
* that we can boot without worrying about the HPA, then turn it on in a0.
|
||||
*/
|
||||
bool pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
|
||||
const hpa_hooks_t *hpa_hooks, const hpa_shard_opts_t *hpa_opts,
|
||||
const sec_opts_t *hpa_sec_opts);
|
||||
const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts);
|
||||
|
||||
/*
|
||||
* We stop using the HPA when custom extent hooks are installed, but still
|
||||
|
@ -52,8 +52,8 @@ enum witness_rank_e {
|
||||
WITNESS_RANK_EXTENTS,
|
||||
WITNESS_RANK_HPA_SHARD = WITNESS_RANK_EXTENTS,
|
||||
|
||||
WITNESS_RANK_HPA_GROW,
|
||||
WITNESS_RANK_HPA,
|
||||
WITNESS_RANK_HPA_CENTRAL_GROW,
|
||||
WITNESS_RANK_HPA_CENTRAL,
|
||||
|
||||
WITNESS_RANK_EDATA_CACHE,
|
||||
|
||||
|
16
src/arena.c
16
src/arena.c
@ -36,6 +36,7 @@ static atomic_zd_t dirty_decay_ms_default;
|
||||
static atomic_zd_t muzzy_decay_ms_default;
|
||||
|
||||
emap_t arena_emap_global;
|
||||
pa_central_t arena_pa_central_global;
|
||||
|
||||
const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = {
|
||||
#define STEP(step, h, x, y) \
|
||||
@ -1541,9 +1542,10 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
|
||||
|
||||
nstime_t cur_time;
|
||||
nstime_init_update(&cur_time);
|
||||
if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind,
|
||||
&arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx),
|
||||
&cur_time, oversize_threshold, arena_dirty_decay_ms_default_get(),
|
||||
if (pa_shard_init(tsdn, &arena->pa_shard, &arena_pa_central_global,
|
||||
&arena_emap_global, base, ind, &arena->stats.pa_shard_stats,
|
||||
LOCKEDINT_MTX(arena->stats.mtx), &cur_time, oversize_threshold,
|
||||
arena_dirty_decay_ms_default_get(),
|
||||
arena_muzzy_decay_ms_default_get())) {
|
||||
goto label_error;
|
||||
}
|
||||
@ -1575,7 +1577,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
|
||||
hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
|
||||
hpa_shard_opts.deferral_allowed = background_thread_enabled();
|
||||
if (pa_shard_enable_hpa(tsdn, &arena->pa_shard,
|
||||
&hpa_hooks_default, &hpa_shard_opts, &opt_hpa_sec_opts)) {
|
||||
&hpa_shard_opts, &opt_hpa_sec_opts)) {
|
||||
goto label_error;
|
||||
}
|
||||
}
|
||||
@ -1664,8 +1666,8 @@ arena_is_huge(unsigned arena_ind) {
|
||||
return (arena_ind == huge_arena_ind);
|
||||
}
|
||||
|
||||
void
|
||||
arena_boot(sc_data_t *sc_data) {
|
||||
bool
|
||||
arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) {
|
||||
arena_dirty_decay_ms_default_set(opt_dirty_decay_ms);
|
||||
arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms);
|
||||
for (unsigned i = 0; i < SC_NBINS; i++) {
|
||||
@ -1680,6 +1682,8 @@ arena_boot(sc_data_t *sc_data) {
|
||||
nbins_total += bin_infos[i].n_shards;
|
||||
cur_offset += (uint32_t)(bin_infos[i].n_shards * sizeof(bin_t));
|
||||
}
|
||||
return pa_central_init(&arena_pa_central_global, base, hpa,
|
||||
&hpa_hooks_default);
|
||||
}
|
||||
|
||||
void
|
||||
|
205
src/hpa.c
205
src/hpa.c
@ -51,9 +51,125 @@ hpa_supported() {
|
||||
}
|
||||
|
||||
bool
|
||||
hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
|
||||
edata_cache_t *edata_cache, unsigned ind,
|
||||
const hpa_hooks_t *hooks, const hpa_shard_opts_t *opts) {
|
||||
hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) {
|
||||
/* malloc_conf processing should have filtered out these cases. */
|
||||
assert(hpa_supported());
|
||||
bool err;
|
||||
err = malloc_mutex_init(¢ral->grow_mtx, "hpa_central_grow",
|
||||
WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive);
|
||||
if (err) {
|
||||
return true;
|
||||
}
|
||||
err = malloc_mutex_init(¢ral->mtx, "hpa_central",
|
||||
WITNESS_RANK_HPA_CENTRAL, malloc_mutex_rank_exclusive);
|
||||
if (err) {
|
||||
return true;
|
||||
}
|
||||
central->base = base;
|
||||
central->eden = NULL;
|
||||
central->eden_len = 0;
|
||||
central->age_counter = 0;
|
||||
central->hooks = *hooks;
|
||||
return false;
|
||||
}
|
||||
|
||||
static hpdata_t *
|
||||
hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) {
|
||||
return (hpdata_t *)base_alloc(tsdn, central->base, sizeof(hpdata_t),
|
||||
CACHELINE);
|
||||
}
|
||||
|
||||
hpdata_t *
|
||||
hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
|
||||
bool *oom) {
|
||||
/* Don't yet support big allocations; these should get filtered out. */
|
||||
assert(size <= HUGEPAGE);
|
||||
/*
|
||||
* Should only try to extract from the central allocator if the local
|
||||
* shard is exhausted. We should hold the grow_mtx on that shard.
|
||||
*/
|
||||
witness_assert_positive_depth_to_rank(
|
||||
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW);
|
||||
|
||||
malloc_mutex_lock(tsdn, ¢ral->grow_mtx);
|
||||
*oom = false;
|
||||
|
||||
hpdata_t *ps = NULL;
|
||||
|
||||
/* Is eden a perfect fit? */
|
||||
if (central->eden != NULL && central->eden_len == HUGEPAGE) {
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
hpdata_init(ps, central->eden, central->age_counter++);
|
||||
central->eden = NULL;
|
||||
central->eden_len = 0;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return ps;
|
||||
}
|
||||
|
||||
/*
|
||||
* We're about to try to allocate from eden by splitting. If eden is
|
||||
* NULL, we have to allocate it too. Otherwise, we just have to
|
||||
* allocate an edata_t for the new psset.
|
||||
*/
|
||||
if (central->eden == NULL) {
|
||||
/*
|
||||
* During development, we're primarily concerned with systems
|
||||
* with overcommit. Eventually, we should be more careful here.
|
||||
*/
|
||||
bool commit = true;
|
||||
/* Allocate address space, bailing if we fail. */
|
||||
void *new_eden = pages_map(NULL, HPA_EDEN_SIZE, HUGEPAGE,
|
||||
&commit);
|
||||
if (new_eden == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
pages_unmap(new_eden, HPA_EDEN_SIZE);
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
central->eden = new_eden;
|
||||
central->eden_len = HPA_EDEN_SIZE;
|
||||
} else {
|
||||
/* Eden is already nonempty; only need an edata for ps. */
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
assert(ps != NULL);
|
||||
assert(central->eden != NULL);
|
||||
assert(central->eden_len > HUGEPAGE);
|
||||
assert(central->eden_len % HUGEPAGE == 0);
|
||||
assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
|
||||
|
||||
hpdata_init(ps, central->eden, central->age_counter++);
|
||||
|
||||
char *eden_char = (char *)central->eden;
|
||||
eden_char += HUGEPAGE;
|
||||
central->eden = (void *)eden_char;
|
||||
central->eden_len -= HUGEPAGE;
|
||||
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
|
||||
return ps;
|
||||
}
|
||||
|
||||
bool
|
||||
hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
|
||||
base_t *base, edata_cache_t *edata_cache, unsigned ind,
|
||||
const hpa_shard_opts_t *opts) {
|
||||
/* malloc_conf processing should have filtered out these cases. */
|
||||
assert(hpa_supported());
|
||||
bool err;
|
||||
@ -69,13 +185,11 @@ hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
|
||||
}
|
||||
|
||||
assert(edata_cache != NULL);
|
||||
shard->central = central;
|
||||
shard->base = base;
|
||||
shard->hooks = *hooks;
|
||||
edata_cache_small_init(&shard->ecs, edata_cache);
|
||||
psset_init(&shard->psset);
|
||||
shard->age_counter = 0;
|
||||
shard->eden = NULL;
|
||||
shard->eden_len = 0;
|
||||
shard->ind = ind;
|
||||
shard->emap = emap;
|
||||
|
||||
@ -136,12 +250,6 @@ hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
|
||||
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
|
||||
}
|
||||
|
||||
static hpdata_t *
|
||||
hpa_alloc_ps(tsdn_t *tsdn, hpa_shard_t *shard) {
|
||||
return (hpdata_t *)base_alloc(tsdn, shard->base, sizeof(hpdata_t),
|
||||
CACHELINE);
|
||||
}
|
||||
|
||||
static bool
|
||||
hpa_good_hugification_candidate(hpa_shard_t *shard, hpdata_t *ps) {
|
||||
/*
|
||||
@ -227,7 +335,7 @@ hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard,
|
||||
if (hpa_good_hugification_candidate(shard, ps)
|
||||
&& !hpdata_huge_get(ps)) {
|
||||
nstime_t now;
|
||||
shard->hooks.curtime(&now);
|
||||
shard->central->hooks.curtime(&now);
|
||||
hpdata_allow_hugify(ps, now);
|
||||
}
|
||||
/*
|
||||
@ -247,64 +355,6 @@ hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard,
|
||||
}
|
||||
}
|
||||
|
||||
static hpdata_t *
|
||||
hpa_grow(tsdn_t *tsdn, hpa_shard_t *shard) {
|
||||
malloc_mutex_assert_owner(tsdn, &shard->grow_mtx);
|
||||
hpdata_t *ps = NULL;
|
||||
|
||||
/* Is eden a perfect fit? */
|
||||
if (shard->eden != NULL && shard->eden_len == HUGEPAGE) {
|
||||
ps = hpa_alloc_ps(tsdn, shard);
|
||||
if (ps == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
hpdata_init(ps, shard->eden, shard->age_counter++);
|
||||
shard->eden = NULL;
|
||||
shard->eden_len = 0;
|
||||
return ps;
|
||||
}
|
||||
|
||||
/*
|
||||
* We're about to try to allocate from eden by splitting. If eden is
|
||||
* NULL, we have to allocate it too. Otherwise, we just have to
|
||||
* allocate an edata_t for the new psset.
|
||||
*/
|
||||
if (shard->eden == NULL) {
|
||||
/* Allocate address space, bailing if we fail. */
|
||||
void *new_eden = shard->hooks.map(HPA_EDEN_SIZE);
|
||||
if (new_eden == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
ps = hpa_alloc_ps(tsdn, shard);
|
||||
if (ps == NULL) {
|
||||
shard->hooks.unmap(new_eden, HPA_EDEN_SIZE);
|
||||
return NULL;
|
||||
}
|
||||
shard->eden = new_eden;
|
||||
shard->eden_len = HPA_EDEN_SIZE;
|
||||
} else {
|
||||
/* Eden is already nonempty; only need an edata for ps. */
|
||||
ps = hpa_alloc_ps(tsdn, shard);
|
||||
if (ps == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
assert(ps != NULL);
|
||||
assert(shard->eden != NULL);
|
||||
assert(shard->eden_len > HUGEPAGE);
|
||||
assert(shard->eden_len % HUGEPAGE == 0);
|
||||
assert(HUGEPAGE_ADDR2BASE(shard->eden) == shard->eden);
|
||||
|
||||
hpdata_init(ps, shard->eden, shard->age_counter++);
|
||||
|
||||
char *eden_char = (char *)shard->eden;
|
||||
eden_char += HUGEPAGE;
|
||||
shard->eden = (void *)eden_char;
|
||||
shard->eden_len -= HUGEPAGE;
|
||||
|
||||
return ps;
|
||||
}
|
||||
|
||||
/* Returns whether or not we purged anything. */
|
||||
static bool
|
||||
hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
|
||||
@ -348,7 +398,8 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
|
||||
|
||||
/* Actually do the purging, now that the lock is dropped. */
|
||||
if (dehugify) {
|
||||
shard->hooks.dehugify(hpdata_addr_get(to_purge), HUGEPAGE);
|
||||
shard->central->hooks.dehugify(hpdata_addr_get(to_purge),
|
||||
HUGEPAGE);
|
||||
}
|
||||
size_t total_purged = 0;
|
||||
uint64_t purges_this_pass = 0;
|
||||
@ -359,7 +410,7 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
|
||||
total_purged += purge_size;
|
||||
assert(total_purged <= HUGEPAGE);
|
||||
purges_this_pass++;
|
||||
shard->hooks.purge(purge_addr, purge_size);
|
||||
shard->central->hooks.purge(purge_addr, purge_size);
|
||||
}
|
||||
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
@ -406,7 +457,7 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) {
|
||||
/* Make sure that it's been hugifiable for long enough. */
|
||||
nstime_t time_hugify_allowed = hpdata_time_hugify_allowed(to_hugify);
|
||||
nstime_t nstime;
|
||||
shard->hooks.curtime(&nstime);
|
||||
shard->central->hooks.curtime(&nstime);
|
||||
nstime_subtract(&nstime, &time_hugify_allowed);
|
||||
uint64_t millis = nstime_msec(&nstime);
|
||||
if (millis < shard->opts.hugify_delay_ms) {
|
||||
@ -427,7 +478,7 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) {
|
||||
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
|
||||
shard->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE);
|
||||
shard->central->hooks.hugify(hpdata_addr_get(to_hugify), HUGEPAGE);
|
||||
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
shard->stats.nhugifies++;
|
||||
@ -604,7 +655,7 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
|
||||
* deallocations (and allocations of smaller sizes) may still succeed
|
||||
* while we're doing this potentially expensive system call.
|
||||
*/
|
||||
hpdata_t *ps = hpa_grow(tsdn, shard);
|
||||
hpdata_t *ps = hpa_central_extract(tsdn, shard->central, size, &oom);
|
||||
if (ps == NULL) {
|
||||
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
|
||||
return nsuccess;
|
||||
@ -833,7 +884,7 @@ hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) {
|
||||
/* There should be no allocations anywhere. */
|
||||
assert(hpdata_empty(ps));
|
||||
psset_remove(&shard->psset, ps);
|
||||
shard->hooks.unmap(hpdata_addr_get(ps), HUGEPAGE);
|
||||
shard->central->hooks.unmap(hpdata_addr_get(ps), HUGEPAGE);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1747,7 +1747,19 @@ malloc_init_hard_a0_locked() {
|
||||
if (config_prof) {
|
||||
prof_boot1();
|
||||
}
|
||||
arena_boot(&sc_data);
|
||||
if (opt_hpa && !hpa_supported()) {
|
||||
malloc_printf("<jemalloc>: HPA not supported in the current "
|
||||
"configuration; %s.",
|
||||
opt_abort_conf ? "aborting" : "disabling");
|
||||
if (opt_abort_conf) {
|
||||
malloc_abort_invalid_conf();
|
||||
} else {
|
||||
opt_hpa = false;
|
||||
}
|
||||
}
|
||||
if (arena_boot(&sc_data, b0get(), opt_hpa)) {
|
||||
return true;
|
||||
}
|
||||
if (tcache_boot(TSDN_NULL, b0get())) {
|
||||
return true;
|
||||
}
|
||||
@ -1786,7 +1798,7 @@ malloc_init_hard_a0_locked() {
|
||||
hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
|
||||
hpa_shard_opts.deferral_allowed = background_thread_enabled();
|
||||
if (pa_shard_enable_hpa(TSDN_NULL, &a0->pa_shard,
|
||||
&hpa_hooks_default, &hpa_shard_opts, &opt_hpa_sec_opts)) {
|
||||
&hpa_shard_opts, &opt_hpa_sec_opts)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
29
src/pa.c
29
src/pa.c
@ -15,10 +15,23 @@ pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) {
|
||||
}
|
||||
|
||||
bool
|
||||
pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
||||
unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
|
||||
nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
|
||||
ssize_t muzzy_decay_ms) {
|
||||
pa_central_init(pa_central_t *central, base_t *base, bool hpa,
|
||||
hpa_hooks_t *hpa_hooks) {
|
||||
bool err;
|
||||
if (hpa) {
|
||||
err = hpa_central_init(¢ral->hpa, base, hpa_hooks);
|
||||
if (err) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
|
||||
emap_t *emap, base_t *base, unsigned ind, pa_shard_stats_t *stats,
|
||||
malloc_mutex_t *stats_mtx, nstime_t *cur_time, size_t oversize_threshold,
|
||||
ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
|
||||
/* This will change eventually, but for now it should hold. */
|
||||
assert(base_ind_get(base) == ind);
|
||||
if (edata_cache_init(&shard->edata_cache, base)) {
|
||||
@ -42,6 +55,7 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
||||
shard->stats = stats;
|
||||
memset(shard->stats, 0, sizeof(*shard->stats));
|
||||
|
||||
shard->central = central;
|
||||
shard->emap = emap;
|
||||
shard->base = base;
|
||||
|
||||
@ -50,10 +64,9 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
||||
|
||||
bool
|
||||
pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
|
||||
const hpa_hooks_t *hpa_hooks, const hpa_shard_opts_t *hpa_opts,
|
||||
const sec_opts_t *hpa_sec_opts) {
|
||||
if (hpa_shard_init(&shard->hpa_shard, shard->emap, shard->base,
|
||||
&shard->edata_cache, shard->ind, hpa_hooks, hpa_opts)) {
|
||||
const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
|
||||
if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap,
|
||||
shard->base, &shard->edata_cache, shard->ind, hpa_opts)) {
|
||||
return true;
|
||||
}
|
||||
if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai,
|
||||
|
@ -13,6 +13,7 @@ struct test_data_s {
|
||||
* test_data_t and the hpa_shard_t;
|
||||
*/
|
||||
hpa_shard_t shard;
|
||||
hpa_central_t central;
|
||||
base_t *base;
|
||||
edata_cache_t shard_edata_cache;
|
||||
|
||||
@ -50,9 +51,12 @@ create_test_data(hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
|
||||
err = emap_init(&test_data->emap, test_data->base, /* zeroed */ false);
|
||||
assert_false(err, "");
|
||||
|
||||
err = hpa_shard_init(&test_data->shard, &test_data->emap,
|
||||
test_data->base, &test_data->shard_edata_cache, SHARD_IND,
|
||||
hooks, opts);
|
||||
err = hpa_central_init(&test_data->central, test_data->base, hooks);
|
||||
assert_false(err, "");
|
||||
|
||||
err = hpa_shard_init(&test_data->shard, &test_data->central,
|
||||
&test_data->emap, test_data->base, &test_data->shard_edata_cache,
|
||||
SHARD_IND, opts);
|
||||
assert_false(err, "");
|
||||
|
||||
return (hpa_shard_t *)test_data;
|
||||
|
@ -40,6 +40,7 @@ init_test_extent_hooks(extent_hooks_t *hooks) {
|
||||
typedef struct test_data_s test_data_t;
|
||||
struct test_data_s {
|
||||
pa_shard_t shard;
|
||||
pa_central_t central;
|
||||
base_t *base;
|
||||
emap_t emap;
|
||||
pa_shard_stats_t stats;
|
||||
@ -63,9 +64,13 @@ test_data_t *init_test_data(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
|
||||
nstime_t time;
|
||||
nstime_init(&time, 0);
|
||||
|
||||
err = pa_central_init(&test_data->central, base, opt_hpa,
|
||||
&hpa_hooks_default);
|
||||
assert_false(err, "");
|
||||
|
||||
const size_t oversize_threshold = 8 * 1024 * 1024;
|
||||
err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap,
|
||||
test_data->base, /* ind */ 1, &test_data->stats,
|
||||
err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->central,
|
||||
&test_data->emap, test_data->base, /* ind */ 1, &test_data->stats,
|
||||
&test_data->stats_mtx, &time, oversize_threshold, dirty_decay_ms,
|
||||
muzzy_decay_ms);
|
||||
assert_false(err, "");
|
||||
|
Loading…
Reference in New Issue
Block a user