PA->PAC: Move in PAI implementation.
This commit is contained in:
David Goldblatt
David Goldblatt
parent
6041aaba97
commit
6107857b7b
@ -66,12 +66,7 @@ struct pa_shard_s {
|
||||
*/
|
||||
atomic_zu_t nactive;
|
||||
|
||||
/*
|
||||
* An interface for page allocation from the ecache framework (i.e. a
|
||||
* cascade of ecache_dirty, ecache_muzzy, ecache_retained). Right now
|
||||
* this is the *only* pai, but we'll soon grow another.
|
||||
*/
|
||||
pai_t ecache_pai;
|
||||
/* Allocates from a PAC. */
|
||||
pac_t pac;
|
||||
|
||||
/* The source of edata_t objects. */
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
#ifndef JEMALLOC_INTERNAL_PAC_H
|
||||
#define JEMALLOC_INTERNAL_PAC_H
|
||||
|
||||
#include "jemalloc/internal/pai.h"
|
||||
|
||||
/*
|
||||
* Page allocator classic; an implementation of the PAI interface that:
|
||||
* - Can be used for arenas with custom extent hooks.
|
||||
@ -71,6 +73,11 @@ struct pac_stats_s {
|
||||
|
||||
typedef struct pac_s pac_t;
|
||||
struct pac_s {
|
||||
/*
|
||||
* Must be the first member (we convert it to a PAC given only a
|
||||
* pointer). The handle to the allocation interface.
|
||||
*/
|
||||
pai_t pai;
|
||||
/*
|
||||
* Collections of extents that were previously allocated. These are
|
||||
* used when allocating extents, in an attempt to re-use address space.
|
||||
|
||||
254
src/pa.c
254
src/pa.c
@ -1,14 +1,6 @@
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/jemalloc_internal_includes.h"
|
||||
|
||||
static edata_t *ecache_pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
|
||||
size_t alignment, bool zero);
|
||||
static bool ecache_pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
||||
size_t old_size, size_t new_size, bool zero);
|
||||
static bool ecache_pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
||||
size_t old_size, size_t new_size);
|
||||
static void ecache_pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata);
|
||||
|
||||
static void
|
||||
pa_nactive_add(pa_shard_t *shard, size_t add_pages) {
|
||||
atomic_fetch_add_zu(&shard->nactive, add_pages, ATOMIC_RELAXED);
|
||||
@ -44,11 +36,6 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
|
||||
shard->emap = emap;
|
||||
shard->base = base;
|
||||
|
||||
shard->ecache_pai.alloc = &ecache_pai_alloc;
|
||||
shard->ecache_pai.expand = &ecache_pai_expand;
|
||||
shard->ecache_pai.shrink = &ecache_pai_shrink;
|
||||
shard->ecache_pai.dalloc = &ecache_pai_dalloc;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -62,43 +49,13 @@ pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
|
||||
pac_destroy(tsdn, &shard->pac);
|
||||
}
|
||||
|
||||
static inline bool
|
||||
pa_shard_may_have_muzzy(pa_shard_t *shard) {
|
||||
return pac_decay_ms_get(&shard->pac, extent_state_muzzy) != 0;
|
||||
}
|
||||
|
||||
static edata_t *
|
||||
ecache_pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
|
||||
bool zero) {
|
||||
pa_shard_t *shard =
|
||||
(pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai));
|
||||
|
||||
ehooks_t *ehooks = pa_shard_ehooks_get(shard);
|
||||
edata_t *edata = ecache_alloc(tsdn, &shard->pac, ehooks,
|
||||
&shard->pac.ecache_dirty, NULL, size, alignment, zero);
|
||||
|
||||
if (edata == NULL && pa_shard_may_have_muzzy(shard)) {
|
||||
edata = ecache_alloc(tsdn, &shard->pac, ehooks,
|
||||
&shard->pac.ecache_muzzy, NULL, size, alignment, zero);
|
||||
}
|
||||
if (edata == NULL) {
|
||||
edata = ecache_alloc_grow(tsdn, &shard->pac, ehooks,
|
||||
&shard->pac.ecache_retained, NULL, size, alignment, zero);
|
||||
if (config_stats && edata != NULL) {
|
||||
atomic_fetch_add_zu(&shard->pac.stats->pac_mapped, size,
|
||||
ATOMIC_RELAXED);
|
||||
}
|
||||
}
|
||||
return edata;
|
||||
}
|
||||
|
||||
edata_t *
|
||||
pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
|
||||
bool slab, szind_t szind, bool zero) {
|
||||
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
|
||||
WITNESS_RANK_CORE, 0);
|
||||
|
||||
edata_t *edata = pai_alloc(tsdn, &shard->ecache_pai, size, alignment,
|
||||
edata_t *edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment,
|
||||
zero);
|
||||
|
||||
if (edata != NULL) {
|
||||
@ -113,48 +70,6 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
|
||||
return edata;
|
||||
}
|
||||
|
||||
static bool
|
||||
ecache_pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
|
||||
size_t new_size, bool zero) {
|
||||
pa_shard_t *shard =
|
||||
(pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai));
|
||||
|
||||
ehooks_t *ehooks = pa_shard_ehooks_get(shard);
|
||||
void *trail_begin = edata_past_get(edata);
|
||||
|
||||
size_t mapped_add = 0;
|
||||
size_t expand_amount = new_size - old_size;
|
||||
|
||||
if (ehooks_merge_will_fail(ehooks)) {
|
||||
return true;
|
||||
}
|
||||
edata_t *trail = ecache_alloc(tsdn, &shard->pac, ehooks,
|
||||
&shard->pac.ecache_dirty, trail_begin, expand_amount, PAGE, zero);
|
||||
if (trail == NULL) {
|
||||
trail = ecache_alloc(tsdn, &shard->pac, ehooks,
|
||||
&shard->pac.ecache_muzzy, trail_begin, expand_amount, PAGE,
|
||||
zero);
|
||||
}
|
||||
if (trail == NULL) {
|
||||
trail = ecache_alloc_grow(tsdn, &shard->pac, ehooks,
|
||||
&shard->pac.ecache_retained, trail_begin, expand_amount,
|
||||
PAGE, zero);
|
||||
mapped_add = expand_amount;
|
||||
}
|
||||
if (trail == NULL) {
|
||||
return true;
|
||||
}
|
||||
if (extent_merge_wrapper(tsdn, &shard->pac, ehooks, edata, trail)) {
|
||||
extent_dalloc_wrapper(tsdn, &shard->pac, ehooks, trail);
|
||||
return true;
|
||||
}
|
||||
if (config_stats && mapped_add > 0) {
|
||||
atomic_fetch_add_zu(&shard->pac.stats->pac_mapped, mapped_add,
|
||||
ATOMIC_RELAXED);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
|
||||
size_t new_size, szind_t szind, bool zero) {
|
||||
@ -164,7 +79,7 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
|
||||
|
||||
size_t expand_amount = new_size - old_size;
|
||||
|
||||
bool error = pai_expand(tsdn, &shard->ecache_pai, edata, old_size,
|
||||
bool error = pai_expand(tsdn, &shard->pac.pai, edata, old_size,
|
||||
new_size, zero);
|
||||
if (error) {
|
||||
return true;
|
||||
@ -176,30 +91,6 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool
|
||||
ecache_pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
|
||||
size_t new_size) {
|
||||
pa_shard_t *shard =
|
||||
(pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai));
|
||||
|
||||
ehooks_t *ehooks = pa_shard_ehooks_get(shard);
|
||||
size_t shrink_amount = old_size - new_size;
|
||||
|
||||
|
||||
if (ehooks_split_will_fail(ehooks)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
edata_t *trail = extent_split_wrapper(tsdn, &shard->pac, ehooks, edata,
|
||||
new_size, shrink_amount);
|
||||
if (trail == NULL) {
|
||||
return true;
|
||||
}
|
||||
ecache_dalloc(tsdn, &shard->pac, ehooks, &shard->pac.ecache_dirty,
|
||||
trail);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
|
||||
size_t new_size, szind_t szind, bool *generated_dirty) {
|
||||
@ -209,7 +100,7 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
|
||||
size_t shrink_amount = old_size - new_size;
|
||||
|
||||
*generated_dirty = false;
|
||||
bool error = pai_shrink(tsdn, &shard->ecache_pai, edata, old_size,
|
||||
bool error = pai_shrink(tsdn, &shard->pac.pai, edata, old_size,
|
||||
new_size);
|
||||
if (error) {
|
||||
return true;
|
||||
@ -222,15 +113,6 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
|
||||
return false;
|
||||
}
|
||||
|
||||
static void
|
||||
ecache_pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
|
||||
pa_shard_t *shard =
|
||||
(pa_shard_t *)((uintptr_t)self - offsetof(pa_shard_t, ecache_pai));
|
||||
ehooks_t *ehooks = pa_shard_ehooks_get(shard);
|
||||
ecache_dalloc(tsdn, &shard->pac, ehooks, &shard->pac.ecache_dirty,
|
||||
edata);
|
||||
}
|
||||
|
||||
void
|
||||
pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
|
||||
bool *generated_dirty) {
|
||||
@ -241,138 +123,10 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
|
||||
}
|
||||
edata_szind_set(edata, SC_NSIZES);
|
||||
pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE);
|
||||
pai_dalloc(tsdn, &shard->ecache_pai, edata);
|
||||
pai_dalloc(tsdn, &shard->pac.pai, edata);
|
||||
*generated_dirty = true;
|
||||
}
|
||||
|
||||
static size_t
|
||||
pa_stash_decayed(tsdn_t *tsdn, pa_shard_t *shard, ecache_t *ecache,
|
||||
size_t npages_limit, size_t npages_decay_max,
|
||||
edata_list_inactive_t *result) {
|
||||
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
|
||||
WITNESS_RANK_CORE, 0);
|
||||
ehooks_t *ehooks = pa_shard_ehooks_get(shard);
|
||||
|
||||
/* Stash extents according to npages_limit. */
|
||||
size_t nstashed = 0;
|
||||
while (nstashed < npages_decay_max) {
|
||||
edata_t *edata = ecache_evict(tsdn, &shard->pac, ehooks, ecache,
|
||||
npages_limit);
|
||||
if (edata == NULL) {
|
||||
break;
|
||||
}
|
||||
edata_list_inactive_append(result, edata);
|
||||
nstashed += edata_size_get(edata) >> LG_PAGE;
|
||||
}
|
||||
return nstashed;
|
||||
}
|
||||
|
||||
static size_t
|
||||
pa_decay_stashed(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
|
||||
pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
|
||||
edata_list_inactive_t *decay_extents) {
|
||||
bool err;
|
||||
|
||||
size_t nmadvise = 0;
|
||||
size_t nunmapped = 0;
|
||||
size_t npurged = 0;
|
||||
|
||||
ehooks_t *ehooks = pa_shard_ehooks_get(shard);
|
||||
|
||||
bool try_muzzy = !fully_decay && pa_shard_may_have_muzzy(shard);
|
||||
|
||||
for (edata_t *edata = edata_list_inactive_first(decay_extents);
|
||||
edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
|
||||
edata_list_inactive_remove(decay_extents, edata);
|
||||
|
||||
size_t size = edata_size_get(edata);
|
||||
size_t npages = size >> LG_PAGE;
|
||||
|
||||
nmadvise++;
|
||||
npurged += npages;
|
||||
|
||||
switch (ecache->state) {
|
||||
case extent_state_active:
|
||||
not_reached();
|
||||
case extent_state_dirty:
|
||||
if (try_muzzy) {
|
||||
err = extent_purge_lazy_wrapper(tsdn, ehooks,
|
||||
edata, /* offset */ 0, size);
|
||||
if (!err) {
|
||||
ecache_dalloc(tsdn, &shard->pac, ehooks,
|
||||
&shard->pac.ecache_muzzy, edata);
|
||||
break;
|
||||
}
|
||||
}
|
||||
JEMALLOC_FALLTHROUGH;
|
||||
case extent_state_muzzy:
|
||||
extent_dalloc_wrapper(tsdn, &shard->pac, ehooks, edata);
|
||||
nunmapped += npages;
|
||||
break;
|
||||
case extent_state_retained:
|
||||
default:
|
||||
not_reached();
|
||||
}
|
||||
}
|
||||
|
||||
if (config_stats) {
|
||||
LOCKEDINT_MTX_LOCK(tsdn, *shard->stats_mtx);
|
||||
locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
|
||||
&decay_stats->npurge, 1);
|
||||
locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
|
||||
&decay_stats->nmadvise, nmadvise);
|
||||
locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
|
||||
&decay_stats->purged, npurged);
|
||||
LOCKEDINT_MTX_UNLOCK(tsdn, *shard->stats_mtx);
|
||||
atomic_fetch_sub_zu(&shard->pac.stats->pac_mapped,
|
||||
nunmapped << LG_PAGE, ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
return npurged;
|
||||
}
|
||||
|
||||
/*
|
||||
* npages_limit: Decay at most npages_decay_max pages without violating the
|
||||
* invariant: (ecache_npages_get(ecache) >= npages_limit). We need an upper
|
||||
* bound on number of pages in order to prevent unbounded growth (namely in
|
||||
* stashed), otherwise unbounded new pages could be added to extents during the
|
||||
* current decay run, so that the purging thread never finishes.
|
||||
*/
|
||||
static void
|
||||
pa_decay_to_limit(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
|
||||
pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
|
||||
size_t npages_limit, size_t npages_decay_max) {
|
||||
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
|
||||
WITNESS_RANK_CORE, 1);
|
||||
|
||||
if (decay->purging || npages_decay_max == 0) {
|
||||
return;
|
||||
}
|
||||
decay->purging = true;
|
||||
malloc_mutex_unlock(tsdn, &decay->mtx);
|
||||
|
||||
edata_list_inactive_t decay_extents;
|
||||
edata_list_inactive_init(&decay_extents);
|
||||
size_t npurge = pa_stash_decayed(tsdn, shard, ecache, npages_limit,
|
||||
npages_decay_max, &decay_extents);
|
||||
if (npurge != 0) {
|
||||
size_t npurged = pa_decay_stashed(tsdn, shard, decay,
|
||||
decay_stats, ecache, fully_decay, &decay_extents);
|
||||
assert(npurged == npurge);
|
||||
}
|
||||
|
||||
malloc_mutex_lock(tsdn, &decay->mtx);
|
||||
decay->purging = false;
|
||||
}
|
||||
|
||||
void
|
||||
pa_decay_all(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
|
||||
pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay) {
|
||||
malloc_mutex_assert_owner(tsdn, &decay->mtx);
|
||||
pa_decay_to_limit(tsdn, shard, decay, decay_stats, ecache, fully_decay,
|
||||
/* npages_limit */ 0, ecache_npages_get(ecache));
|
||||
}
|
||||
|
||||
bool
|
||||
pa_shard_retain_grow_limit_get_set(tsdn_t *tsdn, pa_shard_t *shard,
|
||||
size_t *old_limit, size_t *new_limit) {
|
||||
|
||||
131
src/pac.c
131
src/pac.c
@ -3,6 +3,14 @@
|
||||
|
||||
#include "jemalloc/internal/pac.h"
|
||||
|
||||
static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size,
|
||||
size_t alignment, bool zero);
|
||||
static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
||||
size_t old_size, size_t new_size, bool zero);
|
||||
static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
||||
size_t old_size, size_t new_size);
|
||||
static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata);
|
||||
|
||||
static ehooks_t *
|
||||
pac_ehooks_get(pac_t *pac) {
|
||||
return base_ehooks_get(pac->base);
|
||||
@ -27,8 +35,6 @@ pac_decay_data_get(pac_t *pac, extent_state_t state,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
bool
|
||||
pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
|
||||
edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms,
|
||||
@ -78,9 +84,113 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
|
||||
pac->stats = pac_stats;
|
||||
pac->stats_mtx = stats_mtx;
|
||||
atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
|
||||
|
||||
pac->pai.alloc = &pac_alloc_impl;
|
||||
pac->pai.expand = &pac_expand_impl;
|
||||
pac->pai.shrink = &pac_shrink_impl;
|
||||
pac->pai.dalloc = &pac_dalloc_impl;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
pac_may_have_muzzy(pac_t *pac) {
|
||||
return pac_decay_ms_get(pac, extent_state_muzzy) != 0;
|
||||
}
|
||||
|
||||
static edata_t *
|
||||
pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
|
||||
bool zero) {
|
||||
pac_t *pac = (pac_t *)self;
|
||||
|
||||
ehooks_t *ehooks = pac_ehooks_get(pac);
|
||||
edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
|
||||
NULL, size, alignment, zero);
|
||||
|
||||
if (edata == NULL && pac_may_have_muzzy(pac)) {
|
||||
edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
|
||||
NULL, size, alignment, zero);
|
||||
}
|
||||
if (edata == NULL) {
|
||||
edata = ecache_alloc_grow(tsdn, pac, ehooks,
|
||||
&pac->ecache_retained, NULL, size, alignment, zero);
|
||||
if (config_stats && edata != NULL) {
|
||||
atomic_fetch_add_zu(&pac->stats->pac_mapped, size,
|
||||
ATOMIC_RELAXED);
|
||||
}
|
||||
}
|
||||
return edata;
|
||||
}
|
||||
|
||||
static bool
|
||||
pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
|
||||
size_t new_size, bool zero) {
|
||||
pac_t *pac = (pac_t *)self;
|
||||
|
||||
ehooks_t *ehooks = pac_ehooks_get(pac);
|
||||
void *trail_begin = edata_past_get(edata);
|
||||
|
||||
size_t mapped_add = 0;
|
||||
size_t expand_amount = new_size - old_size;
|
||||
|
||||
if (ehooks_merge_will_fail(ehooks)) {
|
||||
return true;
|
||||
}
|
||||
edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
|
||||
trail_begin, expand_amount, PAGE, zero);
|
||||
if (trail == NULL) {
|
||||
trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
|
||||
trail_begin, expand_amount, PAGE, zero);
|
||||
}
|
||||
if (trail == NULL) {
|
||||
trail = ecache_alloc_grow(tsdn, pac, ehooks,
|
||||
&pac->ecache_retained, trail_begin, expand_amount, PAGE,
|
||||
zero);
|
||||
mapped_add = expand_amount;
|
||||
}
|
||||
if (trail == NULL) {
|
||||
return true;
|
||||
}
|
||||
if (extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) {
|
||||
extent_dalloc_wrapper(tsdn, pac, ehooks, trail);
|
||||
return true;
|
||||
}
|
||||
if (config_stats && mapped_add > 0) {
|
||||
atomic_fetch_add_zu(&pac->stats->pac_mapped, mapped_add,
|
||||
ATOMIC_RELAXED);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool
|
||||
pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
|
||||
size_t new_size) {
|
||||
pac_t *pac = (pac_t *)self;
|
||||
|
||||
ehooks_t *ehooks = pac_ehooks_get(pac);
|
||||
size_t shrink_amount = old_size - new_size;
|
||||
|
||||
|
||||
if (ehooks_split_will_fail(ehooks)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
edata_t *trail = extent_split_wrapper(tsdn, pac, ehooks, edata,
|
||||
new_size, shrink_amount);
|
||||
if (trail == NULL) {
|
||||
return true;
|
||||
}
|
||||
ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail);
|
||||
return false;
|
||||
}
|
||||
|
||||
static void
|
||||
pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
|
||||
pac_t *pac = (pac_t *)self;
|
||||
ehooks_t *ehooks = pac_ehooks_get(pac);
|
||||
ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata);
|
||||
}
|
||||
|
||||
bool
|
||||
pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
|
||||
size_t *new_limit) {
|
||||
@ -107,7 +217,8 @@ pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
|
||||
|
||||
static size_t
|
||||
pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
|
||||
size_t npages_limit, size_t npages_decay_max, edata_list_t *result) {
|
||||
size_t npages_limit, size_t npages_decay_max,
|
||||
edata_list_inactive_t *result) {
|
||||
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
|
||||
WITNESS_RANK_CORE, 0);
|
||||
ehooks_t *ehooks = pac_ehooks_get(pac);
|
||||
@ -120,7 +231,7 @@ pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
|
||||
if (edata == NULL) {
|
||||
break;
|
||||
}
|
||||
edata_list_append(result, edata);
|
||||
edata_list_inactive_append(result, edata);
|
||||
nstashed += edata_size_get(edata) >> LG_PAGE;
|
||||
}
|
||||
return nstashed;
|
||||
@ -129,7 +240,7 @@ pac_stash_decayed(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
|
||||
static size_t
|
||||
pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
|
||||
pac_decay_stats_t *decay_stats, ecache_t *ecache, bool fully_decay,
|
||||
edata_list_t *decay_extents) {
|
||||
edata_list_inactive_t *decay_extents) {
|
||||
bool err;
|
||||
|
||||
size_t nmadvise = 0;
|
||||
@ -141,9 +252,9 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
|
||||
bool try_muzzy = !fully_decay
|
||||
&& pac_decay_ms_get(pac, extent_state_muzzy) != 0;
|
||||
|
||||
for (edata_t *edata = edata_list_first(decay_extents); edata !=
|
||||
NULL; edata = edata_list_first(decay_extents)) {
|
||||
edata_list_remove(decay_extents, edata);
|
||||
for (edata_t *edata = edata_list_inactive_first(decay_extents); edata !=
|
||||
NULL; edata = edata_list_inactive_first(decay_extents)) {
|
||||
edata_list_inactive_remove(decay_extents, edata);
|
||||
|
||||
size_t size = edata_size_get(edata);
|
||||
size_t npages = size >> LG_PAGE;
|
||||
@ -211,8 +322,8 @@ pac_decay_to_limit(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
|
||||
decay->purging = true;
|
||||
malloc_mutex_unlock(tsdn, &decay->mtx);
|
||||
|
||||
edata_list_t decay_extents;
|
||||
edata_list_init(&decay_extents);
|
||||
edata_list_inactive_t decay_extents;
|
||||
edata_list_inactive_init(&decay_extents);
|
||||
size_t npurge = pac_stash_decayed(tsdn, pac, ecache, npages_limit,
|
||||
npages_decay_max, &decay_extents);
|
||||
if (npurge != 0) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user