PA: Make mapped stat atomic.
We always have atomic_zu_t, and mapped/unmapped transitions are always expensive enough that trying to piggyback on a lock is a waste of time.
This commit is contained in:
parent
3c28aa6f17
commit
436789ad96
@ -67,7 +67,7 @@ struct pa_shard_stats_s {
|
|||||||
* Partially derived -- we maintain our own counter, but add in the
|
* Partially derived -- we maintain our own counter, but add in the
|
||||||
* base's own counter at merge.
|
* base's own counter at merge.
|
||||||
*/
|
*/
|
||||||
locked_zu_t mapped;
|
atomic_zu_t mapped;
|
||||||
|
|
||||||
/* Number of edata_t structs allocated by base, but not being used. */
|
/* Number of edata_t structs allocated by base, but not being used. */
|
||||||
size_t edata_avail; /* Derived. */
|
size_t edata_avail; /* Derived. */
|
||||||
@ -135,14 +135,6 @@ struct pa_shard_s {
|
|||||||
base_t *base;
|
base_t *base;
|
||||||
};
|
};
|
||||||
|
|
||||||
static inline void
|
|
||||||
pa_shard_stats_mapped_add(tsdn_t *tsdn, pa_shard_t *shard, size_t size) {
|
|
||||||
LOCKEDINT_MTX_LOCK(tsdn, *shard->stats_mtx);
|
|
||||||
locked_inc_zu(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
|
|
||||||
&shard->stats->mapped, size);
|
|
||||||
LOCKEDINT_MTX_UNLOCK(tsdn, *shard->stats_mtx);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline ssize_t
|
static inline ssize_t
|
||||||
pa_shard_dirty_decay_ms_get(pa_shard_t *shard) {
|
pa_shard_dirty_decay_ms_get(pa_shard_t *shard) {
|
||||||
return decay_ms_read(&shard->decay_dirty);
|
return decay_ms_read(&shard->decay_dirty);
|
||||||
|
19
src/arena.c
19
src/arena.c
@ -90,16 +90,15 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
|
|||||||
size_t base_allocated, base_resident, base_mapped, metadata_thp;
|
size_t base_allocated, base_resident, base_mapped, metadata_thp;
|
||||||
base_stats_get(tsdn, arena->base, &base_allocated, &base_resident,
|
base_stats_get(tsdn, arena->base, &base_allocated, &base_resident,
|
||||||
&base_mapped, &metadata_thp);
|
&base_mapped, &metadata_thp);
|
||||||
|
size_t mapped = atomic_load_zu(&arena->pa_shard.stats->mapped,
|
||||||
|
ATOMIC_RELAXED);
|
||||||
|
atomic_load_add_store_zu(&astats->pa_shard_stats.mapped,
|
||||||
|
base_mapped + mapped);
|
||||||
|
|
||||||
LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
|
LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
|
||||||
|
|
||||||
locked_inc_zu_unsynchronized(&astats->pa_shard_stats.mapped,
|
|
||||||
base_mapped + locked_read_zu(tsdn,
|
|
||||||
LOCKEDINT_MTX(*arena->pa_shard.stats_mtx),
|
|
||||||
&arena->pa_shard.stats->mapped));
|
|
||||||
locked_inc_zu_unsynchronized(&astats->retained,
|
locked_inc_zu_unsynchronized(&astats->retained,
|
||||||
ecache_npages_get(&arena->pa_shard.ecache_retained) << LG_PAGE);
|
ecache_npages_get(&arena->pa_shard.ecache_retained) << LG_PAGE);
|
||||||
|
|
||||||
astats->pa_shard_stats.edata_avail = atomic_load_zu(
|
astats->pa_shard_stats.edata_avail = atomic_load_zu(
|
||||||
&arena->pa_shard.edata_cache.count, ATOMIC_RELAXED);
|
&arena->pa_shard.edata_cache.count, ATOMIC_RELAXED);
|
||||||
|
|
||||||
@ -436,9 +435,9 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
|||||||
LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
|
LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx);
|
||||||
arena_large_malloc_stats_update(tsdn, arena, usize);
|
arena_large_malloc_stats_update(tsdn, arena, usize);
|
||||||
if (mapped_add != 0) {
|
if (mapped_add != 0) {
|
||||||
locked_inc_zu(tsdn,
|
atomic_fetch_add_zu(
|
||||||
LOCKEDINT_MTX(arena->stats.mtx),
|
&arena->pa_shard.stats->mapped, mapped_add,
|
||||||
&arena->pa_shard.stats->mapped, mapped_add);
|
ATOMIC_RELAXED);
|
||||||
}
|
}
|
||||||
LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
|
LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
|
||||||
}
|
}
|
||||||
@ -848,8 +847,8 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard
|
|||||||
edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size,
|
edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size,
|
||||||
PAGE, /* slab */ true, /* szind */ binind, &zero, &mapped_add);
|
PAGE, /* slab */ true, /* szind */ binind, &zero, &mapped_add);
|
||||||
if (config_stats && slab != NULL && mapped_add != 0) {
|
if (config_stats && slab != NULL && mapped_add != 0) {
|
||||||
pa_shard_stats_mapped_add(tsdn, &arena->pa_shard,
|
atomic_fetch_add_zu(&arena->pa_shard.stats->mapped, mapped_add,
|
||||||
bin_info->slab_size);
|
ATOMIC_RELAXED);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (slab == NULL) {
|
if (slab == NULL) {
|
||||||
|
11
src/ctl.c
11
src/ctl.c
@ -861,7 +861,7 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
|
|||||||
ctl_arena_stats_t *astats = ctl_arena->astats;
|
ctl_arena_stats_t *astats = ctl_arena->astats;
|
||||||
|
|
||||||
if (!destroyed) {
|
if (!destroyed) {
|
||||||
ctl_accum_locked_zu(
|
ctl_accum_atomic_zu(
|
||||||
&sdstats->astats.pa_shard_stats.mapped,
|
&sdstats->astats.pa_shard_stats.mapped,
|
||||||
&astats->astats.pa_shard_stats.mapped);
|
&astats->astats.pa_shard_stats.mapped);
|
||||||
ctl_accum_locked_zu(&sdstats->astats.retained,
|
ctl_accum_locked_zu(&sdstats->astats.retained,
|
||||||
@ -1101,8 +1101,9 @@ ctl_refresh(tsdn_t *tsdn) {
|
|||||||
&ctl_sarena->astats->astats.metadata_thp, ATOMIC_RELAXED);
|
&ctl_sarena->astats->astats.metadata_thp, ATOMIC_RELAXED);
|
||||||
ctl_stats->resident = atomic_load_zu(
|
ctl_stats->resident = atomic_load_zu(
|
||||||
&ctl_sarena->astats->astats.resident, ATOMIC_RELAXED);
|
&ctl_sarena->astats->astats.resident, ATOMIC_RELAXED);
|
||||||
ctl_stats->mapped = locked_read_atomic_zu(
|
ctl_stats->mapped = atomic_load_zu(
|
||||||
&ctl_sarena->astats->astats.pa_shard_stats.mapped);
|
&ctl_sarena->astats->astats.pa_shard_stats.mapped,
|
||||||
|
ATOMIC_RELAXED);
|
||||||
ctl_stats->retained = locked_read_atomic_zu(
|
ctl_stats->retained = locked_read_atomic_zu(
|
||||||
&ctl_sarena->astats->astats.retained);
|
&ctl_sarena->astats->astats.retained);
|
||||||
|
|
||||||
@ -2913,8 +2914,8 @@ CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t)
|
|||||||
CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t)
|
CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t)
|
||||||
CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t)
|
CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
|
||||||
locked_read_atomic_zu(&arenas_i(
|
atomic_load_zu(&arenas_i(mib[2])->astats->astats.pa_shard_stats.mapped,
|
||||||
mib[2])->astats->astats.pa_shard_stats.mapped), size_t)
|
ATOMIC_RELAXED), size_t)
|
||||||
CTL_RO_CGEN(config_stats, stats_arenas_i_retained,
|
CTL_RO_CGEN(config_stats, stats_arenas_i_retained,
|
||||||
locked_read_atomic_zu(&arenas_i(mib[2])->astats->astats.retained),
|
locked_read_atomic_zu(&arenas_i(mib[2])->astats->astats.retained),
|
||||||
size_t)
|
size_t)
|
||||||
|
@ -966,10 +966,9 @@ extent_maximally_purge(tsdn_t *tsdn, pa_shard_t *shard, ehooks_t *ehooks,
|
|||||||
LOCKEDINT_MTX(*shard->stats_mtx),
|
LOCKEDINT_MTX(*shard->stats_mtx),
|
||||||
&shard->stats->decay_dirty.purged,
|
&shard->stats->decay_dirty.purged,
|
||||||
extent_size >> LG_PAGE);
|
extent_size >> LG_PAGE);
|
||||||
locked_dec_zu(tsdn,
|
|
||||||
LOCKEDINT_MTX(*shard->stats_mtx),
|
|
||||||
&shard->stats->mapped, extent_size);
|
|
||||||
LOCKEDINT_MTX_UNLOCK(tsdn, *shard->stats_mtx);
|
LOCKEDINT_MTX_UNLOCK(tsdn, *shard->stats_mtx);
|
||||||
|
atomic_fetch_sub_zu(&shard->stats->mapped, extent_size,
|
||||||
|
ATOMIC_RELAXED);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -122,7 +122,8 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (config_stats && mapped_add > 0) {
|
if (config_stats && mapped_add > 0) {
|
||||||
pa_shard_stats_mapped_add(tsdn, &arena->pa_shard, mapped_add);
|
atomic_fetch_add_zu(&arena->pa_shard.stats->mapped, mapped_add,
|
||||||
|
ATOMIC_RELAXED);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (zero) {
|
if (zero) {
|
||||||
|
4
src/pa.c
4
src/pa.c
@ -268,9 +268,9 @@ pa_decay_stashed(tsdn_t *tsdn, pa_shard_t *shard, decay_t *decay,
|
|||||||
&decay_stats->nmadvise, nmadvise);
|
&decay_stats->nmadvise, nmadvise);
|
||||||
locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
|
locked_inc_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
|
||||||
&decay_stats->purged, npurged);
|
&decay_stats->purged, npurged);
|
||||||
locked_dec_zu(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
|
|
||||||
&shard->stats->mapped, nunmapped << LG_PAGE);
|
|
||||||
LOCKEDINT_MTX_UNLOCK(tsdn, *shard->stats_mtx);
|
LOCKEDINT_MTX_UNLOCK(tsdn, *shard->stats_mtx);
|
||||||
|
atomic_fetch_sub_zu(&shard->stats->mapped, nunmapped << LG_PAGE,
|
||||||
|
ATOMIC_RELAXED);
|
||||||
}
|
}
|
||||||
|
|
||||||
return npurged;
|
return npurged;
|
||||||
|
Loading…
Reference in New Issue
Block a user