Convert arena->stats synchronization to atomics.

include/jemalloc/internal/arena_externs.h

@@ -13,6 +13,12 @@ extern ssize_t		opt_decay_time;
 
 extern const arena_bin_info_t	arena_bin_info[NBINS];
 
+void arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
+    szind_t szind, uint64_t nrequests);
+void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
+    const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty,
+    arena_stats_t *astats, malloc_bin_stats_t *bstats,
+    malloc_large_stats_t *lstats);
 void	arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent);
 #ifdef JEMALLOC_JET
@@ -21,7 +27,7 @@ size_t	arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr);
 extent_t	*arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena,
     size_t usize, size_t alignment, bool *zero);
 void	arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
-    extent_t *extent, bool locked);
+    extent_t *extent);
 void	arena_extent_dalloc_large_finish(tsdn_t *tsdn, arena_t *arena,
     extent_t *extent);
 void	arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
@@ -67,10 +73,6 @@ bool	arena_decay_time_default_set(ssize_t decay_time);
 void	arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena,
     unsigned *nthreads, const char **dss, ssize_t *decay_time, size_t *nactive,
     size_t *ndirty);
-void	arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
-    const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty,
-    arena_stats_t *astats, malloc_bin_stats_t *bstats,
-    malloc_large_stats_t *lstats);
 unsigned	arena_nthreads_get(arena_t *arena, bool internal);
 void	arena_nthreads_inc(arena_t *arena, bool internal);
 void	arena_nthreads_dec(arena_t *arena, bool internal);

include/jemalloc/internal/arena_structs_b.h

@@ -127,8 +127,9 @@ struct arena_s {
 	 */
 	malloc_mutex_t		lock;
 
-	/* Synchronization: lock. */
+	/* Synchronization: internal. */
 	arena_stats_t		stats;
+
 	/*
 	 * List of tcaches for extant threads associated with this arena.
 	 * Stats from these are merged incrementally, and at exit if

include/jemalloc/internal/jemalloc_internal.h.in

@@ -411,10 +411,10 @@ typedef unsigned szind_t;
 #include "jemalloc/internal/spin_structs.h"
 #include "jemalloc/internal/ticker_structs.h"
 #include "jemalloc/internal/ckh_structs.h"
-#include "jemalloc/internal/stats_structs.h"
-#include "jemalloc/internal/ctl_structs.h"
 #include "jemalloc/internal/witness_structs.h"
 #include "jemalloc/internal/mutex_structs.h"
+#include "jemalloc/internal/stats_structs.h"
+#include "jemalloc/internal/ctl_structs.h"
 #include "jemalloc/internal/bitmap_structs.h"
 #include "jemalloc/internal/arena_structs_a.h"
 #include "jemalloc/internal/extent_structs.h"

include/jemalloc/internal/private_symbols.txt

@@ -66,6 +66,7 @@ arena_salloc
 arena_sdalloc
 arena_set
 arena_slab_regind
+arena_stats_init
 arena_stats_merge
 arena_tcache_fill_small
 arena_tdata_get

include/jemalloc/internal/stats_structs.h

@@ -76,6 +76,10 @@ struct malloc_large_stats_s {
  * requests.
  */
 struct arena_stats_s {
+#ifndef JEMALLOC_ATOMIC_U64
+	malloc_mutex_t	mtx;
+#endif
+
 	/* Number of bytes currently mapped, excluding retained memory. */
 	size_t		mapped; /* Derived. */
 
@@ -97,7 +101,7 @@ struct arena_stats_s {
 	uint64_t	purged;
 
 	size_t		base; /* Derived. */
-	size_t		internal; /* Protected via atomic_*_zu(). */
+	size_t		internal;
 	size_t		resident; /* Derived. */
 
 	size_t		allocated_large;

include/jemalloc/internal/witness_types.h

@@ -41,10 +41,11 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
 #define WITNESS_RANK_RTREE_ELM		12U
 #define WITNESS_RANK_RTREE		13U
 #define WITNESS_RANK_BASE		14U
+#define WITNESS_RANK_ARENA_LARGE	15U
 
 #define WITNESS_RANK_LEAF		0xffffffffU
 #define WITNESS_RANK_ARENA_BIN		WITNESS_RANK_LEAF
-#define WITNESS_RANK_ARENA_LARGE	WITNESS_RANK_LEAF
+#define WITNESS_RANK_ARENA_STATS	WITNESS_RANK_LEAF
 #define WITNESS_RANK_DSS		WITNESS_RANK_LEAF
 #define WITNESS_RANK_PROF_ACTIVE	WITNESS_RANK_LEAF
 #define WITNESS_RANK_PROF_ACCUM		WITNESS_RANK_LEAF