Refactor huge allocation to be managed by arenas.

Refactor huge allocation to be managed by arenas (though the global
red-black tree of huge allocations remains for lookup during
deallocation).  This is the logical conclusion of recent changes that 1)
made per arena dss precedence apply to huge allocation, and 2) made it
possible to replace the per arena chunk allocation/deallocation
functions.

Remove the top level huge stats, and replace them with per arena huge
stats.

Normalize function names and types to *dalloc* (some were *dealloc*).

Remove the --enable-mremap option.  As jemalloc currently operates, this
is a performace regression for some applications, but planned work to
logarithmically space huge size classes should provide similar amortized
performance.  The motivation for this change was that mremap-based huge
reallocation forced leaky abstractions that prevented refactoring.

include/jemalloc/internal/arena.h

@@ -345,7 +345,7 @@ struct arena_s {
 	 */
 	arena_chunk_t		*spare;
 
-	/* Number of pages in active runs. */
+	/* Number of pages in active runs and huge regions. */
 	size_t			nactive;
 
 	/*
@@ -374,7 +374,7 @@ struct arena_s {
 	 * user-configureable chunk allocation and deallocation functions.
 	 */
 	chunk_alloc_t		*chunk_alloc;
-	chunk_dealloc_t		*chunk_dealloc;
+	chunk_dalloc_t		*chunk_dalloc;
 
 	/* bins is used to store trees of free regions. */
 	arena_bin_t		bins[NBINS];
@@ -403,6 +403,9 @@ extern arena_bin_info_t	arena_bin_info[NBINS];
 /* Number of large size classes. */
 #define			nlclasses (chunk_npages - map_bias)
 
+void	*arena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment,
+    bool *zero);
+void	arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size);
 void	arena_purge_all(arena_t *arena);
 void	arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
     size_t binind, uint64_t prof_accumbytes);

include/jemalloc/internal/base.h

@@ -12,7 +12,7 @@
 void	*base_alloc(size_t size);
 void	*base_calloc(size_t number, size_t size);
 extent_node_t *base_node_alloc(void);
-void	base_node_dealloc(extent_node_t *node);
+void	base_node_dalloc(extent_node_t *node);
 bool	base_boot(void);
 void	base_prefork(void);
 void	base_postfork_parent(void);

include/jemalloc/internal/chunk.h

@@ -43,12 +43,14 @@ extern size_t		chunk_npages;
 extern size_t		map_bias; /* Number of arena chunk header pages. */
 extern size_t		arena_maxclass; /* Max size class for arenas. */
 
-void	*chunk_alloc(arena_t *arena, size_t size, size_t alignment, bool base,
-    bool *zero, dss_prec_t dss_prec);
+void	*chunk_alloc_base(size_t size);
+void	*chunk_alloc_arena(chunk_alloc_t *chunk_alloc,
+    chunk_dalloc_t *chunk_dalloc, unsigned arena_ind, size_t size,
+    size_t alignment, bool *zero);
 void	*chunk_alloc_default(size_t size, size_t alignment, bool *zero,
     unsigned arena_ind);
 void	chunk_unmap(void *chunk, size_t size);
-void	chunk_dealloc(arena_t *arena, void *chunk, size_t size, bool unmap);
+bool	chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind);
 bool	chunk_boot(void);
 void	chunk_prefork(void);
 void	chunk_postfork_parent(void);

include/jemalloc/internal/chunk_mmap.h

@@ -12,7 +12,7 @@
 bool	pages_purge(void *addr, size_t length);
 
 void	*chunk_alloc_mmap(size_t size, size_t alignment, bool *zero);
-bool	chunk_dealloc_mmap(void *chunk, size_t size);
+bool	chunk_dalloc_mmap(void *chunk, size_t size);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/ctl.h

@@ -57,11 +57,6 @@ struct ctl_stats_s {
 		uint64_t	total;		/* stats_chunks.nchunks */
 		size_t		high;		/* stats_chunks.highchunks */
 	} chunks;
-	struct {
-		size_t		allocated;	/* huge_allocated */
-		uint64_t	nmalloc;	/* huge_nmalloc */
-		uint64_t	ndalloc;	/* huge_ndalloc */
-	} huge;
 	unsigned		narenas;
 	ctl_arena_stats_t	*arenas;	/* (narenas + 1) elements. */
 };

include/jemalloc/internal/huge.h

@@ -9,30 +9,18 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-/* Huge allocation statistics. */
-extern uint64_t		huge_nmalloc;
-extern uint64_t		huge_ndalloc;
-extern size_t		huge_allocated;
-
-/* Protects chunk-related data structures. */
-extern malloc_mutex_t	huge_mtx;
-
-void	*huge_malloc(arena_t *arena, size_t size, bool zero,
-    dss_prec_t dss_prec);
-void	*huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero,
-    dss_prec_t dss_prec);
+void	*huge_malloc(arena_t *arena, size_t size, bool zero);
+void	*huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
 bool	huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra);
 void	*huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
-    size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc,
-    dss_prec_t dss_prec);
+    size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc);
 #ifdef JEMALLOC_JET
 typedef void (huge_dalloc_junk_t)(void *, size_t);
 extern huge_dalloc_junk_t *huge_dalloc_junk;
 #endif
-void	huge_dalloc(void *ptr, bool unmap);
+void	huge_dalloc(void *ptr);
 size_t	huge_salloc(const void *ptr);
-dss_prec_t	huge_dss_prec_get(arena_t *arena);
 prof_ctx_t	*huge_prof_ctx_get(const void *ptr);
 void	huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
 bool	huge_boot(void);

include/jemalloc/internal/jemalloc_internal.h.in

@@ -122,13 +122,6 @@ static const bool config_prof_libunwind =
     false
 #endif
     ;
-static const bool config_mremap =
-#ifdef JEMALLOC_MREMAP
-    true
-#else
-    false
-#endif
-    ;
 static const bool config_munmap =
 #ifdef JEMALLOC_MUNMAP
     true
@@ -702,8 +695,7 @@ imalloct(size_t size, bool try_tcache, arena_t *arena)
 	if (size <= arena_maxclass)
 		return (arena_malloc(arena, size, false, try_tcache));
 	else
-		return (huge_malloc(arena, size, false,
-		    huge_dss_prec_get(arena)));
+		return (huge_malloc(arena, size, false));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -720,8 +712,7 @@ icalloct(size_t size, bool try_tcache, arena_t *arena)
 	if (size <= arena_maxclass)
 		return (arena_malloc(arena, size, true, try_tcache));
 	else
-		return (huge_malloc(arena, size, true,
-		    huge_dss_prec_get(arena)));
+		return (huge_malloc(arena, size, true));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -747,11 +738,9 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
 			ret = arena_palloc(choose_arena(arena), usize,
 			    alignment, zero);
 		} else if (alignment <= chunksize)
-			ret = huge_malloc(arena, usize, zero,
-			    huge_dss_prec_get(arena));
+			ret = huge_malloc(arena, usize, zero);
 		else
-			ret = huge_palloc(arena, usize, alignment, zero,
-			    huge_dss_prec_get(arena));
+			ret = huge_palloc(arena, usize, alignment, zero);
 	}
 
 	assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
@@ -833,7 +822,7 @@ idalloct(void *ptr, bool try_tcache)
 	if (chunk != ptr)
 		arena_dalloc(chunk, ptr, try_tcache);
 	else
-		huge_dalloc(ptr, true);
+		huge_dalloc(ptr);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -920,7 +909,7 @@ iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 		    try_tcache_dalloc));
 	} else {
 		return (huge_ralloc(arena, ptr, oldsize, size, extra,
-		    alignment, zero, try_tcache_dalloc, huge_dss_prec_get(arena)));
+		    alignment, zero, try_tcache_dalloc));
 	}
 }
 

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -144,13 +144,6 @@
  */
 #undef JEMALLOC_MUNMAP
 
-/*
- * If defined, use mremap(...MREMAP_FIXED...) for huge realloc().  This is
- * disabled by default because it is Linux-specific and it will cause virtual
- * memory map holes, much like munmap(2) does.
- */
-#undef JEMALLOC_MREMAP
-
 /* TLS is used to map arenas and magazine caches to threads. */
 #undef JEMALLOC_TLS
 

include/jemalloc/internal/private_symbols.txt

@@ -5,6 +5,8 @@ arena_alloc_junk_small
 arena_bin_index
 arena_bin_info
 arena_boot
+arena_chunk_alloc_huge
+arena_chunk_dalloc_huge
 arena_dalloc
 arena_dalloc_bin
 arena_dalloc_bin_locked
@@ -86,7 +88,7 @@ base_alloc
 base_boot
 base_calloc
 base_node_alloc
-base_node_dealloc
+base_node_dalloc
 base_postfork_child
 base_postfork_parent
 base_prefork
@@ -103,13 +105,14 @@ bt_init
 buferror
 choose_arena
 choose_arena_hard
-chunk_alloc
+chunk_alloc_arena
+chunk_alloc_base
 chunk_alloc_default
 chunk_alloc_dss
 chunk_alloc_mmap
 chunk_boot
-chunk_dealloc
-chunk_dealloc_mmap
+chunk_dalloc_default
+chunk_dalloc_mmap
 chunk_dss_boot
 chunk_dss_postfork_child
 chunk_dss_postfork_parent
@@ -198,9 +201,7 @@ huge_allocated
 huge_boot
 huge_dalloc
 huge_dalloc_junk
-huge_dss_prec_get
 huge_malloc
-huge_mtx
 huge_ndalloc
 huge_nmalloc
 huge_palloc

include/jemalloc/internal/stats.h

@@ -101,6 +101,11 @@ struct arena_stats_s {
 	uint64_t	ndalloc_large;
 	uint64_t	nrequests_large;
 
+	size_t		allocated_huge;
+	uint64_t	nmalloc_huge;
+	uint64_t	ndalloc_huge;
+	uint64_t	nrequests_huge;
+
 	/*
 	 * One element for each possible size class, including sizes that
 	 * overlap with bin size classes.  This is necessary because ipalloc()

include/jemalloc/jemalloc_protos.h.in

@@ -46,4 +46,4 @@ JEMALLOC_EXPORT void *	@je_@valloc(size_t size) JEMALLOC_ATTR(malloc);
 #endif
 
 typedef void *(chunk_alloc_t)(size_t, size_t, bool *, unsigned);
-typedef bool (chunk_dealloc_t)(void *, size_t, unsigned);
+typedef bool (chunk_dalloc_t)(void *, size_t, unsigned);