Merge branch 'dev'

.gitignore

@@ -1,25 +1,72 @@
+/*.gcov.*
+
 /autom4te.cache/
+
+/bin/jemalloc.sh
+
 /config.stamp
 /config.log
 /config.status
 /configure
+
 /doc/html.xsl
 /doc/manpages.xsl
 /doc/jemalloc.xml
 /doc/jemalloc.html
 /doc/jemalloc.3
+
 /lib/
+
 /Makefile
-/include/jemalloc/internal/jemalloc_internal\.h
-/include/jemalloc/internal/size_classes\.h
-/include/jemalloc/jemalloc\.h
-/include/jemalloc/jemalloc_defs\.h
-/test/jemalloc_test\.h
+
+/include/jemalloc/internal/jemalloc_internal.h
+/include/jemalloc/internal/jemalloc_internal_defs.h
+/include/jemalloc/internal/private_namespace.h
+/include/jemalloc/internal/private_unnamespace.h
+/include/jemalloc/internal/public_namespace.h
+/include/jemalloc/internal/public_symbols.txt
+/include/jemalloc/internal/public_unnamespace.h
+/include/jemalloc/internal/size_classes.h
+/include/jemalloc/jemalloc.h
+/include/jemalloc/jemalloc_defs.h
+/include/jemalloc/jemalloc_macros.h
+/include/jemalloc/jemalloc_mangle.h
+/include/jemalloc/jemalloc_mangle_jet.h
+/include/jemalloc/jemalloc_protos.h
+/include/jemalloc/jemalloc_protos_jet.h
+/include/jemalloc/jemalloc_rename.h
+
 /src/*.[od]
-/test/*.[od]
-/test/*.out
-/test/[a-zA-Z_]*
-!test/*.c
-!test/*.exp
+/src/*.gcda
+/src/*.gcno
+
+/test/test.sh
+test/include/test/jemalloc_test.h
+test/include/test/jemalloc_test_defs.h
+
+/test/integration/[A-Za-z]*
+!/test/integration/[A-Za-z]*.*
+/test/integration/*.[od]
+/test/integration/*.gcda
+/test/integration/*.gcno
+/test/integration/*.out
+
+/test/src/*.[od]
+/test/src/*.gcda
+/test/src/*.gcno
+
+/test/stress/[A-Za-z]*
+!/test/stress/[A-Za-z]*.*
+/test/stress/*.[od]
+/test/stress/*.gcda
+/test/stress/*.gcno
+/test/stress/*.out
+
+/test/unit/[A-Za-z]*
+!/test/unit/[A-Za-z]*.*
+/test/unit/*.[od]
+/test/unit/*.gcda
+/test/unit/*.gcno
+/test/unit/*.out
+
 /VERSION
-/bin/jemalloc.sh

COPYING

@@ -1,10 +1,10 @@
 Unless otherwise specified, files in the jemalloc source distribution are
 subject to the following license:
 --------------------------------------------------------------------------------
-Copyright (C) 2002-2013 Jason Evans <jasone@canonware.com>.
+Copyright (C) 2002-2014 Jason Evans <jasone@canonware.com>.
 All rights reserved.
 Copyright (C) 2007-2012 Mozilla Foundation.  All rights reserved.
-Copyright (C) 2009-2013 Facebook, Inc.  All rights reserved.
+Copyright (C) 2009-2014 Facebook, Inc.  All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

ChangeLog

@@ -6,6 +6,59 @@ found in the git revision history:
     http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
     git://canonware.com/jemalloc.git
 
+* 3.5.0 (January 22, 2014)
+
+  This version focuses on refactoring and automated testing, though it also
+  includes some non-trivial heap profiling optimizations not mentioned below.
+
+  New features:
+  - Add the *allocx() API, which is a successor to the experimental *allocm()
+    API.  The *allocx() functions are slightly simpler to use because they have
+    fewer parameters, they directly return the results of primary interest, and
+    mallocx()/rallocx() avoid the strict aliasing pitfall that
+    allocm()/rallocx() share with posix_memalign().  Note that *allocm() is
+    slated for removal in the next non-bugfix release.
+  - Add support for LinuxThreads.
+
+  Bug fixes:
+  - Unless heap profiling is enabled, disable floating point code and don't link
+    with libm.  This, in combination with e.g. EXTRA_CFLAGS=-mno-sse on x64
+    systems, makes it possible to completely disable floating point register
+    use.  Some versions of glibc neglect to save/restore caller-saved floating
+    point registers during dynamic lazy symbol loading, and the symbol loading
+    code uses whatever malloc the application happens to have linked/loaded
+    with, the result being potential floating point register corruption.
+  - Report ENOMEM rather than EINVAL if an OOM occurs during heap profiling
+    backtrace creation in imemalign().  This bug impacted posix_memalign() and
+    aligned_alloc().
+  - Fix a file descriptor leak in a prof_dump_maps() error path.
+  - Fix prof_dump() to close the dump file descriptor for all relevant error
+    paths.
+  - Fix rallocm() to use the arena specified by the ALLOCM_ARENA(s) flag for
+    allocation, not just deallocation.
+  - Fix a data race for large allocation stats counters.
+  - Fix a potential infinite loop during thread exit.  This bug occurred on
+    Solaris, and could affect other platforms with similar pthreads TSD
+    implementations.
+  - Don't junk-fill reallocations unless usable size changes.  This fixes a
+    violation of the *allocx()/*allocm() semantics.
+  - Fix growing large reallocation to junk fill new space.
+  - Fix huge deallocation to junk fill when munmap is disabled.
+  - Change the default private namespace prefix from empty to je_, and change
+    --with-private-namespace-prefix so that it prepends an additional prefix
+    rather than replacing je_.  This reduces the likelihood of applications
+    which statically link jemalloc experiencing symbol name collisions.
+  - Add missing private namespace mangling (relevant when
+    --with-private-namespace is specified).
+  - Add and use JEMALLOC_INLINE_C so that static inline functions are marked as
+    static even for debug builds.
+  - Add a missing mutex unlock in a malloc_init_hard() error path.  In practice
+    this error path is never executed.
+  - Fix numerous bugs in malloc_strotumax() error handling/reporting.  These
+    bugs had no impact except for malformed inputs.
+  - Fix numerous bugs in malloc_snprintf().  These bugs were not exercised by
+    existing calls, so they had no impact.
+
 * 3.4.1 (October 20, 2013)
 
   Bug fixes:

INSTALL

@@ -61,10 +61,10 @@ any of the following arguments (not a definitive list) to 'configure':
     allocator on OSX.
 
 --with-private-namespace=<prefix>
-    Prefix all library-private APIs with <prefix>.  For shared libraries,
+    Prefix all library-private APIs with <prefix>je_.  For shared libraries,
     symbol visibility mechanisms prevent these symbols from being exported, but
     for static libraries, naming collisions are a real possibility.  By
-    default, the prefix is "" (empty string).
+    default, <prefix> is empty, which results in a symbol prefix of je_ .
 
 --with-install-suffix=<suffix>
     Append <suffix> to the base name of all installed files, such that multiple
@@ -81,6 +81,19 @@ any of the following arguments (not a definitive list) to 'configure':
     performance hit, but is very useful during application development.
     Implies --enable-ivsalloc.
 
+--enable-code-coverage
+    Enable code coverage support, for use during jemalloc test development.
+    Additional testing targets are available if this option is enabled:
+
+      coverage
+      coverage_unit
+      coverage_integration
+      coverage_stress
+
+    These targets do not clear code coverage results from previous runs, and
+    there are interactions between the various coverage targets, so it is
+    usually advisable to run 'make clean' between repeated code coverage runs.
+
 --enable-ivsalloc
     Enable validation code, which verifies that pointers reside within
     jemalloc-owned chunks before dereferencing them. This incurs a substantial

Makefile.in

@@ -47,6 +47,7 @@ cfghdrs_out := @cfghdrs_out@
 cfgoutputs_in := @cfgoutputs_in@
 cfgoutputs_out := @cfgoutputs_out@
 enable_autogen := @enable_autogen@
+enable_code_coverage := @enable_code_coverage@
 enable_experimental := @enable_experimental@
 enable_zone_allocator := @enable_zone_allocator@
 DSO_LDFLAGS = @DSO_LDFLAGS@
@@ -73,18 +74,17 @@ LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
 
 # Lists of files.
 BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc.sh
-CHDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h \
-	$(objroot)include/jemalloc/jemalloc_defs$(install_suffix).h
-CSRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c $(srcroot)src/atomic.c \
-	$(srcroot)src/base.c $(srcroot)src/bitmap.c $(srcroot)src/chunk.c \
-	$(srcroot)src/chunk_dss.c $(srcroot)src/chunk_mmap.c \
-	$(srcroot)src/ckh.c $(srcroot)src/ctl.c $(srcroot)src/extent.c \
-	$(srcroot)src/hash.c $(srcroot)src/huge.c $(srcroot)src/mb.c \
-	$(srcroot)src/mutex.c $(srcroot)src/prof.c $(srcroot)src/quarantine.c \
-	$(srcroot)src/rtree.c $(srcroot)src/stats.c $(srcroot)src/tcache.c \
-	$(srcroot)src/util.c $(srcroot)src/tsd.c
+C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
+C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \
+	$(srcroot)src/atomic.c $(srcroot)src/base.c $(srcroot)src/bitmap.c \
+	$(srcroot)src/chunk.c $(srcroot)src/chunk_dss.c \
+	$(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \
+	$(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \
+	$(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/prof.c \
+	$(srcroot)src/quarantine.c $(srcroot)src/rtree.c $(srcroot)src/stats.c \
+	$(srcroot)src/tcache.c $(srcroot)src/util.c $(srcroot)src/tsd.c
 ifeq ($(enable_zone_allocator), 1)
-CSRCS += $(srcroot)src/zone.c
+C_SRCS += $(srcroot)src/zone.c
 endif
 ifeq ($(IMPORTLIB),$(SO))
 STATIC_LIBS := $(objroot)lib/$(LIBJEMALLOC).$(A)
@@ -103,27 +103,71 @@ DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml
 DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html)
 DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3)
 DOCS := $(DOCS_HTML) $(DOCS_MAN3)
-CTESTS := $(srcroot)test/aligned_alloc.c $(srcroot)test/allocated.c \
-	$(srcroot)test/ALLOCM_ARENA.c $(srcroot)test/bitmap.c \
-	$(srcroot)test/mremap.c $(srcroot)test/posix_memalign.c \
-	$(srcroot)test/thread_arena.c $(srcroot)test/thread_tcache_enabled.c
+C_TESTLIB_SRCS := $(srcroot)test/src/math.c $(srcroot)test/src/mtx.c \
+	$(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \
+	$(srcroot)test/src/thd.c
+C_UTIL_INTEGRATION_SRCS := $(srcroot)src/util.c
+TESTS_UNIT := $(srcroot)test/unit/bitmap.c \
+	$(srcroot)test/unit/ckh.c \
+	$(srcroot)test/unit/hash.c \
+	$(srcroot)test/unit/junk.c \
+	$(srcroot)test/unit/mallctl.c \
+	$(srcroot)test/unit/math.c \
+	$(srcroot)test/unit/mq.c \
+	$(srcroot)test/unit/mtx.c \
+	$(srcroot)test/unit/prof_accum.c \
+	$(srcroot)test/unit/prof_gdump.c \
+	$(srcroot)test/unit/prof_idump.c \
+	$(srcroot)test/unit/ql.c \
+	$(srcroot)test/unit/qr.c \
+	$(srcroot)test/unit/quarantine.c \
+	$(srcroot)test/unit/rb.c \
+	$(srcroot)test/unit/rtree.c \
+	$(srcroot)test/unit/SFMT.c \
+	$(srcroot)test/unit/stats.c \
+	$(srcroot)test/unit/tsd.c \
+	$(srcroot)test/unit/util.c \
+	$(srcroot)test/unit/zero.c
+TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
+	$(srcroot)test/integration/allocated.c \
+	$(srcroot)test/integration/mallocx.c \
+	$(srcroot)test/integration/mremap.c \
+	$(srcroot)test/integration/posix_memalign.c \
+	$(srcroot)test/integration/rallocx.c \
+	$(srcroot)test/integration/thread_arena.c \
+	$(srcroot)test/integration/thread_tcache_enabled.c \
+	$(srcroot)test/integration/xallocx.c
 ifeq ($(enable_experimental), 1)
-CTESTS += $(srcroot)test/allocm.c $(srcroot)test/rallocm.c
+TESTS_INTEGRATION += $(srcroot)test/integration/allocm.c \
+	$(srcroot)test/integration/ALLOCM_ARENA.c \
+	$(srcroot)test/integration/rallocm.c
 endif
+TESTS_STRESS :=
+TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS)
 
-COBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.$(O))
-CPICOBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.pic.$(O))
-CTESTOBJS := $(CTESTS:$(srcroot)%.c=$(objroot)%.$(O))
+C_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.$(O))
+C_PIC_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.pic.$(O))
+C_JET_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.$(O))
+C_TESTLIB_UNIT_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.unit.$(O))
+C_TESTLIB_INTEGRATION_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
+C_UTIL_INTEGRATION_OBJS := $(C_UTIL_INTEGRATION_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
+C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O))
+C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_STRESS_OBJS)
+
+TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O))
+TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O))
+TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O))
+TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS)
 
 .PHONY: all dist build_doc_html build_doc_man build_doc
 .PHONY: install_bin install_include install_lib
 .PHONY: install_doc_html install_doc_man install_doc install
 .PHONY: tests check clean distclean relclean
 
-.SECONDARY : $(CTESTOBJS)
+.SECONDARY : $(TESTS_OBJS)
 
 # Default target.
-all: build
+all: build_lib
 
 dist: build_doc
 
@@ -141,30 +185,45 @@ build_doc: $(DOCS)
 # Include generated dependency files.
 #
 ifdef CC_MM
--include $(COBJS:%.$(O)=%.d)
--include $(CPICOBJS:%.$(O)=%.d)
--include $(CTESTOBJS:%.$(O)=%.d)
+-include $(C_OBJS:%.$(O)=%.d)
+-include $(C_PIC_OBJS:%.$(O)=%.d)
+-include $(C_JET_OBJS:%.$(O)=%.d)
+-include $(C_TESTLIB_OBJS:%.$(O)=%.d)
+-include $(TESTS_OBJS:%.$(O)=%.d)
 endif
 
-$(COBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c
-$(CPICOBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c
-$(CPICOBJS): CFLAGS += $(PIC_CFLAGS)
-$(CTESTOBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c
-$(CTESTOBJS): CPPFLAGS += -I$(objroot)test
+$(C_OBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c
+$(C_PIC_OBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c
+$(C_PIC_OBJS): CFLAGS += $(PIC_CFLAGS)
+$(C_JET_OBJS): $(objroot)src/%.jet.$(O): $(srcroot)src/%.c
+$(C_JET_OBJS): CFLAGS += -DJEMALLOC_JET
+$(C_TESTLIB_UNIT_OBJS): $(objroot)test/src/%.unit.$(O): $(srcroot)test/src/%.c
+$(C_TESTLIB_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
+$(C_TESTLIB_INTEGRATION_OBJS): $(objroot)test/src/%.integration.$(O): $(srcroot)test/src/%.c
+$(C_TESTLIB_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
+$(C_UTIL_INTEGRATION_OBJS): $(objroot)src/%.integration.$(O): $(srcroot)src/%.c
+$(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/%.c
+$(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB
+$(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
+$(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
+$(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
+$(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST
+$(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c
+$(TESTS_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
 ifneq ($(IMPORTLIB),$(SO))
-$(COBJS): CPPFLAGS += -DDLLEXPORT
+$(C_OBJS): CPPFLAGS += -DDLLEXPORT
 endif
 
 ifndef CC_MM
-# Dependencies
+# Dependencies.
 HEADER_DIRS = $(srcroot)include/jemalloc/internal \
 	$(objroot)include/jemalloc $(objroot)include/jemalloc/internal
 HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h))
-$(COBJS) $(CPICOBJS) $(CTESTOBJS): $(HEADERS)
-$(CTESTOBJS): $(objroot)test/jemalloc_test.h
+$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): $(HEADERS)
+$(TESTS_OBJS): $(objroot)test/unit/jemalloc_test.h
 endif
 
-$(COBJS) $(CPICOBJS) $(CTESTOBJS): %.$(O):
+$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O):
 	@mkdir -p $(@D)
 	$(CC) $(CFLAGS) -c $(CPPFLAGS) $(CTARGET) $<
 ifdef CC_MM
@@ -177,27 +236,33 @@ ifneq ($(SOREV),$(SO))
 	ln -sf $(<F) $@
 endif
 
-$(objroot)lib/$(LIBJEMALLOC).$(SOREV) : $(if $(PIC_CFLAGS),$(CPICOBJS),$(COBJS))
+$(objroot)lib/$(LIBJEMALLOC).$(SOREV) : $(if $(PIC_CFLAGS),$(C_PIC_OBJS),$(C_OBJS))
 	@mkdir -p $(@D)
 	$(CC) $(DSO_LDFLAGS) $(call RPATH,$(RPATH_EXTRA)) $(LDTARGET) $+ $(LDFLAGS) $(LIBS) $(EXTRA_LDFLAGS)
 
-$(objroot)lib/$(LIBJEMALLOC)_pic.$(A) : $(CPICOBJS)
-$(objroot)lib/$(LIBJEMALLOC).$(A) : $(COBJS)
-$(objroot)lib/$(LIBJEMALLOC)_s.$(A) : $(COBJS)
+$(objroot)lib/$(LIBJEMALLOC)_pic.$(A) : $(C_PIC_OBJS)
+$(objroot)lib/$(LIBJEMALLOC).$(A) : $(C_OBJS)
+$(objroot)lib/$(LIBJEMALLOC)_s.$(A) : $(C_OBJS)
 
 $(STATIC_LIBS):
 	@mkdir -p $(@D)
 	$(AR) $(ARFLAGS)@AROUT@ $+
 
-$(objroot)test/bitmap$(EXE): $(objroot)src/bitmap.$(O)
-
-$(objroot)test/%$(EXE): $(objroot)test/%.$(O) $(objroot)src/util.$(O) $(DSOS)
+$(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS)
 	@mkdir -p $(@D)
-	$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(filter -lpthread,$(LIBS)) $(EXTRA_LDFLAGS)
+	$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)
+
+$(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
+	@mkdir -p $(@D)
+	$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(filter -lpthread,$(LIBS))) -lm $(EXTRA_LDFLAGS)
+
+$(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
+	@mkdir -p $(@D)
+	$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)
 
 build_lib_shared: $(DSOS)
 build_lib_static: $(STATIC_LIBS)
-build: build_lib_shared build_lib_static
+build_lib: build_lib_shared build_lib_static
 
 install_bin:
 	install -d $(BINDIR)
@@ -208,7 +273,7 @@ done
 
 install_include:
 	install -d $(INCLUDEDIR)/jemalloc
-	@for h in $(CHDRS); do \
+	@for h in $(C_HDRS); do \
 	echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
 	install -m 644 $$h $(INCLUDEDIR)/jemalloc; \
 done
@@ -247,49 +312,87 @@ install_doc: install_doc_html install_doc_man
 
 install: install_bin install_include install_lib install_doc
 
-tests: $(CTESTS:$(srcroot)%.c=$(objroot)%$(EXE))
+tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE))
+tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE))
+tests_stress: $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%$(EXE))
+tests: tests_unit tests_integration tests_stress
 
-check: tests
-	@mkdir -p $(objroot)test
-	@$(SHELL) -c 'total=0; \
-		failures=0; \
-		echo "========================================="; \
-		for t in $(CTESTS:$(srcroot)%.c=$(objroot)%); do \
-			total=`expr $$total + 1`; \
-			/bin/echo -n "$${t} ... "; \
-			$(TEST_LIBRARY_PATH) $${t}$(EXE) $(abs_srcroot) \
-			  $(abs_objroot) > $(objroot)$${t}.out 2>&1; \
-			if test -e "$(srcroot)$${t}.exp"; then \
-				diff -w -u $(srcroot)$${t}.exp \
-				  $(objroot)$${t}.out >/dev/null 2>&1; \
-				fail=$$?; \
-				if test "$${fail}" -eq "1" ; then \
-					failures=`expr $${failures} + 1`; \
-					echo "*** FAIL ***"; \
-				else \
-					echo "pass"; \
-				fi; \
-			else \
-				echo "*** FAIL *** (.exp file is missing)"; \
-				failures=`expr $${failures} + 1`; \
-			fi; \
-		done; \
-		echo "========================================="; \
-		echo "Failures: $${failures}/$${total}"'
+check_unit_dir:
+	@mkdir -p $(objroot)test/unit
+check_integration_dir:
+	@mkdir -p $(objroot)test/integration
+check_stress_dir:
+	@mkdir -p $(objroot)test/stress
+check_dir: check_unit_dir check_integration_dir check_stress_dir
+
+check_unit: tests_unit check_unit_dir
+	$(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
+check_integration: tests_integration check_integration_dir
+	$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
+check_stress: tests_stress check_stress_dir
+	$(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%)
+check: tests check_dir
+	$(SHELL) $(objroot)test/test.sh $(TESTS:$(srcroot)%.c=$(objroot)%)
+
+ifeq ($(enable_code_coverage), 1)
+coverage_unit: check_unit
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS)
+
+coverage_integration: check_integration
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
+
+coverage_stress: check_stress
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress stress $(TESTS_STRESS_OBJS)
+
+coverage: check
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
+	$(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress integration $(TESTS_STRESS_OBJS)
+endif
 
 clean:
-	rm -f $(COBJS)
-	rm -f $(CPICOBJS)
-	rm -f $(COBJS:%.$(O)=%.d)
-	rm -f $(CPICOBJS:%.$(O)=%.d)
-	rm -f $(CTESTOBJS:%.$(O)=%$(EXE))
-	rm -f $(CTESTOBJS)
-	rm -f $(CTESTOBJS:%.$(O)=%.d)
-	rm -f $(CTESTOBJS:%.$(O)=%.out)
+	rm -f $(C_OBJS)
+	rm -f $(C_PIC_OBJS)
+	rm -f $(C_JET_OBJS)
+	rm -f $(C_TESTLIB_OBJS)
+	rm -f $(C_OBJS:%.$(O)=%.d)
+	rm -f $(C_OBJS:%.$(O)=%.gcda)
+	rm -f $(C_OBJS:%.$(O)=%.gcno)
+	rm -f $(C_PIC_OBJS:%.$(O)=%.d)
+	rm -f $(C_PIC_OBJS:%.$(O)=%.gcda)
+	rm -f $(C_PIC_OBJS:%.$(O)=%.gcno)
+	rm -f $(C_JET_OBJS:%.$(O)=%.d)
+	rm -f $(C_JET_OBJS:%.$(O)=%.gcda)
+	rm -f $(C_JET_OBJS:%.$(O)=%.gcno)
+	rm -f $(C_TESTLIB_OBJS:%.$(O)=%.d)
+	rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcda)
+	rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcno)
+	rm -f $(TESTS_OBJS:%.$(O)=%$(EXE))
+	rm -f $(TESTS_OBJS)
+	rm -f $(TESTS_OBJS:%.$(O)=%.d)
+	rm -f $(TESTS_OBJS:%.$(O)=%.gcda)
+	rm -f $(TESTS_OBJS:%.$(O)=%.gcno)
+	rm -f $(TESTS_OBJS:%.$(O)=%.out)
 	rm -f $(DSOS) $(STATIC_LIBS)
+	rm -f $(objroot)*.gcov.*
 
 distclean: clean
 	rm -rf $(objroot)autom4te.cache
+	rm -f $(objroot)bin/jemalloc.sh
 	rm -f $(objroot)config.log
 	rm -f $(objroot)config.status
 	rm -f $(objroot)config.stamp

config.guess

@@ -1,14 +1,12 @@
 #! /bin/sh
 # Attempt to guess a canonical system name.
-#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-#   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-#   2011, 2012 Free Software Foundation, Inc.
+#   Copyright 1992-2013 Free Software Foundation, Inc.
 
-timestamp='2012-02-10'
+timestamp='2013-06-10'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful, but
@@ -22,19 +20,17 @@ timestamp='2012-02-10'
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
 # configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
-
-
-# Originally written by Per Bothner.  Please send patches (context
-# diff format) to <config-patches@gnu.org> and include a ChangeLog
-# entry.
+# the same distribution terms that you use for the rest of that
+# program.  This Exception is an additional permission under section 7
+# of the GNU General Public License, version 3 ("GPLv3").
 #
-# This script attempts to guess a canonical system name similar to
-# config.sub.  If it succeeds, it prints the system name on stdout, and
-# exits with 0.  Otherwise, it exits with 1.
+# Originally written by Per Bothner.
 #
 # You can get the latest version of this script from:
 # http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+#
+# Please send patches with a ChangeLog entry to config-patches@gnu.org.
+
 
 me=`echo "$0" | sed -e 's,.*/,,'`
 
@@ -54,9 +50,7 @@ version="\
 GNU config.guess ($timestamp)
 
 Originally written by Per Bothner.
-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
-Free Software Foundation, Inc.
+Copyright 1992-2013 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -138,6 +132,27 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
 UNAME_SYSTEM=`(uname -s) 2>/dev/null`  || UNAME_SYSTEM=unknown
 UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
 
+case "${UNAME_SYSTEM}" in
+Linux|GNU|GNU/*)
+	# If the system lacks a compiler, then just pick glibc.
+	# We could probably try harder.
+	LIBC=gnu
+
+	eval $set_cc_for_build
+	cat <<-EOF > $dummy.c
+	#include <features.h>
+	#if defined(__UCLIBC__)
+	LIBC=uclibc
+	#elif defined(__dietlibc__)
+	LIBC=dietlibc
+	#else
+	LIBC=gnu
+	#endif
+	EOF
+	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'`
+	;;
+esac
+
 # Note: order is significant - the case branches are not exclusive.
 
 case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
@@ -200,6 +215,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
 	echo "${machine}-${os}${release}"
 	exit ;;
+    *:Bitrig:*:*)
+	UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
+	echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE}
+	exit ;;
     *:OpenBSD:*:*)
 	UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
 	echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE}
@@ -302,7 +321,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
     arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
 	echo arm-acorn-riscix${UNAME_RELEASE}
 	exit ;;
-    arm:riscos:*:*|arm:RISCOS:*:*)
+    arm*:riscos:*:*|arm*:RISCOS:*:*)
 	echo arm-unknown-riscos
 	exit ;;
     SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*)
@@ -801,6 +820,9 @@ EOF
     i*:CYGWIN*:*)
 	echo ${UNAME_MACHINE}-pc-cygwin
 	exit ;;
+    *:MINGW64*:*)
+	echo ${UNAME_MACHINE}-pc-mingw64
+	exit ;;
     *:MINGW*:*)
 	echo ${UNAME_MACHINE}-pc-mingw32
 	exit ;;
@@ -852,21 +874,21 @@ EOF
 	exit ;;
     *:GNU:*:*)
 	# the GNU system
-	echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
 	exit ;;
     *:GNU/*:*:*)
 	# other systems with GNU libc and userland
-	echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu
+	echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
 	exit ;;
     i*86:Minix:*:*)
 	echo ${UNAME_MACHINE}-pc-minix
 	exit ;;
     aarch64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     aarch64_be:Linux:*:*)
 	UNAME_MACHINE=aarch64_be
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     alpha:Linux:*:*)
 	case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
@@ -879,59 +901,54 @@ EOF
 	  EV68*) UNAME_MACHINE=alphaev68 ;;
 	esac
 	objdump --private-headers /bin/sh | grep -q ld.so.1
-	if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
-	echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
+	if test "$?" = 0 ; then LIBC="gnulibc1" ; fi
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
+    arc:Linux:*:* | arceb:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     arm*:Linux:*:*)
 	eval $set_cc_for_build
 	if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \
 	    | grep -q __ARM_EABI__
 	then
-	    echo ${UNAME_MACHINE}-unknown-linux-gnu
+	    echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	else
 	    if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
 		| grep -q __ARM_PCS_VFP
 	    then
-		echo ${UNAME_MACHINE}-unknown-linux-gnueabi
+		echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi
 	    else
-		echo ${UNAME_MACHINE}-unknown-linux-gnueabihf
+		echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf
 	    fi
 	fi
 	exit ;;
     avr32*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     cris:Linux:*:*)
-	echo ${UNAME_MACHINE}-axis-linux-gnu
+	echo ${UNAME_MACHINE}-axis-linux-${LIBC}
 	exit ;;
     crisv32:Linux:*:*)
-	echo ${UNAME_MACHINE}-axis-linux-gnu
+	echo ${UNAME_MACHINE}-axis-linux-${LIBC}
 	exit ;;
     frv:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     hexagon:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     i*86:Linux:*:*)
-	LIBC=gnu
-	eval $set_cc_for_build
-	sed 's/^	//' << EOF >$dummy.c
-	#ifdef __dietlibc__
-	LIBC=dietlibc
-	#endif
-EOF
-	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'`
-	echo "${UNAME_MACHINE}-pc-linux-${LIBC}"
+	echo ${UNAME_MACHINE}-pc-linux-${LIBC}
 	exit ;;
     ia64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     m32r*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     m68*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     mips:Linux:*:* | mips64:Linux:*:*)
 	eval $set_cc_for_build
@@ -950,54 +967,63 @@ EOF
 	#endif
 EOF
 	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
-	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; }
+	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; }
 	;;
+    or1k:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
     or32:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     padre:Linux:*:*)
-	echo sparc-unknown-linux-gnu
+	echo sparc-unknown-linux-${LIBC}
 	exit ;;
     parisc64:Linux:*:* | hppa64:Linux:*:*)
-	echo hppa64-unknown-linux-gnu
+	echo hppa64-unknown-linux-${LIBC}
 	exit ;;
     parisc:Linux:*:* | hppa:Linux:*:*)
 	# Look for CPU level
 	case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in
-	  PA7*) echo hppa1.1-unknown-linux-gnu ;;
-	  PA8*) echo hppa2.0-unknown-linux-gnu ;;
-	  *)    echo hppa-unknown-linux-gnu ;;
+	  PA7*) echo hppa1.1-unknown-linux-${LIBC} ;;
+	  PA8*) echo hppa2.0-unknown-linux-${LIBC} ;;
+	  *)    echo hppa-unknown-linux-${LIBC} ;;
 	esac
 	exit ;;
     ppc64:Linux:*:*)
-	echo powerpc64-unknown-linux-gnu
+	echo powerpc64-unknown-linux-${LIBC}
 	exit ;;
     ppc:Linux:*:*)
-	echo powerpc-unknown-linux-gnu
+	echo powerpc-unknown-linux-${LIBC}
+	exit ;;
+    ppc64le:Linux:*:*)
+	echo powerpc64le-unknown-linux-${LIBC}
+	exit ;;
+    ppcle:Linux:*:*)
+	echo powerpcle-unknown-linux-${LIBC}
 	exit ;;
     s390:Linux:*:* | s390x:Linux:*:*)
-	echo ${UNAME_MACHINE}-ibm-linux
+	echo ${UNAME_MACHINE}-ibm-linux-${LIBC}
 	exit ;;
     sh64*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     sh*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     sparc:Linux:*:* | sparc64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     tile*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     vax:Linux:*:*)
-	echo ${UNAME_MACHINE}-dec-linux-gnu
+	echo ${UNAME_MACHINE}-dec-linux-${LIBC}
 	exit ;;
     x86_64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     xtensa*:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     i*86:DYNIX/ptx:4*:*)
 	# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.
@@ -1201,6 +1227,9 @@ EOF
     BePC:Haiku:*:*)	# Haiku running on Intel PC compatible.
 	echo i586-pc-haiku
 	exit ;;
+    x86_64:Haiku:*:*)
+	echo x86_64-unknown-haiku
+	exit ;;
     SX-4:SUPER-UX:*:*)
 	echo sx4-nec-superux${UNAME_RELEASE}
 	exit ;;
@@ -1227,19 +1256,21 @@ EOF
 	exit ;;
     *:Darwin:*:*)
 	UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown
-	case $UNAME_PROCESSOR in
-	    i386)
-		eval $set_cc_for_build
-		if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
-		  if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
-		      (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
-		      grep IS_64BIT_ARCH >/dev/null
-		  then
-		      UNAME_PROCESSOR="x86_64"
-		  fi
-		fi ;;
-	    unknown) UNAME_PROCESSOR=powerpc ;;
-	esac
+	eval $set_cc_for_build
+	if test "$UNAME_PROCESSOR" = unknown ; then
+	    UNAME_PROCESSOR=powerpc
+	fi
+	if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
+	    if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
+		(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
+		grep IS_64BIT_ARCH >/dev/null
+	    then
+		case $UNAME_PROCESSOR in
+		    i386) UNAME_PROCESSOR=x86_64 ;;
+		    powerpc) UNAME_PROCESSOR=powerpc64 ;;
+		esac
+	    fi
+	fi
 	echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE}
 	exit ;;
     *:procnto*:*:* | *:QNX:[0123456789]*:*)
@@ -1256,7 +1287,7 @@ EOF
     NEO-?:NONSTOP_KERNEL:*:*)
 	echo neo-tandem-nsk${UNAME_RELEASE}
 	exit ;;
-    NSE-?:NONSTOP_KERNEL:*:*)
+    NSE-*:NONSTOP_KERNEL:*:*)
 	echo nse-tandem-nsk${UNAME_RELEASE}
 	exit ;;
     NSR-?:NONSTOP_KERNEL:*:*)
@@ -1330,9 +1361,6 @@ EOF
 	exit ;;
 esac
 
-#echo '(No uname command or uname output not recognized.)' 1>&2
-#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
-
 eval $set_cc_for_build
 cat >$dummy.c <<EOF
 #ifdef _SEQUENT_

config.sub

@@ -1,24 +1,18 @@
 #! /bin/sh
 # Configuration validation subroutine script.
-#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-#   2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-#   2011, 2012 Free Software Foundation, Inc.
+#   Copyright 1992-2013 Free Software Foundation, Inc.
 
-timestamp='2012-02-10'
+timestamp='2013-10-01'
 
-# This file is (in principle) common to ALL GNU software.
-# The presence of a machine in this file suggests that SOME GNU software
-# can handle that machine.  It does not imply ALL GNU software can.
-#
-# This file is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
@@ -26,11 +20,12 @@ timestamp='2012-02-10'
 # As a special exception to the GNU General Public License, if you
 # distribute this file as part of a program that contains a
 # configuration script generated by Autoconf, you may include it under
-# the same distribution terms that you use for the rest of that program.
+# the same distribution terms that you use for the rest of that
+# program.  This Exception is an additional permission under section 7
+# of the GNU General Public License, version 3 ("GPLv3").
 
 
-# Please send patches to <config-patches@gnu.org>.  Submit a context
-# diff and a properly formatted GNU ChangeLog entry.
+# Please send patches with a ChangeLog entry to config-patches@gnu.org.
 #
 # Configuration subroutine to validate and canonicalize a configuration type.
 # Supply the specified configuration type as an argument.
@@ -73,9 +68,7 @@ Report bugs and patches to <config-patches@gnu.org>."
 version="\
 GNU config.sub ($timestamp)
 
-Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
-Free Software Foundation, Inc.
+Copyright 1992-2013 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -123,7 +116,7 @@ esac
 maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
 case $maybe_os in
   nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
-  linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
+  linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
   knetbsd*-gnu* | netbsd*-gnu* | \
   kopensolaris*-gnu* | \
   storm-chaos* | os2-emx* | rtmk-nova*)
@@ -156,7 +149,7 @@ case $os in
 	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
 	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
 	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
-	-apple | -axis | -knuth | -cray | -microblaze)
+	-apple | -axis | -knuth | -cray | -microblaze*)
 		os=
 		basic_machine=$1
 		;;
@@ -225,6 +218,12 @@ case $os in
 	-isc*)
 		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
 		;;
+	-lynx*178)
+		os=-lynxos178
+		;;
+	-lynx*5)
+		os=-lynxos5
+		;;
 	-lynx*)
 		os=-lynxos
 		;;
@@ -253,10 +252,12 @@ case $basic_machine in
 	| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
 	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
 	| am33_2.0 \
-	| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \
-        | be32 | be64 \
+	| arc | arceb \
+	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
+	| avr | avr32 \
+	| be32 | be64 \
 	| bfin \
-	| c4x | clipper \
+	| c4x | c8051 | clipper \
 	| d10v | d30v | dlx | dsp16xx \
 	| epiphany \
 	| fido | fr30 | frv \
@@ -264,10 +265,11 @@ case $basic_machine in
 	| hexagon \
 	| i370 | i860 | i960 | ia64 \
 	| ip2k | iq2000 \
+	| k1om \
 	| le32 | le64 \
 	| lm32 \
 	| m32c | m32r | m32rle | m68000 | m68k | m88k \
-	| maxq | mb | microblaze | mcore | mep | metag \
+	| maxq | mb | microblaze | microblazeel | mcore | mep | metag \
 	| mips | mipsbe | mipseb | mipsel | mipsle \
 	| mips16 \
 	| mips64 | mips64el \
@@ -285,16 +287,17 @@ case $basic_machine in
 	| mipsisa64r2 | mipsisa64r2el \
 	| mipsisa64sb1 | mipsisa64sb1el \
 	| mipsisa64sr71k | mipsisa64sr71kel \
+	| mipsr5900 | mipsr5900el \
 	| mipstx39 | mipstx39el \
 	| mn10200 | mn10300 \
 	| moxie \
 	| mt \
 	| msp430 \
 	| nds32 | nds32le | nds32be \
-	| nios | nios2 \
+	| nios | nios2 | nios2eb | nios2el \
 	| ns16k | ns32k \
 	| open8 \
-	| or32 \
+	| or1k | or32 \
 	| pdp10 | pdp11 | pj | pjl \
 	| powerpc | powerpc64 | powerpc64le | powerpcle \
 	| pyramid \
@@ -322,7 +325,7 @@ case $basic_machine in
 	c6x)
 		basic_machine=tic6x-unknown
 		;;
-	m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip)
+	m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip)
 		basic_machine=$basic_machine-unknown
 		os=-none
 		;;
@@ -364,13 +367,13 @@ case $basic_machine in
 	| aarch64-* | aarch64_be-* \
 	| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
 	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
-	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
+	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
 	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
 	| avr-* | avr32-* \
 	| be32-* | be64-* \
 	| bfin-* | bs2000-* \
 	| c[123]* | c30-* | [cjt]90-* | c4x-* \
-	| clipper-* | craynv-* | cydra-* \
+	| c8051-* | clipper-* | craynv-* | cydra-* \
 	| d10v-* | d30v-* | dlx-* \
 	| elxsi-* \
 	| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
@@ -379,11 +382,13 @@ case $basic_machine in
 	| hexagon-* \
 	| i*86-* | i860-* | i960-* | ia64-* \
 	| ip2k-* | iq2000-* \
+	| k1om-* \
 	| le32-* | le64-* \
 	| lm32-* \
 	| m32c-* | m32r-* | m32rle-* \
 	| m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
-	| m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \
+	| m88110-* | m88k-* | maxq-* | mcore-* | metag-* \
+	| microblaze-* | microblazeel-* \
 	| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
 	| mips16-* \
 	| mips64-* | mips64el-* \
@@ -401,12 +406,13 @@ case $basic_machine in
 	| mipsisa64r2-* | mipsisa64r2el-* \
 	| mipsisa64sb1-* | mipsisa64sb1el-* \
 	| mipsisa64sr71k-* | mipsisa64sr71kel-* \
+	| mipsr5900-* | mipsr5900el-* \
 	| mipstx39-* | mipstx39el-* \
 	| mmix-* \
 	| mt-* \
 	| msp430-* \
 	| nds32-* | nds32le-* | nds32be-* \
-	| nios-* | nios2-* \
+	| nios-* | nios2-* | nios2eb-* | nios2el-* \
 	| none-* | np1-* | ns16k-* | ns32k-* \
 	| open8-* \
 	| orion-* \
@@ -782,11 +788,15 @@ case $basic_machine in
 		basic_machine=ns32k-utek
 		os=-sysv
 		;;
-	microblaze)
+	microblaze*)
 		basic_machine=microblaze-xilinx
 		;;
+	mingw64)
+		basic_machine=x86_64-pc
+		os=-mingw64
+		;;
 	mingw32)
-		basic_machine=i386-pc
+		basic_machine=i686-pc
 		os=-mingw32
 		;;
 	mingw32ce)
@@ -822,7 +832,7 @@ case $basic_machine in
 		basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'`
 		;;
 	msys)
-		basic_machine=i386-pc
+		basic_machine=i686-pc
 		os=-msys
 		;;
 	mvs)
@@ -1013,7 +1023,11 @@ case $basic_machine in
 		basic_machine=i586-unknown
 		os=-pw32
 		;;
-	rdos)
+	rdos | rdos64)
+		basic_machine=x86_64-pc
+		os=-rdos
+		;;
+	rdos32)
 		basic_machine=i386-pc
 		os=-rdos
 		;;
@@ -1340,21 +1354,21 @@ case $os in
 	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
 	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
 	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
-	      | -sym* | -kopensolaris* \
+	      | -sym* | -kopensolaris* | -plan9* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
 	      | -aos* | -aros* \
 	      | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
 	      | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
 	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
-	      | -openbsd* | -solidbsd* \
+	      | -bitrig* | -openbsd* | -solidbsd* \
 	      | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
 	      | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
 	      | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
 	      | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
 	      | -chorusos* | -chorusrdb* | -cegcc* \
 	      | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
-	      | -mingw32* | -linux-gnu* | -linux-android* \
-	      | -linux-newlib* | -linux-uclibc* \
+	      | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
+	      | -linux-newlib* | -linux-musl* | -linux-uclibc* \
 	      | -uxpv* | -beos* | -mpeix* | -udk* \
 	      | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
 	      | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
@@ -1486,9 +1500,6 @@ case $os in
 	-aros*)
 		os=-aros
 		;;
-	-kaos*)
-		os=-kaos
-		;;
 	-zvmoe)
 		os=-zvmoe
 		;;
@@ -1537,6 +1548,12 @@ case $basic_machine in
 	c4x-* | tic4x-*)
 		os=-coff
 		;;
+	c8051-*)
+		os=-elf
+		;;
+	hexagon-*)
+		os=-elf
+		;;
 	tic54x-*)
 		os=-coff
 		;;
@@ -1577,6 +1594,9 @@ case $basic_machine in
 	mips*-*)
 		os=-elf
 		;;
+	or1k-*)
+		os=-elf
+		;;
 	or32-*)
 		os=-coff
 		;;

configure.ac

@@ -199,6 +199,7 @@ case "${host_cpu}" in
 	if test "x${je_cv_asm}" = "xyes" ; then
 	    CPU_SPINWAIT='__asm__ volatile("pause")'
 	fi
+	AC_DEFINE_UNQUOTED([HAVE_SSE2], [ ])
 	;;
   x86_64)
 	JE_COMPILABLE([__asm__ syntax], [],
@@ -206,6 +207,10 @@ case "${host_cpu}" in
 	if test "x${je_cv_asm}" = "xyes" ; then
 	    CPU_SPINWAIT='__asm__ volatile("pause")'
 	fi
+	AC_DEFINE_UNQUOTED([HAVE_SSE2], [ ])
+	;;
+  powerpc)
+	AC_DEFINE_UNQUOTED([HAVE_ALTIVEC], [ ])
 	;;
   *)
 	;;
@@ -256,6 +261,7 @@ case "${host}" in
 	force_tls="0"
 	DSO_LDFLAGS='-shared -Wl,-dylib_install_name,$(@F)'
 	SOREV="${rev}.${so}"
+	sbrk_deprecated="1"
 	;;
   *-*-freebsd*)
 	CFLAGS="$CFLAGS"
@@ -355,11 +361,6 @@ AC_SUBST([ARFLAGS])
 AC_SUBST([AROUT])
 AC_SUBST([CC_MM])
 
-if test "x$abi" != "xpecoff"; then
-  dnl Heap profiling uses the log(3) function.
-  LIBS="$LIBS -lm"
-fi
-
 JE_COMPILABLE([__attribute__ syntax],
               [static __attribute__((unused)) void foo(void){}],
               [],
@@ -416,7 +417,7 @@ AC_PROG_RANLIB
 AC_PATH_PROG([LD], [ld], [false], [$PATH])
 AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH])
 
-public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free malloc_usable_size malloc_stats_print mallctl mallctlnametomib mallctlbymib"
+public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free mallocx rallocx xallocx sallocx dallocx nallocx mallctl mallctlnametomib mallctlbymib malloc_stats_print malloc_usable_size"
 
 dnl Check for allocator-related functions that should be wrapped.
 AC_CHECK_FUNC([memalign],
@@ -444,18 +445,35 @@ if test "x$enable_experimental" = "x1" ; then
 fi
 AC_SUBST([enable_experimental])
 
+dnl Do not compute test code coverage by default.
+GCOV_FLAGS=
+AC_ARG_ENABLE([code-coverage],
+  [AS_HELP_STRING([--enable-code-coverage],
+   [Enable code coverage])],
+[if test "x$enable_code_coverage" = "xno" ; then
+  enable_code_coverage="0"
+else
+  enable_code_coverage="1"
+fi
+],
+[enable_code_coverage="0"]
+)
+if test "x$enable_code_coverage" = "x1" ; then
+  deoptimize="no"
+  echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || deoptimize="yes"
+  if test "x${deoptimize}" = "xyes" ; then
+    JE_CFLAGS_APPEND([-O0])
+  fi
+  JE_CFLAGS_APPEND([-fprofile-arcs -ftest-coverage])
+  EXTRA_LDFLAGS="$EXTRA_LDFLAGS -fprofile-arcs -ftest-coverage"
+  AC_DEFINE([JEMALLOC_CODE_COVERAGE], [ ])
+fi
+AC_SUBST([enable_code_coverage])
+
 dnl Perform no name mangling by default.
 AC_ARG_WITH([mangling],
   [AS_HELP_STRING([--with-mangling=<map>], [Mangle symbols in <map>])],
   [mangling_map="$with_mangling"], [mangling_map=""])
-for nm in `echo ${mangling_map} |tr ',' ' '` ; do
-  k="`echo ${nm} |tr ':' ' ' |awk '{print $1}'`"
-  n="je_${k}"
-  m=`echo ${nm} |tr ':' ' ' |awk '{print $2}'`
-  AC_DEFINE_UNQUOTED([${n}], [${m}])
-  dnl Remove key from public_syms so that it isn't redefined later.
-  public_syms=`for sym in ${public_syms}; do echo "${sym}"; done |grep -v "^${k}\$" |tr '\n' ' '`
-done
 
 dnl Do not prefix public APIs by default.
 AC_ARG_WITH([jemalloc_prefix],
@@ -472,14 +490,6 @@ if test "x$JEMALLOC_PREFIX" != "x" ; then
   AC_DEFINE_UNQUOTED([JEMALLOC_PREFIX], ["$JEMALLOC_PREFIX"])
   AC_DEFINE_UNQUOTED([JEMALLOC_CPREFIX], ["$JEMALLOC_CPREFIX"])
 fi
-dnl Generate macros to rename public symbols.  All public symbols are prefixed
-dnl with je_ in the source code, so these macro definitions are needed even if
-dnl --with-jemalloc-prefix wasn't specified.
-for stem in ${public_syms}; do
-  n="je_${stem}"
-  m="${JEMALLOC_PREFIX}${stem}"
-  AC_DEFINE_UNQUOTED([${n}], [${m}])
-done
 
 AC_ARG_WITH([export],
   [AS_HELP_STRING([--without-export], [disable exporting jemalloc public APIs])],
@@ -488,18 +498,15 @@ AC_ARG_WITH([export],
 fi]
 )
 
-dnl Do not mangle library-private APIs by default.
+dnl Mangle library-private APIs.
 AC_ARG_WITH([private_namespace],
   [AS_HELP_STRING([--with-private-namespace=<prefix>], [Prefix to prepend to all library-private APIs])],
-  [JEMALLOC_PRIVATE_NAMESPACE="$with_private_namespace"],
-  [JEMALLOC_PRIVATE_NAMESPACE=""]
+  [JEMALLOC_PRIVATE_NAMESPACE="${with_private_namespace}je_"],
+  [JEMALLOC_PRIVATE_NAMESPACE="je_"]
 )
-AC_DEFINE_UNQUOTED([JEMALLOC_PRIVATE_NAMESPACE], ["$JEMALLOC_PRIVATE_NAMESPACE"])
-if test "x$JEMALLOC_PRIVATE_NAMESPACE" != "x" ; then
-  AC_DEFINE_UNQUOTED([JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix)], [${JEMALLOC_PRIVATE_NAMESPACE}##string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix])
-else
-  AC_DEFINE_UNQUOTED([JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix)], [string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix])
-fi
+AC_DEFINE_UNQUOTED([JEMALLOC_PRIVATE_NAMESPACE], [$JEMALLOC_PRIVATE_NAMESPACE])
+private_namespace="$JEMALLOC_PRIVATE_NAMESPACE"
+AC_SUBST([private_namespace])
 
 dnl Do not add suffix to installed files by default.
 AC_ARG_WITH([install_suffix],
@@ -510,37 +517,72 @@ AC_ARG_WITH([install_suffix],
 install_suffix="$INSTALL_SUFFIX"
 AC_SUBST([install_suffix])
 
+dnl Substitute @je_@ in jemalloc_protos.h.in, primarily to make generation of
+dnl jemalloc_protos_jet.h easy.
+je_="je_"
+AC_SUBST([je_])
+
 cfgoutputs_in="${srcroot}Makefile.in"
 cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/html.xsl.in"
 cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/manpages.xsl.in"
 cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/jemalloc.xml.in"
-cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc.h.in"
+cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc_macros.h.in"
+cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc_protos.h.in"
 cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal.h.in"
-cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/jemalloc_test.h.in"
+cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/test.sh.in"
+cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/include/test/jemalloc_test.h.in"
 
 cfgoutputs_out="Makefile"
 cfgoutputs_out="${cfgoutputs_out} doc/html.xsl"
 cfgoutputs_out="${cfgoutputs_out} doc/manpages.xsl"
-cfgoutputs_out="${cfgoutputs_out} doc/jemalloc${install_suffix}.xml"
-cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc${install_suffix}.h"
+cfgoutputs_out="${cfgoutputs_out} doc/jemalloc.xml"
+cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_macros.h"
+cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_protos.h"
 cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal.h"
-cfgoutputs_out="${cfgoutputs_out} test/jemalloc_test.h"
+cfgoutputs_out="${cfgoutputs_out} test/test.sh"
+cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h"
 
 cfgoutputs_tup="Makefile"
 cfgoutputs_tup="${cfgoutputs_tup} doc/html.xsl:doc/html.xsl.in"
 cfgoutputs_tup="${cfgoutputs_tup} doc/manpages.xsl:doc/manpages.xsl.in"
-cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc${install_suffix}.xml:doc/jemalloc.xml.in"
-cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc${install_suffix}.h:include/jemalloc/jemalloc.h.in"
+cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc.xml:doc/jemalloc.xml.in"
+cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_macros.h:include/jemalloc/jemalloc_macros.h.in"
+cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_protos.h:include/jemalloc/jemalloc_protos.h.in"
 cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h"
-cfgoutputs_tup="${cfgoutputs_tup} test/jemalloc_test.h:test/jemalloc_test.h.in"
+cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in"
+cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in"
 
 cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_namespace.sh"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_unnamespace.sh"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_symbols.txt"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/public_namespace.sh"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/public_unnamespace.sh"
 cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/size_classes.sh"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc_rename.sh"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc_mangle.sh"
+cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc.sh"
+cfghdrs_in="${cfghdrs_in} ${srcroot}test/include/test/jemalloc_test_defs.h.in"
 
-cfghdrs_out="include/jemalloc/jemalloc_defs${install_suffix}.h"
+cfghdrs_out="include/jemalloc/jemalloc_defs.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc${install_suffix}.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_namespace.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_unnamespace.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_symbols.txt"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_namespace.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_unnamespace.h"
 cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_protos_jet.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_rename.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle_jet.h"
+cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/jemalloc_internal_defs.h"
+cfghdrs_out="${cfghdrs_out} test/include/test/jemalloc_test_defs.h"
 
-cfghdrs_tup="include/jemalloc/jemalloc_defs${install_suffix}.h:include/jemalloc/jemalloc_defs.h.in"
+cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.in"
+cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in"
+cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:${srcroot}test/include/test/jemalloc_test_defs.h.in"
 
 dnl Do not silence irrelevant compiler warnings by default, since enabling this
 dnl option incurs a performance penalty.
@@ -595,7 +637,7 @@ dnl Only optimize if not debugging.
 if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then
   dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS.
   optimize="no"
-  echo "$EXTRA_CFLAGS" | grep "\-O" >/dev/null || optimize="yes"
+  echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || optimize="yes"
   if test "x${optimize}" = "xyes" ; then
     if test "x$GCC" = "xyes" ; then
       JE_CFLAGS_APPEND([-O3])
@@ -748,6 +790,12 @@ if test "x$enable_prof" = "x1" ; then
     AC_MSG_ERROR([Heap profiling requires TLS]);
   fi
   force_tls="1"
+
+  if test "x$abi" != "xpecoff"; then
+    dnl Heap profiling uses the log(3) function.
+    LIBS="$LIBS -lm"
+  fi
+
   AC_DEFINE([JEMALLOC_PROF], [ ])
 fi
 AC_SUBST([enable_prof])
@@ -781,7 +829,7 @@ fi
 )
 if test "x$enable_mremap" = "x1" ; then
   JE_COMPILABLE([mremap(...MREMAP_FIXED...)], [
-#define _GNU_SOURCE
+#define	_GNU_SOURCE
 #include <sys/mman.h>
 ], [
 void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0);
@@ -825,7 +873,12 @@ fi
 dnl Check whether the BSD/SUSv1 sbrk() exists.  If not, disable DSS support.
 AC_CHECK_FUNC([sbrk], [have_sbrk="1"], [have_sbrk="0"])
 if test "x$have_sbrk" = "x1" ; then
-  AC_DEFINE([JEMALLOC_HAVE_SBRK], [ ])
+  if test "x$sbrk_deprecated" == "x1" ; then
+    AC_MSG_RESULT([Disabling dss allocation because sbrk is deprecated])
+    enable_dss="0"
+  else
+    AC_DEFINE([JEMALLOC_HAVE_SBRK], [ ])
+  fi
 else
   enable_dss="0"
 fi
@@ -1271,9 +1324,102 @@ dnl ============================================================================
 dnl Check for typedefs, structures, and compiler characteristics.
 AC_HEADER_STDBOOL
 
+dnl ============================================================================
+dnl Define commands that generate output files.
+
+AC_CONFIG_COMMANDS([include/jemalloc/internal/private_namespace.h], [
+  mkdir -p "${objroot}include/jemalloc/internal"
+  "${srcdir}/include/jemalloc/internal/private_namespace.sh" "${srcdir}/include/jemalloc/internal/private_symbols.txt" > "${objroot}include/jemalloc/internal/private_namespace.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/internal/private_unnamespace.h], [
+  mkdir -p "${objroot}include/jemalloc/internal"
+  "${srcdir}/include/jemalloc/internal/private_unnamespace.sh" "${srcdir}/include/jemalloc/internal/private_symbols.txt" > "${objroot}include/jemalloc/internal/private_unnamespace.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/internal/public_symbols.txt], [
+  f="${objroot}include/jemalloc/internal/public_symbols.txt"
+  mkdir -p "${objroot}include/jemalloc/internal"
+  cp /dev/null "${f}"
+  for nm in `echo ${mangling_map} |tr ',' ' '` ; do
+    n=`echo ${nm} |tr ':' ' ' |awk '{print $[]1}'`
+    m=`echo ${nm} |tr ':' ' ' |awk '{print $[]2}'`
+    echo "${n}:${m}" >> "${f}"
+    dnl Remove name from public_syms so that it isn't redefined later.
+    public_syms=`for sym in ${public_syms}; do echo "${sym}"; done |grep -v "^${n}\$" |tr '\n' ' '`
+  done
+  for sym in ${public_syms} ; do
+    n="${sym}"
+    m="${JEMALLOC_PREFIX}${sym}"
+    echo "${n}:${m}" >> "${f}"
+  done
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+  mangling_map="${mangling_map}"
+  public_syms="${public_syms}"
+  JEMALLOC_PREFIX="${JEMALLOC_PREFIX}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/internal/public_namespace.h], [
+  mkdir -p "${objroot}include/jemalloc/internal"
+  "${srcdir}/include/jemalloc/internal/public_namespace.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/internal/public_namespace.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [
+  mkdir -p "${objroot}include/jemalloc/internal"
+  "${srcdir}/include/jemalloc/internal/public_unnamespace.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/internal/public_unnamespace.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
 AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [
-  mkdir -p "include/jemalloc/internal"
+  mkdir -p "${objroot}include/jemalloc/internal"
   "${srcdir}/include/jemalloc/internal/size_classes.sh" > "${objroot}include/jemalloc/internal/size_classes.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_protos_jet.h], [
+  mkdir -p "${objroot}include/jemalloc"
+  cat "${srcdir}/include/jemalloc/jemalloc_protos.h.in" | sed -e 's/@je_@/jet_/g' > "${objroot}include/jemalloc/jemalloc_protos_jet.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_rename.h], [
+  mkdir -p "${objroot}include/jemalloc"
+  "${srcdir}/include/jemalloc/jemalloc_rename.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" > "${objroot}include/jemalloc/jemalloc_rename.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_mangle.h], [
+  mkdir -p "${objroot}include/jemalloc"
+  "${srcdir}/include/jemalloc/jemalloc_mangle.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" je_ > "${objroot}include/jemalloc/jemalloc_mangle.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_mangle_jet.h], [
+  mkdir -p "${objroot}include/jemalloc"
+  "${srcdir}/include/jemalloc/jemalloc_mangle.sh" "${objroot}include/jemalloc/internal/public_symbols.txt" jet_ > "${objroot}include/jemalloc/jemalloc_mangle_jet.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+])
+AC_CONFIG_COMMANDS([include/jemalloc/jemalloc.h], [
+  mkdir -p "${objroot}include/jemalloc"
+  "${srcdir}/include/jemalloc/jemalloc.sh" "${objroot}" > "${objroot}include/jemalloc/jemalloc${install_suffix}.h"
+], [
+  srcdir="${srcdir}"
+  objroot="${objroot}"
+  install_suffix="${install_suffix}"
 ])
 
 dnl Process .in files.
@@ -1283,6 +1429,7 @@ AC_CONFIG_HEADERS([$cfghdrs_tup])
 
 dnl ============================================================================
 dnl Generate outputs.
+
 AC_CONFIG_FILES([$cfgoutputs_tup config.stamp bin/jemalloc.sh])
 AC_SUBST([cfgoutputs_in])
 AC_SUBST([cfgoutputs_out])
@@ -1298,6 +1445,7 @@ AC_MSG_RESULT([CC                 : ${CC}])
 AC_MSG_RESULT([CPPFLAGS           : ${CPPFLAGS}])
 AC_MSG_RESULT([CFLAGS             : ${CFLAGS}])
 AC_MSG_RESULT([LDFLAGS            : ${LDFLAGS}])
+AC_MSG_RESULT([EXTRA_LDFLAGS      : ${EXTRA_LDFLAGS}])
 AC_MSG_RESULT([LIBS               : ${LIBS}])
 AC_MSG_RESULT([RPATH_EXTRA        : ${RPATH_EXTRA}])
 AC_MSG_RESULT([])
@@ -1324,6 +1472,7 @@ AC_MSG_RESULT([autogen            : ${enable_autogen}])
 AC_MSG_RESULT([experimental       : ${enable_experimental}])
 AC_MSG_RESULT([cc-silence         : ${enable_cc_silence}])
 AC_MSG_RESULT([debug              : ${enable_debug}])
+AC_MSG_RESULT([code-coverage      : ${enable_code_coverage}])
 AC_MSG_RESULT([stats              : ${enable_stats}])
 AC_MSG_RESULT([prof               : ${enable_prof}])
 AC_MSG_RESULT([prof-libunwind     : ${enable_prof_libunwind}])

coverage.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+objdir=$1
+suffix=$2
+shift 2
+objs=$@
+
+gcov -b -p -f -o "${objdir}" ${objs}
+
+# Move gcov outputs so that subsequent gcov invocations won't clobber results
+# for the same sources with different compilation flags.
+for f in `find . -maxdepth 1 -type f -name '*.gcov'` ; do
+  mv "${f}" "${f}.${suffix}"
+done

doc/jemalloc.xml.in

@@ -33,11 +33,17 @@
     <refname>aligned_alloc</refname>
     <refname>realloc</refname>
     <refname>free</refname>
-    <refname>malloc_usable_size</refname>
-    <refname>malloc_stats_print</refname>
+    <refname>mallocx</refname>
+    <refname>rallocx</refname>
+    <refname>xallocx</refname>
+    <refname>sallocx</refname>
+    <refname>dallocx</refname>
+    <refname>nallocx</refname>
     <refname>mallctl</refname>
     <refname>mallctlnametomib</refname>
     <refname>mallctlbymib</refname>
+    <refname>malloc_stats_print</refname>
+    <refname>malloc_usable_size</refname>
     <refname>allocm</refname>
     <refname>rallocm</refname>
     <refname>sallocm</refname>
@@ -92,16 +98,37 @@
       <refsect2>
         <title>Non-standard API</title>
         <funcprototype>
-          <funcdef>size_t <function>malloc_usable_size</function></funcdef>
-          <paramdef>const void *<parameter>ptr</parameter></paramdef>
+          <funcdef>void *<function>mallocx</function></funcdef>
+          <paramdef>size_t <parameter>size</parameter></paramdef>
+          <paramdef>int <parameter>flags</parameter></paramdef>
         </funcprototype>
         <funcprototype>
-          <funcdef>void <function>malloc_stats_print</function></funcdef>
-          <paramdef>void <parameter>(*write_cb)</parameter>
-            <funcparams>void *, const char *</funcparams>
-          </paramdef>
-          <paramdef>void *<parameter>cbopaque</parameter></paramdef>
-          <paramdef>const char *<parameter>opts</parameter></paramdef>
+          <funcdef>void *<function>rallocx</function></funcdef>
+          <paramdef>void *<parameter>ptr</parameter></paramdef>
+          <paramdef>size_t <parameter>size</parameter></paramdef>
+          <paramdef>int <parameter>flags</parameter></paramdef>
+        </funcprototype>
+        <funcprototype>
+          <funcdef>size_t <function>xallocx</function></funcdef>
+          <paramdef>void *<parameter>ptr</parameter></paramdef>
+          <paramdef>size_t <parameter>size</parameter></paramdef>
+          <paramdef>size_t <parameter>extra</parameter></paramdef>
+          <paramdef>int <parameter>flags</parameter></paramdef>
+        </funcprototype>
+        <funcprototype>
+          <funcdef>size_t <function>sallocx</function></funcdef>
+          <paramdef>void *<parameter>ptr</parameter></paramdef>
+          <paramdef>int <parameter>flags</parameter></paramdef>
+        </funcprototype>
+        <funcprototype>
+          <funcdef>void <function>dallocx</function></funcdef>
+          <paramdef>void *<parameter>ptr</parameter></paramdef>
+          <paramdef>int <parameter>flags</parameter></paramdef>
+        </funcprototype>
+        <funcprototype>
+          <funcdef>size_t <function>nallocx</function></funcdef>
+          <paramdef>size_t <parameter>size</parameter></paramdef>
+          <paramdef>int <parameter>flags</parameter></paramdef>
         </funcprototype>
         <funcprototype>
           <funcdef>int <function>mallctl</function></funcdef>
@@ -126,6 +153,18 @@
           <paramdef>void *<parameter>newp</parameter></paramdef>
           <paramdef>size_t <parameter>newlen</parameter></paramdef>
         </funcprototype>
+        <funcprototype>
+          <funcdef>void <function>malloc_stats_print</function></funcdef>
+          <paramdef>void <parameter>(*write_cb)</parameter>
+            <funcparams>void *, const char *</funcparams>
+          </paramdef>
+          <paramdef>void *<parameter>cbopaque</parameter></paramdef>
+          <paramdef>const char *<parameter>opts</parameter></paramdef>
+        </funcprototype>
+        <funcprototype>
+          <funcdef>size_t <function>malloc_usable_size</function></funcdef>
+          <paramdef>const void *<parameter>ptr</parameter></paramdef>
+        </funcprototype>
         <funcprototype>
           <funcdef>void <function>(*malloc_message)</function></funcdef>
           <paramdef>void *<parameter>cbopaque</parameter></paramdef>
@@ -225,41 +264,102 @@
     </refsect2>
     <refsect2>
       <title>Non-standard API</title>
+      <para>The <function>mallocx<parameter/></function>,
+      <function>rallocx<parameter/></function>,
+      <function>xallocx<parameter/></function>,
+      <function>sallocx<parameter/></function>,
+      <function>dallocx<parameter/></function>, and
+      <function>nallocx<parameter/></function> functions all have a
+      <parameter>flags</parameter> argument that can be used to specify
+      options.  The functions only check the options that are contextually
+      relevant.  Use bitwise or (<code language="C">|</code>) operations to
+      specify one or more of the following:
+        <variablelist>
+          <varlistentry>
+            <term><constant>MALLOCX_LG_ALIGN(<parameter>la</parameter>)
+            </constant></term>
 
-      <para>The <function>malloc_usable_size<parameter/></function> function
-      returns the usable size of the allocation pointed to by
-      <parameter>ptr</parameter>.  The return value may be larger than the size
-      that was requested during allocation.  The
-      <function>malloc_usable_size<parameter/></function> function is not a
-      mechanism for in-place <function>realloc<parameter/></function>; rather
-      it is provided solely as a tool for introspection purposes.  Any
-      discrepancy between the requested allocation size and the size reported
-      by <function>malloc_usable_size<parameter/></function> should not be
-      depended on, since such behavior is entirely implementation-dependent.
+            <listitem><para>Align the memory allocation to start at an address
+            that is a multiple of <code language="C">(1 &lt;&lt;
+            <parameter>la</parameter>)</code>.  This macro does not validate
+            that <parameter>la</parameter> is within the valid
+            range.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term><constant>MALLOCX_ALIGN(<parameter>a</parameter>)
+            </constant></term>
+
+            <listitem><para>Align the memory allocation to start at an address
+            that is a multiple of <parameter>a</parameter>, where
+            <parameter>a</parameter> is a power of two.  This macro does not
+            validate that <parameter>a</parameter> is a power of 2.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term><constant>MALLOCX_ZERO</constant></term>
+
+            <listitem><para>Initialize newly allocated memory to contain zero
+            bytes.  In the growing reallocation case, the real size prior to
+            reallocation defines the boundary between untouched bytes and those
+            that are initialized to contain zero bytes.  If this macro is
+            absent, newly allocated memory is uninitialized.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term><constant>MALLOCX_ARENA(<parameter>a</parameter>)
+            </constant></term>
+
+            <listitem><para>Use the arena specified by the index
+            <parameter>a</parameter> (and by necessity bypass the thread
+            cache).  This macro has no effect for huge regions, nor for regions
+            that were allocated via an arena other than the one specified.
+            This macro does not validate that <parameter>a</parameter>
+            specifies an arena index in the valid range.</para></listitem>
+          </varlistentry>
+        </variablelist>
       </para>
 
-      <para>The <function>malloc_stats_print<parameter/></function> function
-      writes human-readable summary statistics via the
-      <parameter>write_cb</parameter> callback function pointer and
-      <parameter>cbopaque</parameter> data passed to
-      <parameter>write_cb</parameter>, or
-      <function>malloc_message<parameter/></function> if
-      <parameter>write_cb</parameter> is <constant>NULL</constant>.  This
-      function can be called repeatedly.  General information that never
-      changes during execution can be omitted by specifying "g" as a character
-      within the <parameter>opts</parameter> string.  Note that
-      <function>malloc_message<parameter/></function> uses the
-      <function>mallctl*<parameter/></function> functions internally, so
-      inconsistent statistics can be reported if multiple threads use these
-      functions simultaneously.  If <option>--enable-stats</option> is
-      specified during configuration, &ldquo;m&rdquo; and &ldquo;a&rdquo; can
-      be specified to omit merged arena and per arena statistics, respectively;
-      &ldquo;b&rdquo; and &ldquo;l&rdquo; can be specified to omit per size
-      class statistics for bins and large objects, respectively.  Unrecognized
-      characters are silently ignored.  Note that thread caching may prevent
-      some statistics from being completely up to date, since extra locking
-      would be required to merge counters that track thread cache operations.
-      </para>
+      <para>The <function>mallocx<parameter/></function> function allocates at
+      least <parameter>size</parameter> bytes of memory, and returns a pointer
+      to the base address of the allocation.  Behavior is undefined if
+      <parameter>size</parameter> is <constant>0</constant>, or if request size
+      overflows due to size class and/or alignment constraints.</para>
+
+      <para>The <function>rallocx<parameter/></function> function resizes the
+      allocation at <parameter>ptr</parameter> to be at least
+      <parameter>size</parameter> bytes, and returns a pointer to the base
+      address of the resulting allocation, which may or may not have moved from
+      its original location.  Behavior is undefined if
+      <parameter>size</parameter> is <constant>0</constant>, or if request size
+      overflows due to size class and/or alignment constraints.</para>
+
+      <para>The <function>xallocx<parameter/></function> function resizes the
+      allocation at <parameter>ptr</parameter> in place to be at least
+      <parameter>size</parameter> bytes, and returns the real size of the
+      allocation.  If <parameter>extra</parameter> is non-zero, an attempt is
+      made to resize the allocation to be at least <code
+      language="C">(<parameter>size</parameter> +
+      <parameter>extra</parameter>)</code> bytes, though inability to allocate
+      the extra byte(s) will not by itself result in failure to resize.
+      Behavior is undefined if <parameter>size</parameter> is
+      <constant>0</constant>, or if <code
+      language="C">(<parameter>size</parameter> + <parameter>extra</parameter>
+      &gt; <constant>SIZE_T_MAX</constant>)</code>.</para>
+
+      <para>The <function>sallocx<parameter/></function> function returns the
+      real size of the allocation at <parameter>ptr</parameter>.</para>
+
+      <para>The <function>dallocx<parameter/></function> function causes the
+      memory referenced by <parameter>ptr</parameter> to be made available for
+      future allocations.</para>
+
+      <para>The <function>nallocx<parameter/></function> function allocates no
+      memory, but it performs the same size computation as the
+      <function>mallocx<parameter/></function> function, and returns the real
+      size of the allocation that would result from the equivalent
+      <function>mallocx<parameter/></function> function call.  Behavior is
+      undefined if <parameter>size</parameter> is <constant>0</constant>, or if
+      request size overflows due to size class and/or alignment
+      constraints.</para>
 
       <para>The <function>mallctl<parameter/></function> function provides a
       general interface for introspecting the memory allocator, as well as
@@ -297,15 +397,14 @@
       it is legitimate to construct code like the following: <programlisting
       language="C"><![CDATA[
 unsigned nbins, i;
-
-int mib[4];
+size_t mib[4];
 size_t len, miblen;
 
 len = sizeof(nbins);
 mallctl("arenas.nbins", &nbins, &len, NULL, 0);
 
 miblen = 4;
-mallnametomib("arenas.bin.0.size", mib, &miblen);
+mallctlnametomib("arenas.bin.0.size", mib, &miblen);
 for (i = 0; i < nbins; i++) {
 	size_t bin_size;
 
@@ -314,6 +413,41 @@ for (i = 0; i < nbins; i++) {
 	mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0);
 	/* Do something with bin_size... */
 }]]></programlisting></para>
+
+      <para>The <function>malloc_stats_print<parameter/></function> function
+      writes human-readable summary statistics via the
+      <parameter>write_cb</parameter> callback function pointer and
+      <parameter>cbopaque</parameter> data passed to
+      <parameter>write_cb</parameter>, or
+      <function>malloc_message<parameter/></function> if
+      <parameter>write_cb</parameter> is <constant>NULL</constant>.  This
+      function can be called repeatedly.  General information that never
+      changes during execution can be omitted by specifying "g" as a character
+      within the <parameter>opts</parameter> string.  Note that
+      <function>malloc_message<parameter/></function> uses the
+      <function>mallctl*<parameter/></function> functions internally, so
+      inconsistent statistics can be reported if multiple threads use these
+      functions simultaneously.  If <option>--enable-stats</option> is
+      specified during configuration, &ldquo;m&rdquo; and &ldquo;a&rdquo; can
+      be specified to omit merged arena and per arena statistics, respectively;
+      &ldquo;b&rdquo; and &ldquo;l&rdquo; can be specified to omit per size
+      class statistics for bins and large objects, respectively.  Unrecognized
+      characters are silently ignored.  Note that thread caching may prevent
+      some statistics from being completely up to date, since extra locking
+      would be required to merge counters that track thread cache operations.
+      </para>
+
+      <para>The <function>malloc_usable_size<parameter/></function> function
+      returns the usable size of the allocation pointed to by
+      <parameter>ptr</parameter>.  The return value may be larger than the size
+      that was requested during allocation.  The
+      <function>malloc_usable_size<parameter/></function> function is not a
+      mechanism for in-place <function>realloc<parameter/></function>; rather
+      it is provided solely as a tool for introspection purposes.  Any
+      discrepancy between the requested allocation size and the size reported
+      by <function>malloc_usable_size<parameter/></function> should not be
+      depended on, since such behavior is entirely implementation-dependent.
+      </para>
     </refsect2>
     <refsect2>
       <title>Experimental API</title>
@@ -358,7 +492,7 @@ for (i = 0; i < nbins; i++) {
             <listitem><para>Initialize newly allocated memory to contain zero
             bytes.  In the growing reallocation case, the real size prior to
             reallocation defines the boundary between untouched bytes and those
-            that are initialized to contain zero bytes.  If this option is
+            that are initialized to contain zero bytes.  If this macro is
             absent, newly allocated memory is uninitialized.</para></listitem>
           </varlistentry>
           <varlistentry>
@@ -373,9 +507,11 @@ for (i = 0; i < nbins; i++) {
             </constant></term>
 
             <listitem><para>Use the arena specified by the index
-            <parameter>a</parameter>.  This macro does not validate that
-            <parameter>a</parameter> specifies an arena in the valid
-            range.</para></listitem>
+            <parameter>a</parameter> (and by necessity bypass the thread
+            cache).  This macro has no effect for huge regions, nor for regions
+            that were allocated via an arena other than the one specified.
+            This macro does not validate that <parameter>a</parameter>
+            specifies an arena index in the valid range.</para></listitem>
           </varlistentry>
         </variablelist>
       </para>
@@ -385,8 +521,9 @@ for (i = 0; i < nbins; i++) {
       <parameter>*ptr</parameter> to the base address of the allocation, and
       sets <parameter>*rsize</parameter> to the real size of the allocation if
       <parameter>rsize</parameter> is not <constant>NULL</constant>.  Behavior
-      is undefined if <parameter>size</parameter> is
-      <constant>0</constant>.</para>
+      is undefined if <parameter>size</parameter> is <constant>0</constant>, or
+      if request size overflows due to size class and/or alignment
+      constraints.</para>
 
       <para>The <function>rallocm<parameter/></function> function resizes the
       allocation at <parameter>*ptr</parameter> to be at least
@@ -396,11 +533,12 @@ for (i = 0; i < nbins; i++) {
       <parameter>rsize</parameter> is not <constant>NULL</constant>.  If
       <parameter>extra</parameter> is non-zero, an attempt is made to resize
       the allocation to be at least <code
-      language="C"><parameter>size</parameter> +
+      language="C">(<parameter>size</parameter> +
       <parameter>extra</parameter>)</code> bytes, though inability to allocate
       the extra byte(s) will not by itself result in failure.  Behavior is
-      undefined if <parameter>size</parameter> is <constant>0</constant>, or if
-      <code language="C">(<parameter>size</parameter> +
+      undefined if <parameter>size</parameter> is <constant>0</constant>, if
+      request size overflows due to size class and/or alignment constraints, or
+      if <code language="C">(<parameter>size</parameter> +
       <parameter>extra</parameter> &gt;
       <constant>SIZE_T_MAX</constant>)</code>.</para>
 
@@ -417,8 +555,9 @@ for (i = 0; i < nbins; i++) {
       <parameter>rsize</parameter> is not <constant>NULL</constant> it sets
       <parameter>*rsize</parameter> to the real size of the allocation that
       would result from the equivalent <function>allocm<parameter/></function>
-      function call.  Behavior is undefined if
-      <parameter>size</parameter> is <constant>0</constant>.</para>
+      function call.  Behavior is undefined if <parameter>size</parameter> is
+      <constant>0</constant>, or if request size overflows due to size class
+      and/or alignment constraints.</para>
     </refsect2>
   </refsect1>
   <refsect1 id="tuning">
@@ -605,7 +744,7 @@ for (i = 0; i < nbins; i++) {
     which controls refreshing of cached dynamic statistics.</para>
 
     <variablelist>
-      <varlistentry>
+      <varlistentry id="version">
         <term>
           <mallctl>version</mallctl>
           (<type>const char *</type>)
@@ -626,7 +765,7 @@ for (i = 0; i < nbins; i++) {
         detecting whether another thread caused a refresh.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.debug">
         <term>
           <mallctl>config.debug</mallctl>
           (<type>bool</type>)
@@ -636,7 +775,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.dss">
         <term>
           <mallctl>config.dss</mallctl>
           (<type>bool</type>)
@@ -646,7 +785,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.fill">
         <term>
           <mallctl>config.fill</mallctl>
           (<type>bool</type>)
@@ -656,7 +795,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.lazy_lock">
         <term>
           <mallctl>config.lazy_lock</mallctl>
           (<type>bool</type>)
@@ -666,7 +805,7 @@ for (i = 0; i < nbins; i++) {
         during build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.mremap">
         <term>
           <mallctl>config.mremap</mallctl>
           (<type>bool</type>)
@@ -676,7 +815,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.munmap">
         <term>
           <mallctl>config.munmap</mallctl>
           (<type>bool</type>)
@@ -686,7 +825,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.prof">
         <term>
           <mallctl>config.prof</mallctl>
           (<type>bool</type>)
@@ -696,7 +835,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.prof_libgcc">
         <term>
           <mallctl>config.prof_libgcc</mallctl>
           (<type>bool</type>)
@@ -706,7 +845,7 @@ for (i = 0; i < nbins; i++) {
         specified during build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.prof_libunwind">
         <term>
           <mallctl>config.prof_libunwind</mallctl>
           (<type>bool</type>)
@@ -716,7 +855,7 @@ for (i = 0; i < nbins; i++) {
         during build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.stats">
         <term>
           <mallctl>config.stats</mallctl>
           (<type>bool</type>)
@@ -726,7 +865,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.tcache">
         <term>
           <mallctl>config.tcache</mallctl>
           (<type>bool</type>)
@@ -736,7 +875,7 @@ for (i = 0; i < nbins; i++) {
         during build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.tls">
         <term>
           <mallctl>config.tls</mallctl>
           (<type>bool</type>)
@@ -746,7 +885,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.utrace">
         <term>
           <mallctl>config.utrace</mallctl>
           (<type>bool</type>)
@@ -756,7 +895,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.valgrind">
         <term>
           <mallctl>config.valgrind</mallctl>
           (<type>bool</type>)
@@ -766,7 +905,7 @@ for (i = 0; i < nbins; i++) {
         build configuration.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="config.xmalloc">
         <term>
           <mallctl>config.xmalloc</mallctl>
           (<type>bool</type>)
@@ -791,19 +930,6 @@ for (i = 0; i < nbins; i++) {
         </para></listitem>
       </varlistentry>
 
-      <varlistentry id="opt.lg_chunk">
-        <term>
-          <mallctl>opt.lg_chunk</mallctl>
-          (<type>size_t</type>)
-          <literal>r-</literal>
-        </term>
-        <listitem><para>Virtual memory chunk size (log base 2).  If a chunk
-        size outside the supported size range is specified, the size is
-        silently clipped to the minimum/maximum supported size.  The default
-        chunk size is 4 MiB (2^22).
-        </para></listitem>
-      </varlistentry>
-
       <varlistentry id="opt.dss">
         <term>
           <mallctl>opt.dss</mallctl>
@@ -815,7 +941,23 @@ for (i = 0; i < nbins; i++) {
         related to <citerefentry><refentrytitle>mmap</refentrytitle>
         <manvolnum>2</manvolnum></citerefentry> allocation.  The following
         settings are supported: &ldquo;disabled&rdquo;, &ldquo;primary&rdquo;,
-        and &ldquo;secondary&rdquo; (default).</para></listitem>
+        and &ldquo;secondary&rdquo;.  The default is &ldquo;secondary&rdquo; if
+        <link linkend="config.dss"><mallctl>config.dss</mallctl></link> is
+        true, &ldquo;disabled&rdquo; otherwise.
+        </para></listitem>
+      </varlistentry>
+
+      <varlistentry id="opt.lg_chunk">
+        <term>
+          <mallctl>opt.lg_chunk</mallctl>
+          (<type>size_t</type>)
+          <literal>r-</literal>
+        </term>
+        <listitem><para>Virtual memory chunk size (log base 2).  If a chunk
+        size outside the supported size range is specified, the size is
+        silently clipped to the minimum/maximum supported size.  The default
+        chunk size is 4 MiB (2^22).
+        </para></listitem>
       </varlistentry>
 
       <varlistentry id="opt.narenas">
@@ -934,7 +1076,8 @@ for (i = 0; i < nbins; i++) {
         <listitem><para>Zero filling enabled/disabled.  If enabled, each byte
         of uninitialized allocated memory will be initialized to 0.  Note that
         this initialization only happens once for each byte, so
-        <function>realloc<parameter/></function> and
+        <function>realloc<parameter/></function>,
+        <function>rallocx<parameter/></function> and
         <function>rallocm<parameter/></function> calls do not zero memory that
         was previously allocated.  This is intended for debugging and will
         impact performance negatively.  This option is disabled by default.
@@ -1063,7 +1206,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <term>
           <mallctl>opt.prof_active</mallctl>
           (<type>bool</type>)
-          <literal>r-</literal>
+          <literal>rw</literal>
           [<option>--enable-prof</option>]
         </term>
         <listitem><para>Profiling activated/deactivated.  This is a secondary
@@ -1175,7 +1318,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         by default.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="thread.arena">
         <term>
           <mallctl>thread.arena</mallctl>
           (<type>unsigned</type>)
@@ -1202,7 +1345,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         cases.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="thread.allocatedp">
         <term>
           <mallctl>thread.allocatedp</mallctl>
           (<type>uint64_t *</type>)
@@ -1229,7 +1372,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         cases.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="thread.deallocatedp">
         <term>
           <mallctl>thread.deallocatedp</mallctl>
           (<type>uint64_t *</type>)
@@ -1243,7 +1386,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <function>mallctl*<parameter/></function> calls.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="thread.tcache.enabled">
         <term>
           <mallctl>thread.tcache.enabled</mallctl>
           (<type>bool</type>)
@@ -1257,7 +1400,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="thread.tcache.flush">
         <term>
           <mallctl>thread.tcache.flush</mallctl>
           (<type>void</type>)
@@ -1323,7 +1466,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         initialized.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.quantum">
         <term>
           <mallctl>arenas.quantum</mallctl>
           (<type>size_t</type>)
@@ -1332,7 +1475,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Quantum size.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.page">
         <term>
           <mallctl>arenas.page</mallctl>
           (<type>size_t</type>)
@@ -1341,7 +1484,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Page size.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.tcache_max">
         <term>
           <mallctl>arenas.tcache_max</mallctl>
           (<type>size_t</type>)
@@ -1351,7 +1494,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Maximum thread-cached size class.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.nbins">
         <term>
           <mallctl>arenas.nbins</mallctl>
           (<type>unsigned</type>)
@@ -1360,7 +1503,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Number of bin size classes.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.nhbins">
         <term>
           <mallctl>arenas.nhbins</mallctl>
           (<type>unsigned</type>)
@@ -1380,7 +1523,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Maximum size supported by size class.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.bin.i.nregs">
         <term>
           <mallctl>arenas.bin.&lt;i&gt;.nregs</mallctl>
           (<type>uint32_t</type>)
@@ -1389,7 +1532,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Number of regions per page run.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.bin.i.run_size">
         <term>
           <mallctl>arenas.bin.&lt;i&gt;.run_size</mallctl>
           (<type>size_t</type>)
@@ -1398,7 +1541,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Number of bytes per page run.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.nlruns">
         <term>
           <mallctl>arenas.nlruns</mallctl>
           (<type>size_t</type>)
@@ -1407,7 +1550,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Total number of large size classes.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.lrun.i.size">
         <term>
           <mallctl>arenas.lrun.&lt;i&gt;.size</mallctl>
           (<type>size_t</type>)
@@ -1417,7 +1560,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         class.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.purge">
         <term>
           <mallctl>arenas.purge</mallctl>
           (<type>unsigned</type>)
@@ -1427,7 +1570,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         for all arenas if none is specified.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="arenas.extend">
         <term>
           <mallctl>arenas.extend</mallctl>
           (<type>unsigned</type>)
@@ -1451,7 +1594,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="prof.dump">
         <term>
           <mallctl>prof.dump</mallctl>
           (<type>const char *</type>)
@@ -1467,7 +1610,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         option.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="prof.interval">
         <term>
           <mallctl>prof.interval</mallctl>
           (<type>uint64_t</type>)
@@ -1527,7 +1670,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         entirely devoted to allocator metadata.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.mapped">
         <term>
           <mallctl>stats.mapped</mallctl>
           (<type>size_t</type>)
@@ -1541,7 +1684,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         does not include inactive chunks.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.chunks.current">
         <term>
           <mallctl>stats.chunks.current</mallctl>
           (<type>size_t</type>)
@@ -1553,7 +1696,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.chunks.total">
         <term>
           <mallctl>stats.chunks.total</mallctl>
           (<type>uint64_t</type>)
@@ -1563,7 +1706,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Cumulative number of chunks allocated.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.chunks.high">
         <term>
           <mallctl>stats.chunks.high</mallctl>
           (<type>size_t</type>)
@@ -1574,7 +1717,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.huge.allocated">
         <term>
           <mallctl>stats.huge.allocated</mallctl>
           (<type>size_t</type>)
@@ -1585,7 +1728,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.huge.nmalloc">
         <term>
           <mallctl>stats.huge.nmalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1596,7 +1739,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.huge.ndalloc">
         <term>
           <mallctl>stats.huge.ndalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1607,7 +1750,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.dss">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.dss</mallctl>
           (<type>const char *</type>)
@@ -1621,7 +1764,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.nthreads">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.nthreads</mallctl>
           (<type>unsigned</type>)
@@ -1631,7 +1774,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         arena.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.pactive">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.pactive</mallctl>
           (<type>size_t</type>)
@@ -1652,7 +1795,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         similar has not been called.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.mapped">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.mapped</mallctl>
           (<type>size_t</type>)
@@ -1662,7 +1805,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Number of mapped bytes.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.npurge">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.npurge</mallctl>
           (<type>uint64_t</type>)
@@ -1673,7 +1816,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.nmadvise">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.nmadvise</mallctl>
           (<type>uint64_t</type>)
@@ -1685,9 +1828,9 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         similar calls made to purge dirty pages.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.purged">
         <term>
-          <mallctl>stats.arenas.&lt;i&gt;.npurged</mallctl>
+          <mallctl>stats.arenas.&lt;i&gt;.purged</mallctl>
           (<type>uint64_t</type>)
           <literal>r-</literal>
           [<option>--enable-stats</option>]
@@ -1695,7 +1838,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Number of pages purged.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.small.allocated">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.small.allocated</mallctl>
           (<type>size_t</type>)
@@ -1706,7 +1849,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.small.nmalloc">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.small.nmalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1717,7 +1860,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         small bins.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.small.ndalloc">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.small.ndalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1728,7 +1871,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.small.nrequests">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.small.nrequests</mallctl>
           (<type>uint64_t</type>)
@@ -1739,7 +1882,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.large.allocated">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.large.allocated</mallctl>
           (<type>size_t</type>)
@@ -1750,7 +1893,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.large.nmalloc">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.large.nmalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1761,7 +1904,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         directly by the arena.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.large.ndalloc">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.large.ndalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1772,7 +1915,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         directly by the arena.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.large.nrequests">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.large.nrequests</mallctl>
           (<type>uint64_t</type>)
@@ -1783,7 +1926,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.allocated">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.allocated</mallctl>
           (<type>size_t</type>)
@@ -1794,7 +1937,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         bin.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.nmalloc">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nmalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1805,7 +1948,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.ndalloc">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.ndalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1816,7 +1959,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         </para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.nrequests">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nrequests</mallctl>
           (<type>uint64_t</type>)
@@ -1827,7 +1970,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         requests.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.nfills">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nfills</mallctl>
           (<type>uint64_t</type>)
@@ -1837,7 +1980,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Cumulative number of tcache fills.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.nflushes">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nflushes</mallctl>
           (<type>uint64_t</type>)
@@ -1847,7 +1990,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Cumulative number of tcache flushes.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.nruns">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nruns</mallctl>
           (<type>uint64_t</type>)
@@ -1857,7 +2000,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Cumulative number of runs created.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.nreruns">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nreruns</mallctl>
           (<type>uint64_t</type>)
@@ -1868,7 +2011,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         to allocate changed.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.bins.j.curruns">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.curruns</mallctl>
           (<type>size_t</type>)
@@ -1878,7 +2021,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Current number of runs.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.lruns.j.nmalloc">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.lruns.&lt;j&gt;.nmalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1889,7 +2032,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         class served directly by the arena.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.lruns.j.ndalloc">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.lruns.&lt;j&gt;.ndalloc</mallctl>
           (<type>uint64_t</type>)
@@ -1900,7 +2043,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         size class served directly by the arena.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.lruns.j.nrequests">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.lruns.&lt;j&gt;.nrequests</mallctl>
           (<type>uint64_t</type>)
@@ -1911,7 +2054,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         class.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.lruns.j.curruns">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.lruns.&lt;j&gt;.curruns</mallctl>
           (<type>size_t</type>)
@@ -2037,9 +2180,26 @@ malloc_conf = "xmalloc:true";]]></programlisting>
     </refsect2>
     <refsect2>
       <title>Non-standard API</title>
-      <para>The <function>malloc_usable_size<parameter/></function> function
-      returns the usable size of the allocation pointed to by
-      <parameter>ptr</parameter>.  </para>
+      <para>The <function>mallocx<parameter/></function> and
+      <function>rallocx<parameter/></function> functions return a pointer to
+      the allocated memory if successful; otherwise a <constant>NULL</constant>
+      pointer is returned to indicate insufficient contiguous memory was
+      available to service the allocation request.  </para>
+
+      <para>The <function>xallocx<parameter/></function> function returns the
+      real size of the resulting resized allocation pointed to by
+      <parameter>ptr</parameter>, which is a value less than
+      <parameter>size</parameter> if the allocation could not be adequately
+      grown in place.  </para>
+
+      <para>The <function>sallocx<parameter/></function> function returns the
+      real size of the allocation pointed to by <parameter>ptr</parameter>.
+      </para>
+
+      <para>The <function>nallocx<parameter/></function> returns the real size
+      that would result from a successful equivalent
+      <function>mallocx<parameter/></function> function call, or zero if
+      insufficient memory is available to perform the size computation.  </para>
 
       <para>The <function>mallctl<parameter/></function>,
       <function>mallctlnametomib<parameter/></function>, and
@@ -2056,12 +2216,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
             is too large or too small; in this case as much data as possible
             are read despite the error.</para></listitem>
           </varlistentry>
-          <varlistentry>
-            <term><errorname>ENOMEM</errorname></term>
-
-            <listitem><para><parameter>*oldlenp</parameter> is too short to
-            hold the requested value.</para></listitem>
-          </varlistentry>
           <varlistentry>
             <term><errorname>ENOENT</errorname></term>
 
@@ -2090,6 +2244,10 @@ malloc_conf = "xmalloc:true";]]></programlisting>
           </varlistentry>
         </variablelist>
       </para>
+
+      <para>The <function>malloc_usable_size<parameter/></function> function
+      returns the usable size of the allocation pointed to by
+      <parameter>ptr</parameter>.  </para>
     </refsect2>
     <refsect2>
       <title>Experimental API</title>

include/jemalloc/internal/arena.h

@@ -158,6 +158,7 @@ struct arena_chunk_map_s {
 };
 typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
 typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
+typedef ql_head(arena_chunk_map_t) arena_chunk_mapelms_t;
 
 /* Arena chunk header. */
 struct arena_chunk_s {
@@ -174,11 +175,12 @@ struct arena_chunk_s {
 	size_t			nruns_avail;
 
 	/*
-	 * Number of available run adjacencies.  Clean and dirty available runs
-	 * are not coalesced, which causes virtual memory fragmentation.  The
-	 * ratio of (nruns_avail-nruns_adjac):nruns_adjac is used for tracking
-	 * this fragmentation.
-	 * */
+	 * Number of available run adjacencies that purging could coalesce.
+	 * Clean and dirty available runs are not coalesced, which causes
+	 * virtual memory fragmentation.  The ratio of
+	 * (nruns_avail-nruns_adjac):nruns_adjac is used for tracking this
+	 * fragmentation.
+	 */
 	size_t			nruns_adjac;
 
 	/*
@@ -404,7 +406,16 @@ void	arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
     size_t binind, uint64_t prof_accumbytes);
 void	arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info,
     bool zero);
+#ifdef JEMALLOC_JET
+typedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t,
+    uint8_t);
+extern arena_redzone_corruption_t *arena_redzone_corruption;
+typedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *);
+extern arena_dalloc_junk_small_t *arena_dalloc_junk_small;
+#else
 void	arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info);
+#endif
+void	arena_quarantine_junk_small(void *ptr, size_t usize);
 void	*arena_malloc_small(arena_t *arena, size_t size, bool zero);
 void	*arena_malloc_large(arena_t *arena, size_t size, bool zero);
 void	*arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
@@ -415,10 +426,18 @@ void	arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
     size_t pageind, arena_chunk_map_t *mapelm);
 void	arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
     size_t pageind);
+#ifdef JEMALLOC_JET
+typedef void (arena_dalloc_junk_large_t)(void *, size_t);
+extern arena_dalloc_junk_large_t *arena_dalloc_junk_large;
+#endif
 void	arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk,
     void *ptr);
 void	arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
-void	*arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
+#ifdef JEMALLOC_JET
+typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t);
+extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
+#endif
+bool	arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra, bool zero);
 void	*arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
     size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
@@ -473,7 +492,7 @@ size_t	arena_bin_index(arena_t *arena, arena_bin_t *bin);
 unsigned	arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
     const void *ptr);
 prof_ctx_t	*arena_prof_ctx_get(const void *ptr);
-void	arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
+void	arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx);
 void	*arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache);
 size_t	arena_salloc(const void *ptr, bool demote);
 void	arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr,
@@ -885,10 +904,10 @@ arena_prof_ctx_get(const void *ptr)
 }
 
 JEMALLOC_INLINE void
-arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
+arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx)
 {
 	arena_chunk_t *chunk;
-	size_t pageind, mapbits;
+	size_t pageind;
 
 	cassert(config_prof);
 	assert(ptr != NULL);
@@ -896,10 +915,17 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-	mapbits = arena_mapbits_get(chunk, pageind);
-	assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
-	if ((mapbits & CHUNK_MAP_LARGE) == 0) {
+	assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
+
+	if (usize > SMALL_MAXCLASS || (prof_promote &&
+	    ((uintptr_t)ctx != (uintptr_t)1U || arena_mapbits_large_get(chunk,
+	    pageind) != 0))) {
+		assert(arena_mapbits_large_get(chunk, pageind) != 0);
+		arena_mapp_get(chunk, pageind)->prof_ctx = ctx;
+	} else {
+		assert(arena_mapbits_large_get(chunk, pageind) == 0);
 		if (prof_promote == false) {
+			size_t mapbits = arena_mapbits_get(chunk, pageind);
 			arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
 			    (uintptr_t)((pageind - (mapbits >> LG_PAGE)) <<
 			    LG_PAGE));
@@ -911,12 +937,11 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
 			bin_info = &arena_bin_info[binind];
 			regind = arena_run_regind(run, bin_info, ptr);
 
-			*((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset
-			    + (regind * sizeof(prof_ctx_t *)))) = ctx;
-		} else
-			assert((uintptr_t)ctx == (uintptr_t)1U);
-	} else
-		arena_mapp_get(chunk, pageind)->prof_ctx = ctx;
+			*((prof_ctx_t **)((uintptr_t)run +
+			    bin_info->ctx0_offset + (regind * sizeof(prof_ctx_t
+			    *)))) = ctx;
+		}
+	}
 }
 
 JEMALLOC_ALWAYS_INLINE void *

include/jemalloc/internal/chunk_dss.h

@@ -7,7 +7,7 @@ typedef enum {
 	dss_prec_secondary = 2,
 
 	dss_prec_limit     = 3
-} dss_prec_t ;
+} dss_prec_t;
 #define	DSS_PREC_DEFAULT	dss_prec_secondary
 #define	DSS_DEFAULT		"secondary"
 

include/jemalloc/internal/ckh.h

@@ -17,7 +17,7 @@ typedef bool ckh_keycomp_t (const void *, const void *);
  * There are 2^LG_CKH_BUCKET_CELLS cells in each hash table bucket.  Try to fit
  * one bucket per L1 cache line.
  */
-#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1)
+#define	LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1)
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/

include/jemalloc/internal/hash.h

@@ -19,6 +19,11 @@
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
+uint32_t	hash_x86_32(const void *key, int len, uint32_t seed);
+void	hash_x86_128(const void *key, const int len, uint32_t seed,
+    uint64_t r_out[2]);
+void	hash_x64_128(const void *key, const int len, const uint32_t seed,
+    uint64_t r_out[2]);
 void	hash(const void *key, size_t len, const uint32_t seed,
     size_t r_hash[2]);
 #endif
@@ -43,14 +48,14 @@ JEMALLOC_INLINE uint32_t
 hash_get_block_32(const uint32_t *p, int i)
 {
 
-	return p[i];
+	return (p[i]);
 }
 
 JEMALLOC_INLINE uint64_t
 hash_get_block_64(const uint64_t *p, int i)
 {
 
-	return p[i];
+	return (p[i]);
 }
 
 JEMALLOC_INLINE uint32_t
@@ -63,7 +68,7 @@ hash_fmix_32(uint32_t h)
 	h *= 0xc2b2ae35;
 	h ^= h >> 16;
 
-	return h;
+	return (h);
 }
 
 JEMALLOC_INLINE uint64_t
@@ -76,7 +81,7 @@ hash_fmix_64(uint64_t k)
 	k *= QU(0xc4ceb9fe1a85ec53LLU);
 	k ^= k >> 33;
 
-	return k;
+	return (k);
 }
 
 JEMALLOC_INLINE uint32_t
@@ -127,12 +132,12 @@ hash_x86_32(const void *key, int len, uint32_t seed)
 
 	h1 = hash_fmix_32(h1);
 
-	return h1;
+	return (h1);
 }
 
 UNUSED JEMALLOC_INLINE void
 hash_x86_128(const void *key, const int len, uint32_t seed,
-  uint64_t r_out[2])
+    uint64_t r_out[2])
 {
 	const uint8_t * data = (const uint8_t *) key;
 	const int nblocks = len / 16;
@@ -234,7 +239,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed,
 
 UNUSED JEMALLOC_INLINE void
 hash_x64_128(const void *key, const int len, const uint32_t seed,
-  uint64_t r_out[2])
+    uint64_t r_out[2])
 {
 	const uint8_t *data = (const uint8_t *) key;
 	const int nblocks = len / 16;
@@ -310,7 +315,6 @@ hash_x64_128(const void *key, const int len, const uint32_t seed,
 	r_out[1] = h2;
 }
 
-
 /******************************************************************************/
 /* API. */
 JEMALLOC_INLINE void

include/jemalloc/internal/huge.h

@@ -19,10 +19,14 @@ extern malloc_mutex_t	huge_mtx;
 
 void	*huge_malloc(size_t size, bool zero);
 void	*huge_palloc(size_t size, size_t alignment, bool zero);
-void	*huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
+bool	huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra);
 void	*huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
     size_t alignment, bool zero, bool try_tcache_dalloc);
+#ifdef JEMALLOC_JET
+typedef void (huge_dalloc_junk_t)(void *, size_t);
+extern huge_dalloc_junk_t *huge_dalloc_junk;
+#endif
 void	huge_dalloc(void *ptr, bool unmap);
 size_t	huge_salloc(const void *ptr);
 prof_ctx_t	*huge_prof_ctx_get(const void *ptr);

include/jemalloc/internal/jemalloc_internal.h.in

@@ -1,5 +1,5 @@
 #ifndef JEMALLOC_INTERNAL_H
-#define JEMALLOC_INTERNAL_H
+#define	JEMALLOC_INTERNAL_H
 #include <math.h>
 #ifdef _WIN32
 #  include <windows.h>
@@ -54,8 +54,7 @@ typedef intptr_t ssize_t;
 #endif
 #include <fcntl.h>
 
-#define	JEMALLOC_NO_DEMANGLE
-#include "../jemalloc@install_suffix@.h"
+#include "jemalloc_internal_defs.h"
 
 #ifdef JEMALLOC_UTRACE
 #include <sys/ktrace.h>
@@ -66,13 +65,18 @@ typedef intptr_t ssize_t;
 #include <valgrind/memcheck.h>
 #endif
 
-#include "jemalloc/internal/private_namespace.h"
-
-#ifdef JEMALLOC_CC_SILENCE
-#define	UNUSED JEMALLOC_ATTR(unused)
+#define	JEMALLOC_NO_DEMANGLE
+#ifdef JEMALLOC_JET
+#  define JEMALLOC_N(n) jet_##n
+#  include "jemalloc/internal/public_namespace.h"
+#  define JEMALLOC_NO_RENAME
+#  include "../jemalloc@install_suffix@.h"
+#  undef JEMALLOC_NO_RENAME
 #else
-#define	UNUSED
+#  define JEMALLOC_N(n) @private_namespace@##n
+#  include "../jemalloc@install_suffix@.h"
 #endif
+#include "jemalloc/internal/private_namespace.h"
 
 static const bool config_debug =
 #ifdef JEMALLOC_DEBUG
@@ -221,48 +225,13 @@ static const bool config_ivsalloc =
  *   JEMALLOC_H_INLINES : Inline functions.
  */
 /******************************************************************************/
-#define JEMALLOC_H_TYPES
+#define	JEMALLOC_H_TYPES
 
+#include "jemalloc/internal/jemalloc_internal_macros.h"
+
+#define	MALLOCX_LG_ALIGN_MASK	((int)0x3f)
 #define	ALLOCM_LG_ALIGN_MASK	((int)0x3f)
 
-#define	ZU(z)	((size_t)z)
-#define	QU(q)	((uint64_t)q)
-
-#ifndef __DECONST
-#  define	__DECONST(type, var)	((type)(uintptr_t)(const void *)(var))
-#endif
-
-/*
- * JEMALLOC_ALWAYS_INLINE is used within header files for functions that are
- * static inline functions if inlining is enabled, and single-definition
- * library-private functions if inlining is disabled.
- *
- * JEMALLOC_ALWAYS_INLINE_C is for use in .c files, in which case the denoted
- * functions are always static, regardless of whether inlining is enabled.
- */
-#ifdef JEMALLOC_DEBUG
-   /* Disable inlining to make debugging easier. */
-#  define JEMALLOC_ALWAYS_INLINE
-#  define JEMALLOC_ALWAYS_INLINE_C static
-#  define JEMALLOC_INLINE
-#  define inline
-#else
-#  define JEMALLOC_ENABLE_INLINE
-#  ifdef JEMALLOC_HAVE_ATTR
-#    define JEMALLOC_ALWAYS_INLINE \
-	 static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline)
-#    define JEMALLOC_ALWAYS_INLINE_C \
-	 static inline JEMALLOC_ATTR(always_inline)
-#  else
-#    define JEMALLOC_ALWAYS_INLINE static inline
-#    define JEMALLOC_ALWAYS_INLINE_C static inline
-#  endif
-#  define JEMALLOC_INLINE static inline
-#  ifdef _MSC_VER
-#    define inline _inline
-#  endif
-#endif
-
 /* Smallest size class to support. */
 #define	LG_TINY_MIN		3
 #define	TINY_MIN		(1U << LG_TINY_MIN)
@@ -493,7 +462,7 @@ static const bool config_ivsalloc =
 
 #undef JEMALLOC_H_TYPES
 /******************************************************************************/
-#define JEMALLOC_H_STRUCTS
+#define	JEMALLOC_H_STRUCTS
 
 #include "jemalloc/internal/util.h"
 #include "jemalloc/internal/atomic.h"
@@ -522,14 +491,14 @@ typedef struct {
 	uint64_t	deallocated;
 } thread_allocated_t;
 /*
- * The JEMALLOC_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro
+ * The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro
  * argument.
  */
-#define	THREAD_ALLOCATED_INITIALIZER	JEMALLOC_CONCAT({0, 0})
+#define	THREAD_ALLOCATED_INITIALIZER	JEMALLOC_ARG_CONCAT({0, 0})
 
 #undef JEMALLOC_H_STRUCTS
 /******************************************************************************/
-#define JEMALLOC_H_EXTERNS
+#define	JEMALLOC_H_EXTERNS
 
 extern bool	opt_abort;
 extern bool	opt_junk;
@@ -589,7 +558,7 @@ void	jemalloc_postfork_child(void);
 
 #undef JEMALLOC_H_EXTERNS
 /******************************************************************************/
-#define JEMALLOC_H_INLINES
+#define	JEMALLOC_H_INLINES
 
 #include "jemalloc/internal/util.h"
 #include "jemalloc/internal/atomic.h"
@@ -764,32 +733,36 @@ choose_arena(arena_t *arena)
 #include "jemalloc/internal/quarantine.h"
 
 #ifndef JEMALLOC_ENABLE_INLINE
-void	*imallocx(size_t size, bool try_tcache, arena_t *arena);
+void	*imalloct(size_t size, bool try_tcache, arena_t *arena);
 void	*imalloc(size_t size);
-void	*icallocx(size_t size, bool try_tcache, arena_t *arena);
+void	*icalloct(size_t size, bool try_tcache, arena_t *arena);
 void	*icalloc(size_t size);
-void	*ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
+void	*ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
     arena_t *arena);
 void	*ipalloc(size_t usize, size_t alignment, bool zero);
 size_t	isalloc(const void *ptr, bool demote);
 size_t	ivsalloc(const void *ptr, bool demote);
 size_t	u2rz(size_t usize);
 size_t	p2rz(const void *ptr);
-void	idallocx(void *ptr, bool try_tcache);
+void	idalloct(void *ptr, bool try_tcache);
 void	idalloc(void *ptr);
-void	iqallocx(void *ptr, bool try_tcache);
+void	iqalloct(void *ptr, bool try_tcache);
 void	iqalloc(void *ptr);
-void	*irallocx(void *ptr, size_t size, size_t extra, size_t alignment,
-    bool zero, bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc,
+void	*iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
+    size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
     arena_t *arena);
+void	*iralloct(void *ptr, size_t size, size_t extra, size_t alignment,
+    bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
 void	*iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
-    bool zero, bool no_move);
+    bool zero);
+bool	ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
+    bool zero);
 malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
 JEMALLOC_ALWAYS_INLINE void *
-imallocx(size_t size, bool try_tcache, arena_t *arena)
+imalloct(size_t size, bool try_tcache, arena_t *arena)
 {
 
 	assert(size != 0);
@@ -804,11 +777,11 @@ JEMALLOC_ALWAYS_INLINE void *
 imalloc(size_t size)
 {
 
-	return (imallocx(size, true, NULL));
+	return (imalloct(size, true, NULL));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-icallocx(size_t size, bool try_tcache, arena_t *arena)
+icalloct(size_t size, bool try_tcache, arena_t *arena)
 {
 
 	if (size <= arena_maxclass)
@@ -821,11 +794,11 @@ JEMALLOC_ALWAYS_INLINE void *
 icalloc(size_t size)
 {
 
-	return (icallocx(size, true, NULL));
+	return (icalloct(size, true, NULL));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
+ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache,
     arena_t *arena)
 {
 	void *ret;
@@ -853,7 +826,7 @@ JEMALLOC_ALWAYS_INLINE void *
 ipalloc(size_t usize, size_t alignment, bool zero)
 {
 
-	return (ipallocx(usize, alignment, zero, true, NULL));
+	return (ipalloct(usize, alignment, zero, true, NULL));
 }
 
 /*
@@ -885,7 +858,7 @@ ivsalloc(const void *ptr, bool demote)
 {
 
 	/* Return 0 if ptr is not within a chunk managed by jemalloc. */
-	if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL)
+	if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0)
 		return (0);
 
 	return (isalloc(ptr, demote));
@@ -914,7 +887,7 @@ p2rz(const void *ptr)
 }
 
 JEMALLOC_ALWAYS_INLINE void
-idallocx(void *ptr, bool try_tcache)
+idalloct(void *ptr, bool try_tcache)
 {
 	arena_chunk_t *chunk;
 
@@ -931,31 +904,63 @@ JEMALLOC_ALWAYS_INLINE void
 idalloc(void *ptr)
 {
 
-	idallocx(ptr, true);
+	idalloct(ptr, true);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-iqallocx(void *ptr, bool try_tcache)
+iqalloct(void *ptr, bool try_tcache)
 {
 
 	if (config_fill && opt_quarantine)
 		quarantine(ptr);
 	else
-		idallocx(ptr, try_tcache);
+		idalloct(ptr, try_tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 iqalloc(void *ptr)
 {
 
-	iqallocx(ptr, true);
+	iqalloct(ptr, true);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
-    bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
+iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
+    size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
+    arena_t *arena)
+{
+	void *p;
+	size_t usize, copysize;
+
+	usize = sa2u(size + extra, alignment);
+	if (usize == 0)
+		return (NULL);
+	p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
+	if (p == NULL) {
+		if (extra == 0)
+			return (NULL);
+		/* Try again, without extra this time. */
+		usize = sa2u(size, alignment);
+		if (usize == 0)
+			return (NULL);
+		p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
+		if (p == NULL)
+			return (NULL);
+	}
+	/*
+	 * Copy at most size bytes (not size+extra), since the caller has no
+	 * expectation that the extra bytes will be reliably preserved.
+	 */
+	copysize = (size < oldsize) ? size : oldsize;
+	memcpy(p, ptr, copysize);
+	iqalloct(ptr, try_tcache_dalloc);
+	return (p);
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
+    bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
 {
-	void *ret;
 	size_t oldsize;
 
 	assert(ptr != NULL);
@@ -965,68 +970,50 @@ irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 
 	if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
 	    != 0) {
-		size_t usize, copysize;
-
 		/*
 		 * Existing object alignment is inadequate; allocate new space
 		 * and copy.
 		 */
-		if (no_move)
-			return (NULL);
-		usize = sa2u(size + extra, alignment);
-		if (usize == 0)
-			return (NULL);
-		ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena);
-		if (ret == NULL) {
-			if (extra == 0)
-				return (NULL);
-			/* Try again, without extra this time. */
-			usize = sa2u(size, alignment);
-			if (usize == 0)
-				return (NULL);
-			ret = ipallocx(usize, alignment, zero, try_tcache_alloc,
-			    arena);
-			if (ret == NULL)
-				return (NULL);
-		}
-		/*
-		 * Copy at most size bytes (not size+extra), since the caller
-		 * has no expectation that the extra bytes will be reliably
-		 * preserved.
-		 */
-		copysize = (size < oldsize) ? size : oldsize;
-		memcpy(ret, ptr, copysize);
-		iqallocx(ptr, try_tcache_dalloc);
-		return (ret);
+		return (iralloct_realign(ptr, oldsize, size, extra, alignment,
+		    zero, try_tcache_alloc, try_tcache_dalloc, arena));
 	}
 
-	if (no_move) {
-		if (size <= arena_maxclass) {
-			return (arena_ralloc_no_move(ptr, oldsize, size,
-			    extra, zero));
-		} else {
-			return (huge_ralloc_no_move(ptr, oldsize, size,
-			    extra));
-		}
+	if (size + extra <= arena_maxclass) {
+		return (arena_ralloc(arena, ptr, oldsize, size, extra,
+		    alignment, zero, try_tcache_alloc,
+		    try_tcache_dalloc));
 	} else {
-		if (size + extra <= arena_maxclass) {
-			return (arena_ralloc(arena, ptr, oldsize, size, extra,
-			    alignment, zero, try_tcache_alloc,
-			    try_tcache_dalloc));
-		} else {
-			return (huge_ralloc(ptr, oldsize, size, extra,
-			    alignment, zero, try_tcache_dalloc));
-		}
+		return (huge_ralloc(ptr, oldsize, size, extra,
+		    alignment, zero, try_tcache_dalloc));
 	}
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
-    bool no_move)
+iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
 {
 
-	return (irallocx(ptr, size, extra, alignment, zero, no_move, true, true,
-	    NULL));
+	return (iralloct(ptr, size, extra, alignment, zero, true, true, NULL));
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
+{
+	size_t oldsize;
+
+	assert(ptr != NULL);
+	assert(size != 0);
+
+	oldsize = isalloc(ptr, config_prof);
+	if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
+	    != 0) {
+		/* Existing object alignment is inadequate. */
+		return (true);
+	}
+
+	if (size <= arena_maxclass)
+		return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
+	else
+		return (huge_ralloc_no_move(ptr, oldsize, size, extra));
 }
 
 malloc_tsd_externs(thread_allocated, thread_allocated_t)

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -0,0 +1,199 @@
+#ifndef JEMALLOC_INTERNAL_DEFS_H_
+#define	JEMALLOC_INTERNAL_DEFS_H_
+/*
+ * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all
+ * public APIs to be prefixed.  This makes it possible, with some care, to use
+ * multiple allocators simultaneously.
+ */
+#undef JEMALLOC_PREFIX
+#undef JEMALLOC_CPREFIX
+
+/*
+ * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs.
+ * For shared libraries, symbol visibility mechanisms prevent these symbols
+ * from being exported, but for static libraries, naming collisions are a real
+ * possibility.
+ */
+#undef JEMALLOC_PRIVATE_NAMESPACE
+
+/*
+ * Hyper-threaded CPUs may need a special instruction inside spin loops in
+ * order to yield to another virtual CPU.
+ */
+#undef CPU_SPINWAIT
+
+/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */
+#undef JEMALLOC_ATOMIC9
+
+/*
+ * Defined if OSAtomic*() functions are available, as provided by Darwin, and
+ * documented in the atomic(3) manual page.
+ */
+#undef JEMALLOC_OSATOMIC
+
+/*
+ * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
+ * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
+ * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
+ * functions are defined in libgcc instead of being inlines)
+ */
+#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4
+
+/*
+ * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
+ * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
+ * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
+ * functions are defined in libgcc instead of being inlines)
+ */
+#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8
+
+/*
+ * Defined if OSSpin*() functions are available, as provided by Darwin, and
+ * documented in the spinlock(3) manual page.
+ */
+#undef JEMALLOC_OSSPIN
+
+/*
+ * Defined if _malloc_thread_cleanup() exists.  At least in the case of
+ * FreeBSD, pthread_key_create() allocates, which if used during malloc
+ * bootstrapping will cause recursion into the pthreads library.  Therefore, if
+ * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in
+ * malloc_tsd.
+ */
+#undef JEMALLOC_MALLOC_THREAD_CLEANUP
+
+/*
+ * Defined if threaded initialization is known to be safe on this platform.
+ * Among other things, it must be possible to initialize a mutex without
+ * triggering allocation in order for threaded allocation to be safe.
+ */
+#undef JEMALLOC_THREADED_INIT
+
+/*
+ * Defined if the pthreads implementation defines
+ * _pthread_mutex_init_calloc_cb(), in which case the function is used in order
+ * to avoid recursive allocation during mutex initialization.
+ */
+#undef JEMALLOC_MUTEX_INIT_CB
+
+/* Defined if sbrk() is supported. */
+#undef JEMALLOC_HAVE_SBRK
+
+/* Non-empty if the tls_model attribute is supported. */
+#undef JEMALLOC_TLS_MODEL
+
+/* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */
+#undef JEMALLOC_CC_SILENCE
+
+/* JEMALLOC_CODE_COVERAGE enables test code coverage analysis. */
+#undef JEMALLOC_CODE_COVERAGE
+
+/*
+ * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
+ * inline functions.
+ */
+#undef JEMALLOC_DEBUG
+
+/* JEMALLOC_STATS enables statistics calculation. */
+#undef JEMALLOC_STATS
+
+/* JEMALLOC_PROF enables allocation profiling. */
+#undef JEMALLOC_PROF
+
+/* Use libunwind for profile backtracing if defined. */
+#undef JEMALLOC_PROF_LIBUNWIND
+
+/* Use libgcc for profile backtracing if defined. */
+#undef JEMALLOC_PROF_LIBGCC
+
+/* Use gcc intrinsics for profile backtracing if defined. */
+#undef JEMALLOC_PROF_GCC
+
+/*
+ * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects.
+ * This makes it possible to allocate/deallocate objects without any locking
+ * when the cache is in the steady state.
+ */
+#undef JEMALLOC_TCACHE
+
+/*
+ * JEMALLOC_DSS enables use of sbrk(2) to allocate chunks from the data storage
+ * segment (DSS).
+ */
+#undef JEMALLOC_DSS
+
+/* Support memory filling (junk/zero/quarantine/redzone). */
+#undef JEMALLOC_FILL
+
+/* Support utrace(2)-based tracing. */
+#undef JEMALLOC_UTRACE
+
+/* Support Valgrind. */
+#undef JEMALLOC_VALGRIND
+
+/* Support optional abort() on OOM. */
+#undef JEMALLOC_XMALLOC
+
+/* Support lazy locking (avoid locking unless a second thread is launched). */
+#undef JEMALLOC_LAZY_LOCK
+
+/* One page is 2^STATIC_PAGE_SHIFT bytes. */
+#undef STATIC_PAGE_SHIFT
+
+/*
+ * If defined, use munmap() to unmap freed chunks, rather than storing them for
+ * later reuse.  This is disabled by default on Linux because common sequences
+ * of mmap()/munmap() calls will cause virtual memory map holes.
+ */
+#undef JEMALLOC_MUNMAP
+
+/*
+ * If defined, use mremap(...MREMAP_FIXED...) for huge realloc().  This is
+ * disabled by default because it is Linux-specific and it will cause virtual
+ * memory map holes, much like munmap(2) does.
+ */
+#undef JEMALLOC_MREMAP
+
+/* TLS is used to map arenas and magazine caches to threads. */
+#undef JEMALLOC_TLS
+
+/*
+ * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside
+ * within jemalloc-owned chunks before dereferencing them.
+ */
+#undef JEMALLOC_IVSALLOC
+
+/*
+ * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
+ */
+#undef JEMALLOC_ZONE
+#undef JEMALLOC_ZONE_VERSION
+
+/*
+ * Methods for purging unused pages differ between operating systems.
+ *
+ *   madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages,
+ *                                 such that new pages will be demand-zeroed if
+ *                                 the address region is later touched.
+ *   madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being
+ *                             unused, such that they will be discarded rather
+ *                             than swapped out.
+ */
+#undef JEMALLOC_PURGE_MADVISE_DONTNEED
+#undef JEMALLOC_PURGE_MADVISE_FREE
+
+/*
+ * Define if operating system has alloca.h header.
+ */
+#undef JEMALLOC_HAS_ALLOCA_H
+
+/* sizeof(int) == 2^LG_SIZEOF_INT. */
+#undef LG_SIZEOF_INT
+
+/* sizeof(long) == 2^LG_SIZEOF_LONG. */
+#undef LG_SIZEOF_LONG
+
+/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */
+#undef LG_SIZEOF_INTMAX_T
+
+#endif /* JEMALLOC_INTERNAL_DEFS_H_ */

include/jemalloc/internal/jemalloc_internal_macros.h

@@ -0,0 +1,47 @@
+/*
+ * JEMALLOC_ALWAYS_INLINE and JEMALLOC_INLINE are used within header files for
+ * functions that are static inline functions if inlining is enabled, and
+ * single-definition library-private functions if inlining is disabled.
+ *
+ * JEMALLOC_ALWAYS_INLINE_C and JEMALLOC_INLINE_C are for use in .c files, in
+ * which case the denoted functions are always static, regardless of whether
+ * inlining is enabled.
+ */
+#if defined(JEMALLOC_DEBUG) || defined(JEMALLOC_CODE_COVERAGE)
+   /* Disable inlining to make debugging/profiling easier. */
+#  define JEMALLOC_ALWAYS_INLINE
+#  define JEMALLOC_ALWAYS_INLINE_C static
+#  define JEMALLOC_INLINE
+#  define JEMALLOC_INLINE_C static
+#  define inline
+#else
+#  define JEMALLOC_ENABLE_INLINE
+#  ifdef JEMALLOC_HAVE_ATTR
+#    define JEMALLOC_ALWAYS_INLINE \
+	 static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline)
+#    define JEMALLOC_ALWAYS_INLINE_C \
+	 static inline JEMALLOC_ATTR(always_inline)
+#  else
+#    define JEMALLOC_ALWAYS_INLINE static inline
+#    define JEMALLOC_ALWAYS_INLINE_C static inline
+#  endif
+#  define JEMALLOC_INLINE static inline
+#  define JEMALLOC_INLINE_C static inline
+#  ifdef _MSC_VER
+#    define inline _inline
+#  endif
+#endif
+
+#ifdef JEMALLOC_CC_SILENCE
+#  define UNUSED JEMALLOC_ATTR(unused)
+#else
+#  define UNUSED
+#endif
+
+#define	ZU(z)	((size_t)z)
+#define	QU(q)	((uint64_t)q)
+#define	QI(q)	((int64_t)q)
+
+#ifndef __DECONST
+#  define	__DECONST(type, var)	((type)(uintptr_t)(const void *)(var))
+#endif

include/jemalloc/internal/private_namespace.h

@@ -1,392 +0,0 @@
-#define	a0calloc JEMALLOC_N(a0calloc)
-#define	a0free JEMALLOC_N(a0free)
-#define	a0malloc JEMALLOC_N(a0malloc)
-#define	arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small)
-#define	arena_bin_index JEMALLOC_N(arena_bin_index)
-#define	arena_bin_info JEMALLOC_N(arena_bin_info)
-#define	arena_boot JEMALLOC_N(arena_boot)
-#define	arena_dalloc JEMALLOC_N(arena_dalloc)
-#define	arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin)
-#define	arena_dalloc_bin_locked JEMALLOC_N(arena_dalloc_bin_locked)
-#define	arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small)
-#define	arena_dalloc_large JEMALLOC_N(arena_dalloc_large)
-#define	arena_dalloc_large_locked JEMALLOC_N(arena_dalloc_large_locked)
-#define	arena_dalloc_small JEMALLOC_N(arena_dalloc_small)
-#define	arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get)
-#define	arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set)
-#define	arena_malloc JEMALLOC_N(arena_malloc)
-#define	arena_malloc_large JEMALLOC_N(arena_malloc_large)
-#define	arena_malloc_small JEMALLOC_N(arena_malloc_small)
-#define	arena_mapbits_allocated_get JEMALLOC_N(arena_mapbits_allocated_get)
-#define	arena_mapbits_binind_get JEMALLOC_N(arena_mapbits_binind_get)
-#define	arena_mapbits_dirty_get JEMALLOC_N(arena_mapbits_dirty_get)
-#define	arena_mapbits_get JEMALLOC_N(arena_mapbits_get)
-#define	arena_mapbits_large_binind_set JEMALLOC_N(arena_mapbits_large_binind_set)
-#define	arena_mapbits_large_get JEMALLOC_N(arena_mapbits_large_get)
-#define	arena_mapbits_large_set JEMALLOC_N(arena_mapbits_large_set)
-#define	arena_mapbits_large_size_get JEMALLOC_N(arena_mapbits_large_size_get)
-#define	arena_mapbits_small_runind_get JEMALLOC_N(arena_mapbits_small_runind_get)
-#define	arena_mapbits_small_set JEMALLOC_N(arena_mapbits_small_set)
-#define	arena_mapbits_unallocated_set JEMALLOC_N(arena_mapbits_unallocated_set)
-#define	arena_mapbits_unallocated_size_get JEMALLOC_N(arena_mapbits_unallocated_size_get)
-#define	arena_mapbits_unallocated_size_set JEMALLOC_N(arena_mapbits_unallocated_size_set)
-#define	arena_mapbits_unzeroed_get JEMALLOC_N(arena_mapbits_unzeroed_get)
-#define	arena_mapbits_unzeroed_set JEMALLOC_N(arena_mapbits_unzeroed_set)
-#define	arena_mapbitsp_get JEMALLOC_N(arena_mapbitsp_get)
-#define	arena_mapbitsp_read JEMALLOC_N(arena_mapbitsp_read)
-#define	arena_mapbitsp_write JEMALLOC_N(arena_mapbitsp_write)
-#define	arena_mapp_get JEMALLOC_N(arena_mapp_get)
-#define	arena_maxclass JEMALLOC_N(arena_maxclass)
-#define	arena_new JEMALLOC_N(arena_new)
-#define	arena_palloc JEMALLOC_N(arena_palloc)
-#define	arena_postfork_child JEMALLOC_N(arena_postfork_child)
-#define	arena_postfork_parent JEMALLOC_N(arena_postfork_parent)
-#define	arena_prefork JEMALLOC_N(arena_prefork)
-#define	arena_prof_accum JEMALLOC_N(arena_prof_accum)
-#define	arena_prof_accum_impl JEMALLOC_N(arena_prof_accum_impl)
-#define	arena_prof_accum_locked JEMALLOC_N(arena_prof_accum_locked)
-#define	arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get)
-#define	arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set)
-#define	arena_prof_promoted JEMALLOC_N(arena_prof_promoted)
-#define	arena_ptr_small_binind_get JEMALLOC_N(arena_ptr_small_binind_get)
-#define	arena_purge_all JEMALLOC_N(arena_purge_all)
-#define	arena_ralloc JEMALLOC_N(arena_ralloc)
-#define	arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move)
-#define	arena_run_regind JEMALLOC_N(arena_run_regind)
-#define	arena_salloc JEMALLOC_N(arena_salloc)
-#define	arena_stats_merge JEMALLOC_N(arena_stats_merge)
-#define	arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small)
-#define	arenas JEMALLOC_N(arenas)
-#define	arenas_booted JEMALLOC_N(arenas_booted)
-#define	arenas_cleanup JEMALLOC_N(arenas_cleanup)
-#define	arenas_extend JEMALLOC_N(arenas_extend)
-#define	arenas_initialized JEMALLOC_N(arenas_initialized)
-#define	arenas_lock JEMALLOC_N(arenas_lock)
-#define	arenas_tls JEMALLOC_N(arenas_tls)
-#define	arenas_tsd JEMALLOC_N(arenas_tsd)
-#define	arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot)
-#define	arenas_tsd_cleanup_wrapper JEMALLOC_N(arenas_tsd_cleanup_wrapper)
-#define	arenas_tsd_get JEMALLOC_N(arenas_tsd_get)
-#define	arenas_tsd_get_wrapper JEMALLOC_N(arenas_tsd_get_wrapper)
-#define	arenas_tsd_set JEMALLOC_N(arenas_tsd_set)
-#define	atomic_add_u JEMALLOC_N(atomic_add_u)
-#define	atomic_add_uint32 JEMALLOC_N(atomic_add_uint32)
-#define	atomic_add_uint64 JEMALLOC_N(atomic_add_uint64)
-#define	atomic_add_z JEMALLOC_N(atomic_add_z)
-#define	atomic_sub_u JEMALLOC_N(atomic_sub_u)
-#define	atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32)
-#define	atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64)
-#define	atomic_sub_z JEMALLOC_N(atomic_sub_z)
-#define	base_alloc JEMALLOC_N(base_alloc)
-#define	base_boot JEMALLOC_N(base_boot)
-#define	base_calloc JEMALLOC_N(base_calloc)
-#define	base_node_alloc JEMALLOC_N(base_node_alloc)
-#define	base_node_dealloc JEMALLOC_N(base_node_dealloc)
-#define	base_postfork_child JEMALLOC_N(base_postfork_child)
-#define	base_postfork_parent JEMALLOC_N(base_postfork_parent)
-#define	base_prefork JEMALLOC_N(base_prefork)
-#define	bitmap_full JEMALLOC_N(bitmap_full)
-#define	bitmap_get JEMALLOC_N(bitmap_get)
-#define	bitmap_info_init JEMALLOC_N(bitmap_info_init)
-#define	bitmap_info_ngroups JEMALLOC_N(bitmap_info_ngroups)
-#define	bitmap_init JEMALLOC_N(bitmap_init)
-#define	bitmap_set JEMALLOC_N(bitmap_set)
-#define	bitmap_sfu JEMALLOC_N(bitmap_sfu)
-#define	bitmap_size JEMALLOC_N(bitmap_size)
-#define	bitmap_unset JEMALLOC_N(bitmap_unset)
-#define	bt_init JEMALLOC_N(bt_init)
-#define	buferror JEMALLOC_N(buferror)
-#define	choose_arena JEMALLOC_N(choose_arena)
-#define	choose_arena_hard JEMALLOC_N(choose_arena_hard)
-#define	chunk_alloc JEMALLOC_N(chunk_alloc)
-#define	chunk_alloc_dss JEMALLOC_N(chunk_alloc_dss)
-#define	chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap)
-#define	chunk_boot JEMALLOC_N(chunk_boot)
-#define	chunk_dealloc JEMALLOC_N(chunk_dealloc)
-#define	chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap)
-#define	chunk_dss_boot JEMALLOC_N(chunk_dss_boot)
-#define	chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child)
-#define	chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent)
-#define	chunk_dss_prec_get JEMALLOC_N(chunk_dss_prec_get)
-#define	chunk_dss_prec_set JEMALLOC_N(chunk_dss_prec_set)
-#define	chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork)
-#define	chunk_in_dss JEMALLOC_N(chunk_in_dss)
-#define	chunk_npages JEMALLOC_N(chunk_npages)
-#define	chunk_postfork_child JEMALLOC_N(chunk_postfork_child)
-#define	chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent)
-#define	chunk_prefork JEMALLOC_N(chunk_prefork)
-#define	chunk_unmap JEMALLOC_N(chunk_unmap)
-#define	chunks_mtx JEMALLOC_N(chunks_mtx)
-#define	chunks_rtree JEMALLOC_N(chunks_rtree)
-#define	chunksize JEMALLOC_N(chunksize)
-#define	chunksize_mask JEMALLOC_N(chunksize_mask)
-#define	ckh_bucket_search JEMALLOC_N(ckh_bucket_search)
-#define	ckh_count JEMALLOC_N(ckh_count)
-#define	ckh_delete JEMALLOC_N(ckh_delete)
-#define	ckh_evict_reloc_insert JEMALLOC_N(ckh_evict_reloc_insert)
-#define	ckh_insert JEMALLOC_N(ckh_insert)
-#define	ckh_isearch JEMALLOC_N(ckh_isearch)
-#define	ckh_iter JEMALLOC_N(ckh_iter)
-#define	ckh_new JEMALLOC_N(ckh_new)
-#define	ckh_pointer_hash JEMALLOC_N(ckh_pointer_hash)
-#define	ckh_pointer_keycomp JEMALLOC_N(ckh_pointer_keycomp)
-#define	ckh_rebuild JEMALLOC_N(ckh_rebuild)
-#define	ckh_remove JEMALLOC_N(ckh_remove)
-#define	ckh_search JEMALLOC_N(ckh_search)
-#define	ckh_string_hash JEMALLOC_N(ckh_string_hash)
-#define	ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp)
-#define	ckh_try_bucket_insert JEMALLOC_N(ckh_try_bucket_insert)
-#define	ckh_try_insert JEMALLOC_N(ckh_try_insert)
-#define	ctl_boot JEMALLOC_N(ctl_boot)
-#define	ctl_bymib JEMALLOC_N(ctl_bymib)
-#define	ctl_byname JEMALLOC_N(ctl_byname)
-#define	ctl_nametomib JEMALLOC_N(ctl_nametomib)
-#define	ctl_postfork_child JEMALLOC_N(ctl_postfork_child)
-#define	ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent)
-#define	ctl_prefork JEMALLOC_N(ctl_prefork)
-#define	dss_prec_names JEMALLOC_N(dss_prec_names)
-#define	extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first)
-#define	extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert)
-#define	extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter)
-#define	extent_tree_ad_iter_recurse JEMALLOC_N(extent_tree_ad_iter_recurse)
-#define	extent_tree_ad_iter_start JEMALLOC_N(extent_tree_ad_iter_start)
-#define	extent_tree_ad_last JEMALLOC_N(extent_tree_ad_last)
-#define	extent_tree_ad_new JEMALLOC_N(extent_tree_ad_new)
-#define	extent_tree_ad_next JEMALLOC_N(extent_tree_ad_next)
-#define	extent_tree_ad_nsearch JEMALLOC_N(extent_tree_ad_nsearch)
-#define	extent_tree_ad_prev JEMALLOC_N(extent_tree_ad_prev)
-#define	extent_tree_ad_psearch JEMALLOC_N(extent_tree_ad_psearch)
-#define	extent_tree_ad_remove JEMALLOC_N(extent_tree_ad_remove)
-#define	extent_tree_ad_reverse_iter JEMALLOC_N(extent_tree_ad_reverse_iter)
-#define	extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse)
-#define	extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start)
-#define	extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search)
-#define	extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first)
-#define	extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert)
-#define	extent_tree_szad_iter JEMALLOC_N(extent_tree_szad_iter)
-#define	extent_tree_szad_iter_recurse JEMALLOC_N(extent_tree_szad_iter_recurse)
-#define	extent_tree_szad_iter_start JEMALLOC_N(extent_tree_szad_iter_start)
-#define	extent_tree_szad_last JEMALLOC_N(extent_tree_szad_last)
-#define	extent_tree_szad_new JEMALLOC_N(extent_tree_szad_new)
-#define	extent_tree_szad_next JEMALLOC_N(extent_tree_szad_next)
-#define	extent_tree_szad_nsearch JEMALLOC_N(extent_tree_szad_nsearch)
-#define	extent_tree_szad_prev JEMALLOC_N(extent_tree_szad_prev)
-#define	extent_tree_szad_psearch JEMALLOC_N(extent_tree_szad_psearch)
-#define	extent_tree_szad_remove JEMALLOC_N(extent_tree_szad_remove)
-#define	extent_tree_szad_reverse_iter JEMALLOC_N(extent_tree_szad_reverse_iter)
-#define	extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse)
-#define	extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start)
-#define	extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search)
-#define	get_errno JEMALLOC_N(get_errno)
-#define	hash JEMALLOC_N(hash)
-#define	hash_fmix_32 JEMALLOC_N(hash_fmix_32)
-#define	hash_fmix_64 JEMALLOC_N(hash_fmix_64)
-#define	hash_get_block_32 JEMALLOC_N(hash_get_block_32)
-#define	hash_get_block_64 JEMALLOC_N(hash_get_block_64)
-#define	hash_rotl_32 JEMALLOC_N(hash_rotl_32)
-#define	hash_rotl_64 JEMALLOC_N(hash_rotl_64)
-#define	hash_x64_128 JEMALLOC_N(hash_x64_128)
-#define	hash_x86_128 JEMALLOC_N(hash_x86_128)
-#define	hash_x86_32 JEMALLOC_N(hash_x86_32)
-#define	huge_allocated JEMALLOC_N(huge_allocated)
-#define	huge_boot JEMALLOC_N(huge_boot)
-#define	huge_dalloc JEMALLOC_N(huge_dalloc)
-#define	huge_malloc JEMALLOC_N(huge_malloc)
-#define	huge_mtx JEMALLOC_N(huge_mtx)
-#define	huge_ndalloc JEMALLOC_N(huge_ndalloc)
-#define	huge_nmalloc JEMALLOC_N(huge_nmalloc)
-#define	huge_palloc JEMALLOC_N(huge_palloc)
-#define	huge_postfork_child JEMALLOC_N(huge_postfork_child)
-#define	huge_postfork_parent JEMALLOC_N(huge_postfork_parent)
-#define	huge_prefork JEMALLOC_N(huge_prefork)
-#define	huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get)
-#define	huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set)
-#define	huge_ralloc JEMALLOC_N(huge_ralloc)
-#define	huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move)
-#define	huge_salloc JEMALLOC_N(huge_salloc)
-#define	iallocm JEMALLOC_N(iallocm)
-#define	icalloc JEMALLOC_N(icalloc)
-#define	icallocx JEMALLOC_N(icallocx)
-#define	idalloc JEMALLOC_N(idalloc)
-#define	idallocx JEMALLOC_N(idallocx)
-#define	imalloc JEMALLOC_N(imalloc)
-#define	imallocx JEMALLOC_N(imallocx)
-#define	ipalloc JEMALLOC_N(ipalloc)
-#define	ipallocx JEMALLOC_N(ipallocx)
-#define	iqalloc JEMALLOC_N(iqalloc)
-#define	iqallocx JEMALLOC_N(iqallocx)
-#define	iralloc JEMALLOC_N(iralloc)
-#define	irallocx JEMALLOC_N(irallocx)
-#define	isalloc JEMALLOC_N(isalloc)
-#define	isthreaded JEMALLOC_N(isthreaded)
-#define	ivsalloc JEMALLOC_N(ivsalloc)
-#define	jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child)
-#define	jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent)
-#define	jemalloc_prefork JEMALLOC_N(jemalloc_prefork)
-#define	malloc_cprintf JEMALLOC_N(malloc_cprintf)
-#define	malloc_mutex_init JEMALLOC_N(malloc_mutex_init)
-#define	malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock)
-#define	malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child)
-#define	malloc_mutex_postfork_parent JEMALLOC_N(malloc_mutex_postfork_parent)
-#define	malloc_mutex_prefork JEMALLOC_N(malloc_mutex_prefork)
-#define	malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock)
-#define	malloc_printf JEMALLOC_N(malloc_printf)
-#define	malloc_snprintf JEMALLOC_N(malloc_snprintf)
-#define	malloc_strtoumax JEMALLOC_N(malloc_strtoumax)
-#define	malloc_tsd_boot JEMALLOC_N(malloc_tsd_boot)
-#define	malloc_tsd_cleanup_register JEMALLOC_N(malloc_tsd_cleanup_register)
-#define	malloc_tsd_dalloc JEMALLOC_N(malloc_tsd_dalloc)
-#define	malloc_tsd_malloc JEMALLOC_N(malloc_tsd_malloc)
-#define	malloc_tsd_no_cleanup JEMALLOC_N(malloc_tsd_no_cleanup)
-#define	malloc_vcprintf JEMALLOC_N(malloc_vcprintf)
-#define	malloc_vsnprintf JEMALLOC_N(malloc_vsnprintf)
-#define	malloc_write JEMALLOC_N(malloc_write)
-#define	map_bias JEMALLOC_N(map_bias)
-#define	mb_write JEMALLOC_N(mb_write)
-#define	mutex_boot JEMALLOC_N(mutex_boot)
-#define	narenas_auto JEMALLOC_N(narenas_auto)
-#define	narenas_total JEMALLOC_N(narenas_total)
-#define	narenas_total_get JEMALLOC_N(narenas_total_get)
-#define	ncpus JEMALLOC_N(ncpus)
-#define	nhbins JEMALLOC_N(nhbins)
-#define	opt_abort JEMALLOC_N(opt_abort)
-#define	opt_junk JEMALLOC_N(opt_junk)
-#define	opt_lg_chunk JEMALLOC_N(opt_lg_chunk)
-#define	opt_lg_dirty_mult JEMALLOC_N(opt_lg_dirty_mult)
-#define	opt_lg_prof_interval JEMALLOC_N(opt_lg_prof_interval)
-#define	opt_lg_prof_sample JEMALLOC_N(opt_lg_prof_sample)
-#define	opt_lg_tcache_max JEMALLOC_N(opt_lg_tcache_max)
-#define	opt_narenas JEMALLOC_N(opt_narenas)
-#define	opt_prof JEMALLOC_N(opt_prof)
-#define	opt_prof_accum JEMALLOC_N(opt_prof_accum)
-#define	opt_prof_active JEMALLOC_N(opt_prof_active)
-#define	opt_prof_final JEMALLOC_N(opt_prof_final)
-#define	opt_prof_gdump JEMALLOC_N(opt_prof_gdump)
-#define	opt_prof_leak JEMALLOC_N(opt_prof_leak)
-#define	opt_prof_prefix JEMALLOC_N(opt_prof_prefix)
-#define	opt_quarantine JEMALLOC_N(opt_quarantine)
-#define	opt_redzone JEMALLOC_N(opt_redzone)
-#define	opt_stats_print JEMALLOC_N(opt_stats_print)
-#define	opt_tcache JEMALLOC_N(opt_tcache)
-#define	opt_utrace JEMALLOC_N(opt_utrace)
-#define	opt_valgrind JEMALLOC_N(opt_valgrind)
-#define	opt_xmalloc JEMALLOC_N(opt_xmalloc)
-#define	opt_zero JEMALLOC_N(opt_zero)
-#define	p2rz JEMALLOC_N(p2rz)
-#define	pages_purge JEMALLOC_N(pages_purge)
-#define	pow2_ceil JEMALLOC_N(pow2_ceil)
-#define	prof_backtrace JEMALLOC_N(prof_backtrace)
-#define	prof_boot0 JEMALLOC_N(prof_boot0)
-#define	prof_boot1 JEMALLOC_N(prof_boot1)
-#define	prof_boot2 JEMALLOC_N(prof_boot2)
-#define	prof_ctx_get JEMALLOC_N(prof_ctx_get)
-#define	prof_ctx_set JEMALLOC_N(prof_ctx_set)
-#define	prof_free JEMALLOC_N(prof_free)
-#define	prof_gdump JEMALLOC_N(prof_gdump)
-#define	prof_idump JEMALLOC_N(prof_idump)
-#define	prof_interval JEMALLOC_N(prof_interval)
-#define	prof_lookup JEMALLOC_N(prof_lookup)
-#define	prof_malloc JEMALLOC_N(prof_malloc)
-#define	prof_mdump JEMALLOC_N(prof_mdump)
-#define	prof_postfork_child JEMALLOC_N(prof_postfork_child)
-#define	prof_postfork_parent JEMALLOC_N(prof_postfork_parent)
-#define	prof_prefork JEMALLOC_N(prof_prefork)
-#define	prof_promote JEMALLOC_N(prof_promote)
-#define	prof_realloc JEMALLOC_N(prof_realloc)
-#define	prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update)
-#define	prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update)
-#define	prof_tdata_booted JEMALLOC_N(prof_tdata_booted)
-#define	prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup)
-#define	prof_tdata_get JEMALLOC_N(prof_tdata_get)
-#define	prof_tdata_init JEMALLOC_N(prof_tdata_init)
-#define	prof_tdata_initialized JEMALLOC_N(prof_tdata_initialized)
-#define	prof_tdata_tls JEMALLOC_N(prof_tdata_tls)
-#define	prof_tdata_tsd JEMALLOC_N(prof_tdata_tsd)
-#define	prof_tdata_tsd_boot JEMALLOC_N(prof_tdata_tsd_boot)
-#define	prof_tdata_tsd_cleanup_wrapper JEMALLOC_N(prof_tdata_tsd_cleanup_wrapper)
-#define	prof_tdata_tsd_get JEMALLOC_N(prof_tdata_tsd_get)
-#define	prof_tdata_tsd_get_wrapper JEMALLOC_N(prof_tdata_tsd_get_wrapper)
-#define	prof_tdata_tsd_set JEMALLOC_N(prof_tdata_tsd_set)
-#define	quarantine JEMALLOC_N(quarantine)
-#define	quarantine_alloc_hook JEMALLOC_N(quarantine_alloc_hook)
-#define	quarantine_boot JEMALLOC_N(quarantine_boot)
-#define	quarantine_booted JEMALLOC_N(quarantine_booted)
-#define	quarantine_cleanup JEMALLOC_N(quarantine_cleanup)
-#define	quarantine_init JEMALLOC_N(quarantine_init)
-#define	quarantine_tls JEMALLOC_N(quarantine_tls)
-#define	quarantine_tsd JEMALLOC_N(quarantine_tsd)
-#define	quarantine_tsd_boot JEMALLOC_N(quarantine_tsd_boot)
-#define	quarantine_tsd_cleanup_wrapper JEMALLOC_N(quarantine_tsd_cleanup_wrapper)
-#define	quarantine_tsd_get JEMALLOC_N(quarantine_tsd_get)
-#define	quarantine_tsd_get_wrapper JEMALLOC_N(quarantine_tsd_get_wrapper)
-#define	quarantine_tsd_set JEMALLOC_N(quarantine_tsd_set)
-#define	register_zone JEMALLOC_N(register_zone)
-#define	rtree_get JEMALLOC_N(rtree_get)
-#define	rtree_get_locked JEMALLOC_N(rtree_get_locked)
-#define	rtree_new JEMALLOC_N(rtree_new)
-#define	rtree_postfork_child JEMALLOC_N(rtree_postfork_child)
-#define	rtree_postfork_parent JEMALLOC_N(rtree_postfork_parent)
-#define	rtree_prefork JEMALLOC_N(rtree_prefork)
-#define	rtree_set JEMALLOC_N(rtree_set)
-#define	s2u JEMALLOC_N(s2u)
-#define	sa2u JEMALLOC_N(sa2u)
-#define	set_errno JEMALLOC_N(set_errno)
-#define	stats_cactive JEMALLOC_N(stats_cactive)
-#define	stats_cactive_add JEMALLOC_N(stats_cactive_add)
-#define	stats_cactive_get JEMALLOC_N(stats_cactive_get)
-#define	stats_cactive_sub JEMALLOC_N(stats_cactive_sub)
-#define	stats_chunks JEMALLOC_N(stats_chunks)
-#define	stats_print JEMALLOC_N(stats_print)
-#define	tcache_alloc_easy JEMALLOC_N(tcache_alloc_easy)
-#define	tcache_alloc_large JEMALLOC_N(tcache_alloc_large)
-#define	tcache_alloc_small JEMALLOC_N(tcache_alloc_small)
-#define	tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard)
-#define	tcache_arena_associate JEMALLOC_N(tcache_arena_associate)
-#define	tcache_arena_dissociate JEMALLOC_N(tcache_arena_dissociate)
-#define	tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large)
-#define	tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small)
-#define	tcache_bin_info JEMALLOC_N(tcache_bin_info)
-#define	tcache_boot0 JEMALLOC_N(tcache_boot0)
-#define	tcache_boot1 JEMALLOC_N(tcache_boot1)
-#define	tcache_booted JEMALLOC_N(tcache_booted)
-#define	tcache_create JEMALLOC_N(tcache_create)
-#define	tcache_dalloc_large JEMALLOC_N(tcache_dalloc_large)
-#define	tcache_dalloc_small JEMALLOC_N(tcache_dalloc_small)
-#define	tcache_destroy JEMALLOC_N(tcache_destroy)
-#define	tcache_enabled_booted JEMALLOC_N(tcache_enabled_booted)
-#define	tcache_enabled_get JEMALLOC_N(tcache_enabled_get)
-#define	tcache_enabled_initialized JEMALLOC_N(tcache_enabled_initialized)
-#define	tcache_enabled_set JEMALLOC_N(tcache_enabled_set)
-#define	tcache_enabled_tls JEMALLOC_N(tcache_enabled_tls)
-#define	tcache_enabled_tsd JEMALLOC_N(tcache_enabled_tsd)
-#define	tcache_enabled_tsd_boot JEMALLOC_N(tcache_enabled_tsd_boot)
-#define	tcache_enabled_tsd_cleanup_wrapper JEMALLOC_N(tcache_enabled_tsd_cleanup_wrapper)
-#define	tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get)
-#define	tcache_enabled_tsd_get_wrapper JEMALLOC_N(tcache_enabled_tsd_get_wrapper)
-#define	tcache_enabled_tsd_set JEMALLOC_N(tcache_enabled_tsd_set)
-#define	tcache_event JEMALLOC_N(tcache_event)
-#define	tcache_event_hard JEMALLOC_N(tcache_event_hard)
-#define	tcache_flush JEMALLOC_N(tcache_flush)
-#define	tcache_get JEMALLOC_N(tcache_get)
-#define	tcache_initialized JEMALLOC_N(tcache_initialized)
-#define	tcache_maxclass JEMALLOC_N(tcache_maxclass)
-#define	tcache_salloc JEMALLOC_N(tcache_salloc)
-#define	tcache_stats_merge JEMALLOC_N(tcache_stats_merge)
-#define	tcache_thread_cleanup JEMALLOC_N(tcache_thread_cleanup)
-#define	tcache_tls JEMALLOC_N(tcache_tls)
-#define	tcache_tsd JEMALLOC_N(tcache_tsd)
-#define	tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot)
-#define	tcache_tsd_cleanup_wrapper JEMALLOC_N(tcache_tsd_cleanup_wrapper)
-#define	tcache_tsd_get JEMALLOC_N(tcache_tsd_get)
-#define	tcache_tsd_get_wrapper JEMALLOC_N(tcache_tsd_get_wrapper)
-#define	tcache_tsd_set JEMALLOC_N(tcache_tsd_set)
-#define	thread_allocated_booted JEMALLOC_N(thread_allocated_booted)
-#define	thread_allocated_initialized JEMALLOC_N(thread_allocated_initialized)
-#define	thread_allocated_tls JEMALLOC_N(thread_allocated_tls)
-#define	thread_allocated_tsd JEMALLOC_N(thread_allocated_tsd)
-#define	thread_allocated_tsd_boot JEMALLOC_N(thread_allocated_tsd_boot)
-#define	thread_allocated_tsd_cleanup_wrapper JEMALLOC_N(thread_allocated_tsd_cleanup_wrapper)
-#define	thread_allocated_tsd_get JEMALLOC_N(thread_allocated_tsd_get)
-#define	thread_allocated_tsd_get_wrapper JEMALLOC_N(thread_allocated_tsd_get_wrapper)
-#define	thread_allocated_tsd_set JEMALLOC_N(thread_allocated_tsd_set)
-#define	u2rz JEMALLOC_N(u2rz)

include/jemalloc/internal/private_namespace.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for symbol in `cat $1` ; do
+  echo "#define	${symbol} JEMALLOC_N(${symbol})"
+done

include/jemalloc/internal/private_symbols.txt

@@ -0,0 +1,412 @@
+a0calloc
+a0free
+a0malloc
+arena_alloc_junk_small
+arena_bin_index
+arena_bin_info
+arena_boot
+arena_dalloc
+arena_dalloc_bin
+arena_dalloc_bin_locked
+arena_dalloc_junk_large
+arena_dalloc_junk_small
+arena_dalloc_large
+arena_dalloc_large_locked
+arena_dalloc_small
+arena_dss_prec_get
+arena_dss_prec_set
+arena_malloc
+arena_malloc_large
+arena_malloc_small
+arena_mapbits_allocated_get
+arena_mapbits_binind_get
+arena_mapbits_dirty_get
+arena_mapbits_get
+arena_mapbits_large_binind_set
+arena_mapbits_large_get
+arena_mapbits_large_set
+arena_mapbits_large_size_get
+arena_mapbits_small_runind_get
+arena_mapbits_small_set
+arena_mapbits_unallocated_set
+arena_mapbits_unallocated_size_get
+arena_mapbits_unallocated_size_set
+arena_mapbits_unzeroed_get
+arena_mapbits_unzeroed_set
+arena_mapbitsp_get
+arena_mapbitsp_read
+arena_mapbitsp_write
+arena_mapp_get
+arena_maxclass
+arena_new
+arena_palloc
+arena_postfork_child
+arena_postfork_parent
+arena_prefork
+arena_prof_accum
+arena_prof_accum_impl
+arena_prof_accum_locked
+arena_prof_ctx_get
+arena_prof_ctx_set
+arena_prof_promoted
+arena_ptr_small_binind_get
+arena_purge_all
+arena_quarantine_junk_small
+arena_ralloc
+arena_ralloc_junk_large
+arena_ralloc_no_move
+arena_redzone_corruption
+arena_run_regind
+arena_salloc
+arena_stats_merge
+arena_tcache_fill_small
+arenas
+arenas_booted
+arenas_cleanup
+arenas_extend
+arenas_initialized
+arenas_lock
+arenas_tls
+arenas_tsd
+arenas_tsd_boot
+arenas_tsd_cleanup_wrapper
+arenas_tsd_get
+arenas_tsd_get_wrapper
+arenas_tsd_init_head
+arenas_tsd_set
+atomic_add_u
+atomic_add_uint32
+atomic_add_uint64
+atomic_add_z
+atomic_sub_u
+atomic_sub_uint32
+atomic_sub_uint64
+atomic_sub_z
+base_alloc
+base_boot
+base_calloc
+base_node_alloc
+base_node_dealloc
+base_postfork_child
+base_postfork_parent
+base_prefork
+bitmap_full
+bitmap_get
+bitmap_info_init
+bitmap_info_ngroups
+bitmap_init
+bitmap_set
+bitmap_sfu
+bitmap_size
+bitmap_unset
+bt_init
+buferror
+choose_arena
+choose_arena_hard
+chunk_alloc
+chunk_alloc_dss
+chunk_alloc_mmap
+chunk_boot
+chunk_dealloc
+chunk_dealloc_mmap
+chunk_dss_boot
+chunk_dss_postfork_child
+chunk_dss_postfork_parent
+chunk_dss_prec_get
+chunk_dss_prec_set
+chunk_dss_prefork
+chunk_in_dss
+chunk_npages
+chunk_postfork_child
+chunk_postfork_parent
+chunk_prefork
+chunk_unmap
+chunks_mtx
+chunks_rtree
+chunksize
+chunksize_mask
+ckh_bucket_search
+ckh_count
+ckh_delete
+ckh_evict_reloc_insert
+ckh_insert
+ckh_isearch
+ckh_iter
+ckh_new
+ckh_pointer_hash
+ckh_pointer_keycomp
+ckh_rebuild
+ckh_remove
+ckh_search
+ckh_string_hash
+ckh_string_keycomp
+ckh_try_bucket_insert
+ckh_try_insert
+ctl_boot
+ctl_bymib
+ctl_byname
+ctl_nametomib
+ctl_postfork_child
+ctl_postfork_parent
+ctl_prefork
+dss_prec_names
+extent_tree_ad_first
+extent_tree_ad_insert
+extent_tree_ad_iter
+extent_tree_ad_iter_recurse
+extent_tree_ad_iter_start
+extent_tree_ad_last
+extent_tree_ad_new
+extent_tree_ad_next
+extent_tree_ad_nsearch
+extent_tree_ad_prev
+extent_tree_ad_psearch
+extent_tree_ad_remove
+extent_tree_ad_reverse_iter
+extent_tree_ad_reverse_iter_recurse
+extent_tree_ad_reverse_iter_start
+extent_tree_ad_search
+extent_tree_szad_first
+extent_tree_szad_insert
+extent_tree_szad_iter
+extent_tree_szad_iter_recurse
+extent_tree_szad_iter_start
+extent_tree_szad_last
+extent_tree_szad_new
+extent_tree_szad_next
+extent_tree_szad_nsearch
+extent_tree_szad_prev
+extent_tree_szad_psearch
+extent_tree_szad_remove
+extent_tree_szad_reverse_iter
+extent_tree_szad_reverse_iter_recurse
+extent_tree_szad_reverse_iter_start
+extent_tree_szad_search
+get_errno
+hash
+hash_fmix_32
+hash_fmix_64
+hash_get_block_32
+hash_get_block_64
+hash_rotl_32
+hash_rotl_64
+hash_x64_128
+hash_x86_128
+hash_x86_32
+huge_allocated
+huge_boot
+huge_dalloc
+huge_dalloc_junk
+huge_malloc
+huge_mtx
+huge_ndalloc
+huge_nmalloc
+huge_palloc
+huge_postfork_child
+huge_postfork_parent
+huge_prefork
+huge_prof_ctx_get
+huge_prof_ctx_set
+huge_ralloc
+huge_ralloc_no_move
+huge_salloc
+iallocm
+icalloc
+icalloct
+idalloc
+idalloct
+imalloc
+imalloct
+ipalloc
+ipalloct
+iqalloc
+iqalloct
+iralloc
+iralloct
+iralloct_realign
+isalloc
+isthreaded
+ivsalloc
+ixalloc
+jemalloc_postfork_child
+jemalloc_postfork_parent
+jemalloc_prefork
+malloc_cprintf
+malloc_mutex_init
+malloc_mutex_lock
+malloc_mutex_postfork_child
+malloc_mutex_postfork_parent
+malloc_mutex_prefork
+malloc_mutex_unlock
+malloc_printf
+malloc_snprintf
+malloc_strtoumax
+malloc_tsd_boot
+malloc_tsd_cleanup_register
+malloc_tsd_dalloc
+malloc_tsd_malloc
+malloc_tsd_no_cleanup
+malloc_vcprintf
+malloc_vsnprintf
+malloc_write
+map_bias
+mb_write
+mutex_boot
+narenas_auto
+narenas_total
+narenas_total_get
+ncpus
+nhbins
+opt_abort
+opt_dss
+opt_junk
+opt_lg_chunk
+opt_lg_dirty_mult
+opt_lg_prof_interval
+opt_lg_prof_sample
+opt_lg_tcache_max
+opt_narenas
+opt_prof
+opt_prof_accum
+opt_prof_active
+opt_prof_final
+opt_prof_gdump
+opt_prof_leak
+opt_prof_prefix
+opt_quarantine
+opt_redzone
+opt_stats_print
+opt_tcache
+opt_utrace
+opt_valgrind
+opt_xmalloc
+opt_zero
+p2rz
+pages_purge
+pow2_ceil
+prof_backtrace
+prof_boot0
+prof_boot1
+prof_boot2
+prof_bt_count
+prof_ctx_get
+prof_ctx_set
+prof_dump_open
+prof_free
+prof_gdump
+prof_idump
+prof_interval
+prof_lookup
+prof_malloc
+prof_mdump
+prof_postfork_child
+prof_postfork_parent
+prof_prefork
+prof_promote
+prof_realloc
+prof_sample_accum_update
+prof_sample_threshold_update
+prof_tdata_booted
+prof_tdata_cleanup
+prof_tdata_get
+prof_tdata_init
+prof_tdata_initialized
+prof_tdata_tls
+prof_tdata_tsd
+prof_tdata_tsd_boot
+prof_tdata_tsd_cleanup_wrapper
+prof_tdata_tsd_get
+prof_tdata_tsd_get_wrapper
+prof_tdata_tsd_init_head
+prof_tdata_tsd_set
+quarantine
+quarantine_alloc_hook
+quarantine_boot
+quarantine_booted
+quarantine_cleanup
+quarantine_init
+quarantine_tls
+quarantine_tsd
+quarantine_tsd_boot
+quarantine_tsd_cleanup_wrapper
+quarantine_tsd_get
+quarantine_tsd_get_wrapper
+quarantine_tsd_init_head
+quarantine_tsd_set
+register_zone
+rtree_delete
+rtree_get
+rtree_get_locked
+rtree_new
+rtree_postfork_child
+rtree_postfork_parent
+rtree_prefork
+rtree_set
+s2u
+sa2u
+set_errno
+small_size2bin
+stats_cactive
+stats_cactive_add
+stats_cactive_get
+stats_cactive_sub
+stats_chunks
+stats_print
+tcache_alloc_easy
+tcache_alloc_large
+tcache_alloc_small
+tcache_alloc_small_hard
+tcache_arena_associate
+tcache_arena_dissociate
+tcache_bin_flush_large
+tcache_bin_flush_small
+tcache_bin_info
+tcache_boot0
+tcache_boot1
+tcache_booted
+tcache_create
+tcache_dalloc_large
+tcache_dalloc_small
+tcache_destroy
+tcache_enabled_booted
+tcache_enabled_get
+tcache_enabled_initialized
+tcache_enabled_set
+tcache_enabled_tls
+tcache_enabled_tsd
+tcache_enabled_tsd_boot
+tcache_enabled_tsd_cleanup_wrapper
+tcache_enabled_tsd_get
+tcache_enabled_tsd_get_wrapper
+tcache_enabled_tsd_init_head
+tcache_enabled_tsd_set
+tcache_event
+tcache_event_hard
+tcache_flush
+tcache_get
+tcache_initialized
+tcache_maxclass
+tcache_salloc
+tcache_stats_merge
+tcache_thread_cleanup
+tcache_tls
+tcache_tsd
+tcache_tsd_boot
+tcache_tsd_cleanup_wrapper
+tcache_tsd_get
+tcache_tsd_get_wrapper
+tcache_tsd_init_head
+tcache_tsd_set
+thread_allocated_booted
+thread_allocated_initialized
+thread_allocated_tls
+thread_allocated_tsd
+thread_allocated_tsd_boot
+thread_allocated_tsd_cleanup_wrapper
+thread_allocated_tsd_get
+thread_allocated_tsd_get_wrapper
+thread_allocated_tsd_init_head
+thread_allocated_tsd_set
+tsd_init_check_recursion
+tsd_init_finish
+u2rz

include/jemalloc/internal/private_unnamespace.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for symbol in `cat $1` ; do
+  echo "#undef ${symbol}"
+done

include/jemalloc/internal/prng.h

@@ -25,7 +25,7 @@
  *   uint32_t state      : Seed value.
  *   const uint32_t a, c : See above discussion.
  */
-#define prng32(r, lg_range, state, a, c) do {				\
+#define	prng32(r, lg_range, state, a, c) do {				\
 	assert(lg_range > 0);						\
 	assert(lg_range <= 32);						\
 									\
@@ -35,7 +35,7 @@
 } while (false)
 
 /* Same as prng32(), but 64 bits of pseudo-randomness, using uint64_t. */
-#define prng64(r, lg_range, state, a, c) do {				\
+#define	prng64(r, lg_range, state, a, c) do {				\
 	assert(lg_range > 0);						\
 	assert(lg_range <= 64);						\
 									\

include/jemalloc/internal/prof.h

@@ -129,6 +129,7 @@ struct prof_ctx_s {
 	 * limbo due to one of:
 	 *   - Initializing per thread counters associated with this ctx.
 	 *   - Preparing to destroy this ctx.
+	 *   - Dumping a heap profile that includes this ctx.
 	 * nlimbo must be 1 (single destroyer) in order to safely destroy the
 	 * ctx.
 	 */
@@ -145,7 +146,11 @@ struct prof_ctx_s {
 	 * this context.
 	 */
 	ql_head(prof_thr_cnt_t)	cnts_ql;
+
+	/* Linkage for list of contexts to be dumped. */
+	ql_elm(prof_ctx_t)	dump_link;
 };
+typedef ql_head(prof_ctx_t) prof_ctx_list_t;
 
 struct prof_tdata_s {
 	/*
@@ -195,7 +200,12 @@ extern bool	opt_prof_gdump;       /* High-water memory dumping. */
 extern bool	opt_prof_final;       /* Final profile dumping. */
 extern bool	opt_prof_leak;        /* Dump leak summary at exit. */
 extern bool	opt_prof_accum;       /* Report cumulative bytes. */
-extern char	opt_prof_prefix[PATH_MAX + 1];
+extern char	opt_prof_prefix[
+    /* Minimize memory bloat for non-prof builds. */
+#ifdef JEMALLOC_PROF
+    PATH_MAX +
+#endif
+    1];
 
 /*
  * Profile dump interval, measured in bytes allocated.  Each arena triggers a
@@ -215,6 +225,11 @@ extern bool	prof_promote;
 void	bt_init(prof_bt_t *bt, void **vec);
 void	prof_backtrace(prof_bt_t *bt, unsigned nignore);
 prof_thr_cnt_t	*prof_lookup(prof_bt_t *bt);
+#ifdef JEMALLOC_JET
+size_t	prof_bt_count(void);
+typedef int (prof_dump_open_t)(bool, const char *);
+extern prof_dump_open_t *prof_dump_open;
+#endif
 void	prof_idump(void);
 bool	prof_mdump(const char *filename);
 void	prof_gdump(void);
@@ -289,11 +304,11 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
 prof_tdata_t	*prof_tdata_get(bool create);
 void	prof_sample_threshold_update(prof_tdata_t *prof_tdata);
 prof_ctx_t	*prof_ctx_get(const void *ptr);
-void	prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
+void	prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx);
 bool	prof_sample_accum_update(size_t size);
-void	prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt);
-void	prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
-    size_t old_size, prof_ctx_t *old_ctx);
+void	prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt);
+void	prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt,
+    size_t old_usize, prof_ctx_t *old_ctx);
 void	prof_free(const void *ptr, size_t size);
 #endif
 
@@ -320,6 +335,20 @@ prof_tdata_get(bool create)
 JEMALLOC_INLINE void
 prof_sample_threshold_update(prof_tdata_t *prof_tdata)
 {
+	/*
+	 * The body of this function is compiled out unless heap profiling is
+	 * enabled, so that it is possible to compile jemalloc with floating
+	 * point support completely disabled.  Avoiding floating point code is
+	 * important on memory-constrained systems, but it also enables a
+	 * workaround for versions of glibc that don't properly save/restore
+	 * floating point registers during dynamic lazy symbol loading (which
+	 * internally calls into whatever malloc implementation happens to be
+	 * integrated into the application).  Note that some compilers (e.g.
+	 * gcc 4.8) may use floating point registers for fast memory moves, so
+	 * jemalloc must be compiled with such optimizations disabled (e.g.
+	 * -mno-sse) in order for the workaround to be complete.
+	 */
+#ifdef JEMALLOC_PROF
 	uint64_t r;
 	double u;
 
@@ -341,7 +370,7 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata)
 	 *   Luc Devroye
 	 *   Springer-Verlag, New York, 1986
 	 *   pp 500
-	 *   (http://cg.scs.carleton.ca/~luc/rnbookindex.html)
+	 *   (http://luc.devroye.org/rnbookindex.html)
 	 */
 	prng64(r, 53, prof_tdata->prng_state,
 	    UINT64_C(6364136223846793005), UINT64_C(1442695040888963407));
@@ -349,6 +378,7 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata)
 	prof_tdata->threshold = (uint64_t)(log(u) /
 	    log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample))))
 	    + (uint64_t)1U;
+#endif
 }
 
 JEMALLOC_INLINE prof_ctx_t *
@@ -371,7 +401,7 @@ prof_ctx_get(const void *ptr)
 }
 
 JEMALLOC_INLINE void
-prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
+prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx)
 {
 	arena_chunk_t *chunk;
 
@@ -381,7 +411,7 @@ prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	if (chunk != ptr) {
 		/* Region. */
-		arena_prof_ctx_set(ptr, ctx);
+		arena_prof_ctx_set(ptr, usize, ctx);
 	} else
 		huge_prof_ctx_set(ptr, ctx);
 }
@@ -416,20 +446,20 @@ prof_sample_accum_update(size_t size)
 }
 
 JEMALLOC_INLINE void
-prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt)
+prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt)
 {
 
 	cassert(config_prof);
 	assert(ptr != NULL);
-	assert(size == isalloc(ptr, true));
+	assert(usize == isalloc(ptr, true));
 
 	if (opt_lg_prof_sample != 0) {
-		if (prof_sample_accum_update(size)) {
+		if (prof_sample_accum_update(usize)) {
 			/*
 			 * Don't sample.  For malloc()-like allocation, it is
 			 * always possible to tell in advance how large an
 			 * object's usable size will be, so there should never
-			 * be a difference between the size passed to
+			 * be a difference between the usize passed to
 			 * PROF_ALLOC_PREP() and prof_malloc().
 			 */
 			assert((uintptr_t)cnt == (uintptr_t)1U);
@@ -437,17 +467,17 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt)
 	}
 
 	if ((uintptr_t)cnt > (uintptr_t)1U) {
-		prof_ctx_set(ptr, cnt->ctx);
+		prof_ctx_set(ptr, usize, cnt->ctx);
 
 		cnt->epoch++;
 		/*********/
 		mb_write();
 		/*********/
 		cnt->cnts.curobjs++;
-		cnt->cnts.curbytes += size;
+		cnt->cnts.curbytes += usize;
 		if (opt_prof_accum) {
 			cnt->cnts.accumobjs++;
-			cnt->cnts.accumbytes += size;
+			cnt->cnts.accumbytes += usize;
 		}
 		/*********/
 		mb_write();
@@ -457,12 +487,12 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt)
 		mb_write();
 		/*********/
 	} else
-		prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U);
+		prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U);
 }
 
 JEMALLOC_INLINE void
-prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
-    size_t old_size, prof_ctx_t *old_ctx)
+prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt,
+    size_t old_usize, prof_ctx_t *old_ctx)
 {
 	prof_thr_cnt_t *told_cnt;
 
@@ -470,15 +500,15 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
 	assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U);
 
 	if (ptr != NULL) {
-		assert(size == isalloc(ptr, true));
+		assert(usize == isalloc(ptr, true));
 		if (opt_lg_prof_sample != 0) {
-			if (prof_sample_accum_update(size)) {
+			if (prof_sample_accum_update(usize)) {
 				/*
-				 * Don't sample.  The size passed to
+				 * Don't sample.  The usize passed to
 				 * PROF_ALLOC_PREP() was larger than what
 				 * actually got allocated, so a backtrace was
 				 * captured for this allocation, even though
-				 * its actual size was insufficient to cross
+				 * its actual usize was insufficient to cross
 				 * the sample threshold.
 				 */
 				cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
@@ -495,7 +525,7 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
 			 */
 			malloc_mutex_lock(old_ctx->lock);
 			old_ctx->cnt_merged.curobjs--;
-			old_ctx->cnt_merged.curbytes -= old_size;
+			old_ctx->cnt_merged.curbytes -= old_usize;
 			malloc_mutex_unlock(old_ctx->lock);
 			told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
 		}
@@ -505,23 +535,23 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
 	if ((uintptr_t)told_cnt > (uintptr_t)1U)
 		told_cnt->epoch++;
 	if ((uintptr_t)cnt > (uintptr_t)1U) {
-		prof_ctx_set(ptr, cnt->ctx);
+		prof_ctx_set(ptr, usize, cnt->ctx);
 		cnt->epoch++;
 	} else if (ptr != NULL)
-		prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U);
+		prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U);
 	/*********/
 	mb_write();
 	/*********/
 	if ((uintptr_t)told_cnt > (uintptr_t)1U) {
 		told_cnt->cnts.curobjs--;
-		told_cnt->cnts.curbytes -= old_size;
+		told_cnt->cnts.curbytes -= old_usize;
 	}
 	if ((uintptr_t)cnt > (uintptr_t)1U) {
 		cnt->cnts.curobjs++;
-		cnt->cnts.curbytes += size;
+		cnt->cnts.curbytes += usize;
 		if (opt_prof_accum) {
 			cnt->cnts.accumobjs++;
-			cnt->cnts.accumbytes += size;
+			cnt->cnts.accumbytes += usize;
 		}
 	}
 	/*********/

include/jemalloc/internal/public_namespace.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+for nm in `cat $1` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  echo "#define	je_${n} JEMALLOC_N(${n})"
+done

include/jemalloc/internal/public_unnamespace.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+for nm in `cat $1` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  echo "#undef je_${n}"
+done

include/jemalloc/internal/ql.h

@@ -1,61 +1,61 @@
 /*
  * List definitions.
  */
-#define ql_head(a_type)							\
+#define	ql_head(a_type)							\
 struct {								\
 	a_type *qlh_first;						\
 }
 
-#define ql_head_initializer(a_head) {NULL}
+#define	ql_head_initializer(a_head) {NULL}
 
-#define ql_elm(a_type)	qr(a_type)
+#define	ql_elm(a_type)	qr(a_type)
 
 /* List functions. */
-#define ql_new(a_head) do {						\
+#define	ql_new(a_head) do {						\
 	(a_head)->qlh_first = NULL;					\
 } while (0)
 
-#define ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field)
+#define	ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field)
 
-#define ql_first(a_head) ((a_head)->qlh_first)
+#define	ql_first(a_head) ((a_head)->qlh_first)
 
-#define ql_last(a_head, a_field)					\
+#define	ql_last(a_head, a_field)					\
 	((ql_first(a_head) != NULL)					\
 	    ? qr_prev(ql_first(a_head), a_field) : NULL)
 
-#define ql_next(a_head, a_elm, a_field)					\
+#define	ql_next(a_head, a_elm, a_field)					\
 	((ql_last(a_head, a_field) != (a_elm))				\
 	    ? qr_next((a_elm), a_field)	: NULL)
 
-#define ql_prev(a_head, a_elm, a_field)					\
+#define	ql_prev(a_head, a_elm, a_field)					\
 	((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field)	\
 				       : NULL)
 
-#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do {		\
+#define	ql_before_insert(a_head, a_qlelm, a_elm, a_field) do {		\
 	qr_before_insert((a_qlelm), (a_elm), a_field);			\
 	if (ql_first(a_head) == (a_qlelm)) {				\
 		ql_first(a_head) = (a_elm);				\
 	}								\
 } while (0)
 
-#define ql_after_insert(a_qlelm, a_elm, a_field)			\
+#define	ql_after_insert(a_qlelm, a_elm, a_field)			\
 	qr_after_insert((a_qlelm), (a_elm), a_field)
 
-#define ql_head_insert(a_head, a_elm, a_field) do {			\
+#define	ql_head_insert(a_head, a_elm, a_field) do {			\
 	if (ql_first(a_head) != NULL) {					\
 		qr_before_insert(ql_first(a_head), (a_elm), a_field);	\
 	}								\
 	ql_first(a_head) = (a_elm);					\
 } while (0)
 
-#define ql_tail_insert(a_head, a_elm, a_field) do {			\
+#define	ql_tail_insert(a_head, a_elm, a_field) do {			\
 	if (ql_first(a_head) != NULL) {					\
 		qr_before_insert(ql_first(a_head), (a_elm), a_field);	\
 	}								\
 	ql_first(a_head) = qr_next((a_elm), a_field);			\
 } while (0)
 
-#define ql_remove(a_head, a_elm, a_field) do {				\
+#define	ql_remove(a_head, a_elm, a_field) do {				\
 	if (ql_first(a_head) == (a_elm)) {				\
 		ql_first(a_head) = qr_next(ql_first(a_head), a_field);	\
 	}								\
@@ -66,18 +66,18 @@ struct {								\
 	}								\
 } while (0)
 
-#define ql_head_remove(a_head, a_type, a_field) do {			\
+#define	ql_head_remove(a_head, a_type, a_field) do {			\
 	a_type *t = ql_first(a_head);					\
 	ql_remove((a_head), t, a_field);				\
 } while (0)
 
-#define ql_tail_remove(a_head, a_type, a_field) do {			\
+#define	ql_tail_remove(a_head, a_type, a_field) do {			\
 	a_type *t = ql_last(a_head, a_field);				\
 	ql_remove((a_head), t, a_field);				\
 } while (0)
 
-#define ql_foreach(a_var, a_head, a_field)				\
+#define	ql_foreach(a_var, a_head, a_field)				\
 	qr_foreach((a_var), ql_first(a_head), a_field)
 
-#define ql_reverse_foreach(a_var, a_head, a_field)			\
+#define	ql_reverse_foreach(a_var, a_head, a_field)			\
 	qr_reverse_foreach((a_var), ql_first(a_head), a_field)

include/jemalloc/internal/qr.h

@@ -1,28 +1,28 @@
 /* Ring definitions. */
-#define qr(a_type)							\
+#define	qr(a_type)							\
 struct {								\
 	a_type	*qre_next;						\
 	a_type	*qre_prev;						\
 }
 
 /* Ring functions. */
-#define qr_new(a_qr, a_field) do {					\
+#define	qr_new(a_qr, a_field) do {					\
 	(a_qr)->a_field.qre_next = (a_qr);				\
 	(a_qr)->a_field.qre_prev = (a_qr);				\
 } while (0)
 
-#define qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next)
+#define	qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next)
 
-#define qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev)
+#define	qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev)
 
-#define qr_before_insert(a_qrelm, a_qr, a_field) do {			\
+#define	qr_before_insert(a_qrelm, a_qr, a_field) do {			\
 	(a_qr)->a_field.qre_prev = (a_qrelm)->a_field.qre_prev;		\
 	(a_qr)->a_field.qre_next = (a_qrelm);				\
 	(a_qr)->a_field.qre_prev->a_field.qre_next = (a_qr);		\
 	(a_qrelm)->a_field.qre_prev = (a_qr);				\
 } while (0)
 
-#define qr_after_insert(a_qrelm, a_qr, a_field)				\
+#define	qr_after_insert(a_qrelm, a_qr, a_field)				\
     do									\
     {									\
 	(a_qr)->a_field.qre_next = (a_qrelm)->a_field.qre_next;		\
@@ -31,7 +31,7 @@ struct {								\
 	(a_qrelm)->a_field.qre_next = (a_qr);				\
     } while (0)
 
-#define qr_meld(a_qr_a, a_qr_b, a_field) do {				\
+#define	qr_meld(a_qr_a, a_qr_b, a_field) do {				\
 	void *t;							\
 	(a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_b);	\
 	(a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_a);	\
@@ -42,10 +42,10 @@ struct {								\
 
 /* qr_meld() and qr_split() are functionally equivalent, so there's no need to
  * have two copies of the code. */
-#define qr_split(a_qr_a, a_qr_b, a_field)				\
+#define	qr_split(a_qr_a, a_qr_b, a_field)				\
 	qr_meld((a_qr_a), (a_qr_b), a_field)
 
-#define qr_remove(a_qr, a_field) do {					\
+#define	qr_remove(a_qr, a_field) do {					\
 	(a_qr)->a_field.qre_prev->a_field.qre_next			\
 	    = (a_qr)->a_field.qre_next;					\
 	(a_qr)->a_field.qre_next->a_field.qre_prev			\
@@ -54,13 +54,13 @@ struct {								\
 	(a_qr)->a_field.qre_prev = (a_qr);				\
 } while (0)
 
-#define qr_foreach(var, a_qr, a_field)					\
+#define	qr_foreach(var, a_qr, a_field)					\
 	for ((var) = (a_qr);						\
 	    (var) != NULL;						\
 	    (var) = (((var)->a_field.qre_next != (a_qr))		\
 	    ? (var)->a_field.qre_next : NULL))
 
-#define qr_reverse_foreach(var, a_qr, a_field)				\
+#define	qr_reverse_foreach(var, a_qr, a_field)				\
 	for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL;	\
 	    (var) != NULL;						\
 	    (var) = (((var) != (a_qr))					\

include/jemalloc/internal/rb.h

@@ -22,10 +22,6 @@
 #ifndef RB_H_
 #define	RB_H_
 
-#if 0
-__FBSDID("$FreeBSD: head/lib/libc/stdlib/rb.h 204493 2010-02-28 22:57:13Z jasone $");
-#endif
-
 #ifdef RB_COMPACT
 /* Node structure. */
 #define	rb_node(a_type)							\

include/jemalloc/internal/rtree.h

@@ -14,17 +14,18 @@ typedef struct rtree_s rtree_t;
  * Size of each radix tree node (must be a power of 2).  This impacts tree
  * depth.
  */
-#if (LG_SIZEOF_PTR == 2)
-#  define RTREE_NODESIZE (1U << 14)
-#else
-#  define RTREE_NODESIZE CACHELINE
-#endif
+#define	RTREE_NODESIZE (1U << 16)
+
+typedef void *(rtree_alloc_t)(size_t);
+typedef void (rtree_dalloc_t)(void *);
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
 struct rtree_s {
+	rtree_alloc_t	*alloc;
+	rtree_dalloc_t	*dalloc;
 	malloc_mutex_t	mutex;
 	void		**root;
 	unsigned	height;
@@ -35,7 +36,8 @@ struct rtree_s {
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-rtree_t	*rtree_new(unsigned bits);
+rtree_t	*rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc);
+void	rtree_delete(rtree_t *rtree);
 void	rtree_prefork(rtree_t *rtree);
 void	rtree_postfork_parent(rtree_t *rtree);
 void	rtree_postfork_child(rtree_t *rtree);
@@ -45,20 +47,20 @@ void	rtree_postfork_child(rtree_t *rtree);
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
-#ifndef JEMALLOC_DEBUG
-void	*rtree_get_locked(rtree_t *rtree, uintptr_t key);
+#ifdef JEMALLOC_DEBUG
+uint8_t rtree_get_locked(rtree_t *rtree, uintptr_t key);
 #endif
-void	*rtree_get(rtree_t *rtree, uintptr_t key);
-bool	rtree_set(rtree_t *rtree, uintptr_t key, void *val);
+uint8_t	rtree_get(rtree_t *rtree, uintptr_t key);
+bool	rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_))
 #define	RTREE_GET_GENERATE(f)						\
 /* The least significant bits of the key are ignored. */		\
-JEMALLOC_INLINE void *							\
+JEMALLOC_INLINE uint8_t							\
 f(rtree_t *rtree, uintptr_t key)					\
 {									\
-	void *ret;							\
+	uint8_t ret;							\
 	uintptr_t subkey;						\
 	unsigned i, lshift, height, bits;				\
 	void **node, **child;						\
@@ -68,12 +70,12 @@ f(rtree_t *rtree, uintptr_t key)					\
 	    i < height - 1;						\
 	    i++, lshift += bits, node = child) {			\
 		bits = rtree->level2bits[i];				\
-		subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \
+		subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR +	\
 		    3)) - bits);					\
 		child = (void**)node[subkey];				\
 		if (child == NULL) {					\
 			RTREE_UNLOCK(&rtree->mutex);			\
-			return (NULL);					\
+			return (0);					\
 		}							\
 	}								\
 									\
@@ -84,7 +86,10 @@ f(rtree_t *rtree, uintptr_t key)					\
 	bits = rtree->level2bits[i];					\
 	subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) -	\
 	    bits);							\
-	ret = node[subkey];						\
+	{								\
+		uint8_t *leaf = (uint8_t *)node;			\
+		ret = leaf[subkey];					\
+	}								\
 	RTREE_UNLOCK(&rtree->mutex);					\
 									\
 	RTREE_GET_VALIDATE						\
@@ -123,7 +128,7 @@ RTREE_GET_GENERATE(rtree_get)
 #undef RTREE_GET_VALIDATE
 
 JEMALLOC_INLINE bool
-rtree_set(rtree_t *rtree, uintptr_t key, void *val)
+rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val)
 {
 	uintptr_t subkey;
 	unsigned i, lshift, height, bits;
@@ -138,14 +143,14 @@ rtree_set(rtree_t *rtree, uintptr_t key, void *val)
 		    bits);
 		child = (void**)node[subkey];
 		if (child == NULL) {
-			child = (void**)base_alloc(sizeof(void *) <<
-			    rtree->level2bits[i+1]);
+			size_t size = ((i + 1 < height - 1) ? sizeof(void *)
+			    : (sizeof(uint8_t))) << rtree->level2bits[i+1];
+			child = (void**)rtree->alloc(size);
 			if (child == NULL) {
 				malloc_mutex_unlock(&rtree->mutex);
 				return (true);
 			}
-			memset(child, 0, sizeof(void *) <<
-			    rtree->level2bits[i+1]);
+			memset(child, 0, size);
 			node[subkey] = child;
 		}
 	}
@@ -153,7 +158,10 @@ rtree_set(rtree_t *rtree, uintptr_t key, void *val)
 	/* node is a leaf, so it contains values rather than node pointers. */
 	bits = rtree->level2bits[i];
 	subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - bits);
-	node[subkey] = val;
+	{
+		uint8_t *leaf = (uint8_t *)node;
+		leaf[subkey] = val;
+	}
 	malloc_mutex_unlock(&rtree->mutex);
 
 	return (false);

include/jemalloc/internal/tcache.h

@@ -297,6 +297,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 	binind = SMALL_SIZE2BIN(size);
 	assert(binind < NBINS);
 	tbin = &tcache->tbins[binind];
+	size = arena_bin_info[binind].reg_size;
 	ret = tcache_alloc_easy(tbin);
 	if (ret == NULL) {
 		ret = tcache_alloc_small_hard(tcache, tbin, binind);

include/jemalloc/internal/tsd.h

@@ -6,6 +6,12 @@
 
 typedef bool (*malloc_tsd_cleanup_t)(void);
 
+#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
+    !defined(_WIN32))
+typedef struct tsd_init_block_s tsd_init_block_t;
+typedef struct tsd_init_head_s tsd_init_head_t;
+#endif
+
 /*
  * TLS/TSD-agnostic macro-based implementation of thread-specific data.  There
  * are four macros that support (at least) three use cases: file-private,
@@ -75,12 +81,13 @@ extern __thread a_type	a_name##_tls;					\
 extern pthread_key_t	a_name##_tsd;					\
 extern bool		a_name##_booted;
 #elif (defined(_WIN32))
-#define malloc_tsd_externs(a_name, a_type)				\
+#define	malloc_tsd_externs(a_name, a_type)				\
 extern DWORD		a_name##_tsd;					\
 extern bool		a_name##_booted;
 #else
 #define	malloc_tsd_externs(a_name, a_type)				\
 extern pthread_key_t	a_name##_tsd;					\
+extern tsd_init_head_t	a_name##_tsd_init_head;				\
 extern bool		a_name##_booted;
 #endif
 
@@ -105,6 +112,10 @@ a_attr bool		a_name##_booted = false;
 #else
 #define	malloc_tsd_data(a_attr, a_name, a_type, a_initializer)		\
 a_attr pthread_key_t	a_name##_tsd;					\
+a_attr tsd_init_head_t	a_name##_tsd_init_head = {			\
+	ql_head_initializer(blocks),					\
+	MALLOC_MUTEX_INITIALIZER					\
+};									\
 a_attr bool		a_name##_booted = false;
 #endif
 
@@ -333,8 +344,14 @@ a_name##_tsd_get_wrapper(void)						\
 	    pthread_getspecific(a_name##_tsd);				\
 									\
 	if (wrapper == NULL) {						\
+		tsd_init_block_t block;					\
+		wrapper = tsd_init_check_recursion(			\
+		    &a_name##_tsd_init_head, &block);			\
+		if (wrapper)						\
+		    return (wrapper);					\
 		wrapper = (a_name##_tsd_wrapper_t *)			\
 		    malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t));	\
+		block.data = wrapper;					\
 		if (wrapper == NULL) {					\
 			malloc_write("<jemalloc>: Error allocating"	\
 			    " TSD for "#a_name"\n");			\
@@ -350,6 +367,7 @@ a_name##_tsd_get_wrapper(void)						\
 			    " TSD for "#a_name"\n");			\
 			abort();					\
 		}							\
+		tsd_init_finish(&a_name##_tsd_init_head, &block);	\
 	}								\
 	return (wrapper);						\
 }									\
@@ -379,6 +397,19 @@ a_name##_tsd_set(a_type *val)						\
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
+#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
+    !defined(_WIN32))
+struct tsd_init_block_s {
+	ql_elm(tsd_init_block_t)	link;
+	pthread_t			thread;
+	void				*data;
+};
+struct tsd_init_head_s {
+	ql_head(tsd_init_block_t)	blocks;
+	malloc_mutex_t			lock;
+};
+#endif
+
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
@@ -388,6 +419,12 @@ void	malloc_tsd_dalloc(void *wrapper);
 void	malloc_tsd_no_cleanup(void *);
 void	malloc_tsd_cleanup_register(bool (*f)(void));
 void	malloc_tsd_boot(void);
+#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
+    !defined(_WIN32))
+void	*tsd_init_check_recursion(tsd_init_head_t *head,
+    tsd_init_block_t *block);
+void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
+#endif
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/util.h

@@ -14,7 +14,7 @@
  * Wrap a cpp argument that contains commas such that it isn't broken up into
  * multiple arguments.
  */
-#define JEMALLOC_CONCAT(...) __VA_ARGS__
+#define	JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
 
 /*
  * Silence compiler warnings due to uninitialized values.  This is used
@@ -42,12 +42,6 @@
 } while (0)
 #endif
 
-/* Use to assert a particular configuration, e.g., cassert(config_debug). */
-#define	cassert(c) do {							\
-	if ((c) == false)						\
-		assert(false);						\
-} while (0)
-
 #ifndef not_reached
 #define	not_reached() do {						\
 	if (config_debug) {						\
@@ -69,10 +63,18 @@
 } while (0)
 #endif
 
+#ifndef assert_not_implemented
 #define	assert_not_implemented(e) do {					\
 	if (config_debug && !(e))					\
 		not_implemented();					\
 } while (0)
+#endif
+
+/* Use to assert a particular configuration, e.g., cassert(config_debug). */
+#define	cassert(c) do {							\
+	if ((c) == false)						\
+		not_reached();						\
+} while (0)
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
@@ -82,8 +84,9 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-int	buferror(char *buf, size_t buflen);
-uintmax_t	malloc_strtoumax(const char *nptr, char **endptr, int base);
+int	buferror(int err, char *buf, size_t buflen);
+uintmax_t	malloc_strtoumax(const char *restrict nptr,
+    char **restrict endptr, int base);
 void	malloc_write(const char *s);
 
 /*
@@ -107,7 +110,6 @@ void	malloc_printf(const char *format, ...)
 
 #ifndef JEMALLOC_ENABLE_INLINE
 size_t	pow2_ceil(size_t x);
-void	malloc_write(const char *s);
 void	set_errno(int errnum);
 int	get_errno(void);
 #endif

include/jemalloc/jemalloc.h.in

@@ -1,157 +0,0 @@
-#ifndef JEMALLOC_H_
-#define	JEMALLOC_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <limits.h>
-#include <strings.h>
-
-#define	JEMALLOC_VERSION "@jemalloc_version@"
-#define	JEMALLOC_VERSION_MAJOR @jemalloc_version_major@
-#define	JEMALLOC_VERSION_MINOR @jemalloc_version_minor@
-#define	JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@
-#define	JEMALLOC_VERSION_NREV @jemalloc_version_nrev@
-#define	JEMALLOC_VERSION_GID "@jemalloc_version_gid@"
-
-#include "jemalloc_defs@install_suffix@.h"
-
-#ifdef JEMALLOC_EXPERIMENTAL
-#define	ALLOCM_LG_ALIGN(la)	(la)
-#if LG_SIZEOF_PTR == 2
-#define	ALLOCM_ALIGN(a)	(ffs(a)-1)
-#else
-#define	ALLOCM_ALIGN(a)	((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31)
-#endif
-#define	ALLOCM_ZERO	((int)0x40)
-#define	ALLOCM_NO_MOVE	((int)0x80)
-/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */
-#define	ALLOCM_ARENA(a)	((int)(((a)+1) << 8))
-
-#define	ALLOCM_SUCCESS		0
-#define	ALLOCM_ERR_OOM		1
-#define	ALLOCM_ERR_NOT_MOVED	2
-#endif
-
-/*
- * The je_ prefix on the following public symbol declarations is an artifact of
- * namespace management, and should be omitted in application code unless
- * JEMALLOC_NO_DEMANGLE is defined (see below).
- */
-extern JEMALLOC_EXPORT const char	*je_malloc_conf;
-extern JEMALLOC_EXPORT void		(*je_malloc_message)(void *cbopaque,
-    const char *s);
-
-JEMALLOC_EXPORT void	*je_malloc(size_t size) JEMALLOC_ATTR(malloc);
-JEMALLOC_EXPORT void	*je_calloc(size_t num, size_t size)
-    JEMALLOC_ATTR(malloc);
-JEMALLOC_EXPORT int	je_posix_memalign(void **memptr, size_t alignment,
-    size_t size) JEMALLOC_ATTR(nonnull(1));
-JEMALLOC_EXPORT void	*je_aligned_alloc(size_t alignment, size_t size)
-    JEMALLOC_ATTR(malloc);
-JEMALLOC_EXPORT void	*je_realloc(void *ptr, size_t size);
-JEMALLOC_EXPORT void	je_free(void *ptr);
-
-#ifdef JEMALLOC_OVERRIDE_MEMALIGN
-JEMALLOC_EXPORT void *	je_memalign(size_t alignment, size_t size)
-    JEMALLOC_ATTR(malloc);
-#endif
-
-#ifdef JEMALLOC_OVERRIDE_VALLOC
-JEMALLOC_EXPORT void *	je_valloc(size_t size) JEMALLOC_ATTR(malloc);
-#endif
-
-JEMALLOC_EXPORT size_t	je_malloc_usable_size(
-    JEMALLOC_USABLE_SIZE_CONST void *ptr);
-JEMALLOC_EXPORT void	je_malloc_stats_print(void (*write_cb)(void *,
-    const char *), void *je_cbopaque, const char *opts);
-JEMALLOC_EXPORT int	je_mallctl(const char *name, void *oldp,
-    size_t *oldlenp, void *newp, size_t newlen);
-JEMALLOC_EXPORT int	je_mallctlnametomib(const char *name, size_t *mibp,
-    size_t *miblenp);
-JEMALLOC_EXPORT int	je_mallctlbymib(const size_t *mib, size_t miblen,
-    void *oldp, size_t *oldlenp, void *newp, size_t newlen);
-
-#ifdef JEMALLOC_EXPERIMENTAL
-JEMALLOC_EXPORT int	je_allocm(void **ptr, size_t *rsize, size_t size,
-    int flags) JEMALLOC_ATTR(nonnull(1));
-JEMALLOC_EXPORT int	je_rallocm(void **ptr, size_t *rsize, size_t size,
-    size_t extra, int flags) JEMALLOC_ATTR(nonnull(1));
-JEMALLOC_EXPORT int	je_sallocm(const void *ptr, size_t *rsize, int flags)
-    JEMALLOC_ATTR(nonnull(1));
-JEMALLOC_EXPORT int	je_dallocm(void *ptr, int flags)
-    JEMALLOC_ATTR(nonnull(1));
-JEMALLOC_EXPORT int	je_nallocm(size_t *rsize, size_t size, int flags);
-#endif
-
-/*
- * By default application code must explicitly refer to mangled symbol names,
- * so that it is possible to use jemalloc in conjunction with another allocator
- * in the same application.  Define JEMALLOC_MANGLE in order to cause automatic
- * name mangling that matches the API prefixing that happened as a result of
- * --with-mangling and/or --with-jemalloc-prefix configuration settings.
- */
-#ifdef JEMALLOC_MANGLE
-#ifndef JEMALLOC_NO_DEMANGLE
-#define	JEMALLOC_NO_DEMANGLE
-#endif
-#define	malloc_conf je_malloc_conf
-#define	malloc_message je_malloc_message
-#define	malloc je_malloc
-#define	calloc je_calloc
-#define	posix_memalign je_posix_memalign
-#define	aligned_alloc je_aligned_alloc
-#define	realloc je_realloc
-#define	free je_free
-#define	malloc_usable_size je_malloc_usable_size
-#define	malloc_stats_print je_malloc_stats_print
-#define	mallctl je_mallctl
-#define	mallctlnametomib je_mallctlnametomib
-#define	mallctlbymib je_mallctlbymib
-#define	memalign je_memalign
-#define	valloc je_valloc
-#ifdef JEMALLOC_EXPERIMENTAL
-#define	allocm je_allocm
-#define	rallocm je_rallocm
-#define	sallocm je_sallocm
-#define	dallocm je_dallocm
-#define	nallocm je_nallocm
-#endif
-#endif
-
-/*
- * The je_* macros can be used as stable alternative names for the public
- * jemalloc API if JEMALLOC_NO_DEMANGLE is defined.  This is primarily meant
- * for use in jemalloc itself, but it can be used by application code to
- * provide isolation from the name mangling specified via --with-mangling
- * and/or --with-jemalloc-prefix.
- */
-#ifndef JEMALLOC_NO_DEMANGLE
-#undef je_malloc_conf
-#undef je_malloc_message
-#undef je_malloc
-#undef je_calloc
-#undef je_posix_memalign
-#undef je_aligned_alloc
-#undef je_realloc
-#undef je_free
-#undef je_malloc_usable_size
-#undef je_malloc_stats_print
-#undef je_mallctl
-#undef je_mallctlnametomib
-#undef je_mallctlbymib
-#undef je_memalign
-#undef je_valloc
-#ifdef JEMALLOC_EXPERIMENTAL
-#undef je_allocm
-#undef je_rallocm
-#undef je_sallocm
-#undef je_dallocm
-#undef je_nallocm
-#endif
-#endif
-
-#ifdef __cplusplus
-};
-#endif
-#endif /* JEMALLOC_H_ */

include/jemalloc/jemalloc.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+objroot=$1
+
+cat <<EOF
+#ifndef JEMALLOC_H_
+#define	JEMALLOC_H_
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+EOF
+
+for hdr in jemalloc_defs.h jemalloc_rename.h jemalloc_macros.h \
+           jemalloc_protos.h jemalloc_mangle.h ; do
+  cat "${objroot}include/jemalloc/${hdr}" \
+      | grep -v 'Generated from .* by configure\.' \
+      | sed -e 's/^#define /#define	/g' \
+      | sed -e 's/ $//g'
+  echo
+done
+
+cat <<EOF
+#ifdef __cplusplus
+};
+#endif
+#endif /* JEMALLOC_H_ */
+EOF

include/jemalloc/jemalloc_defs.h.in

@@ -1,222 +1,12 @@
-/*
- * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all
- * public APIs to be prefixed.  This makes it possible, with some care, to use
- * multiple allocators simultaneously.
- */
-#undef JEMALLOC_PREFIX
-#undef JEMALLOC_CPREFIX
-
-/*
- * Name mangling for public symbols is controlled by --with-mangling and
- * --with-jemalloc-prefix.  With default settings the je_ prefix is stripped by
- * these macro definitions.
- */
-#undef je_malloc_conf
-#undef je_malloc_message
-#undef je_malloc
-#undef je_calloc
-#undef je_posix_memalign
-#undef je_aligned_alloc
-#undef je_realloc
-#undef je_free
-#undef je_malloc_usable_size
-#undef je_malloc_stats_print
-#undef je_mallctl
-#undef je_mallctlnametomib
-#undef je_mallctlbymib
-#undef je_memalign
-#undef je_valloc
-#undef je_allocm
-#undef je_rallocm
-#undef je_sallocm
-#undef je_dallocm
-#undef je_nallocm
-
-/*
- * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs.
- * For shared libraries, symbol visibility mechanisms prevent these symbols
- * from being exported, but for static libraries, naming collisions are a real
- * possibility.
- */
-#undef JEMALLOC_PRIVATE_NAMESPACE
-#undef JEMALLOC_N
-
-/*
- * Hyper-threaded CPUs may need a special instruction inside spin loops in
- * order to yield to another virtual CPU.
- */
-#undef CPU_SPINWAIT
-
-/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */
-#undef JEMALLOC_ATOMIC9
-
-/*
- * Defined if OSAtomic*() functions are available, as provided by Darwin, and
- * documented in the atomic(3) manual page.
- */
-#undef JEMALLOC_OSATOMIC
-
-/*
- * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
- * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
- * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
- * functions are defined in libgcc instead of being inlines)
- */
-#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4
-
-/*
- * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
- * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
- * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
- * functions are defined in libgcc instead of being inlines)
- */
-#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8
-
-/*
- * Defined if OSSpin*() functions are available, as provided by Darwin, and
- * documented in the spinlock(3) manual page.
- */
-#undef JEMALLOC_OSSPIN
-
-/*
- * Defined if _malloc_thread_cleanup() exists.  At least in the case of
- * FreeBSD, pthread_key_create() allocates, which if used during malloc
- * bootstrapping will cause recursion into the pthreads library.  Therefore, if
- * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in
- * malloc_tsd.
- */
-#undef JEMALLOC_MALLOC_THREAD_CLEANUP
-
-/*
- * Defined if threaded initialization is known to be safe on this platform.
- * Among other things, it must be possible to initialize a mutex without
- * triggering allocation in order for threaded allocation to be safe.
- */
-#undef JEMALLOC_THREADED_INIT
-
-/*
- * Defined if the pthreads implementation defines
- * _pthread_mutex_init_calloc_cb(), in which case the function is used in order
- * to avoid recursive allocation during mutex initialization.
- */
-#undef JEMALLOC_MUTEX_INIT_CB
-
 /* Defined if __attribute__((...)) syntax is supported. */
 #undef JEMALLOC_HAVE_ATTR
-#ifdef JEMALLOC_HAVE_ATTR
-#  define JEMALLOC_ATTR(s) __attribute__((s))
-#  define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
-#  define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s))
-#  define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s))
-#  define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline)
-#elif _MSC_VER
-#  define JEMALLOC_ATTR(s)
-#  ifdef DLLEXPORT
-#    define JEMALLOC_EXPORT __declspec(dllexport)
-#  else
-#    define JEMALLOC_EXPORT __declspec(dllimport)
-#  endif
-#  define JEMALLOC_ALIGNED(s) __declspec(align(s))
-#  define JEMALLOC_SECTION(s) __declspec(allocate(s))
-#  define JEMALLOC_NOINLINE __declspec(noinline)
-#else
-#  define JEMALLOC_ATTR(s)
-#  define JEMALLOC_EXPORT
-#  define JEMALLOC_ALIGNED(s)
-#  define JEMALLOC_SECTION(s)
-#  define JEMALLOC_NOINLINE
-#endif
-
-/* Defined if sbrk() is supported. */
-#undef JEMALLOC_HAVE_SBRK
-
-/* Non-empty if the tls_model attribute is supported. */
-#undef JEMALLOC_TLS_MODEL
-
-/* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */
-#undef JEMALLOC_CC_SILENCE
-
-/*
- * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
- * inline functions.
- */
-#undef JEMALLOC_DEBUG
-
-/* JEMALLOC_STATS enables statistics calculation. */
-#undef JEMALLOC_STATS
-
-/* JEMALLOC_PROF enables allocation profiling. */
-#undef JEMALLOC_PROF
-
-/* Use libunwind for profile backtracing if defined. */
-#undef JEMALLOC_PROF_LIBUNWIND
-
-/* Use libgcc for profile backtracing if defined. */
-#undef JEMALLOC_PROF_LIBGCC
-
-/* Use gcc intrinsics for profile backtracing if defined. */
-#undef JEMALLOC_PROF_GCC
-
-/*
- * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects.
- * This makes it possible to allocate/deallocate objects without any locking
- * when the cache is in the steady state.
- */
-#undef JEMALLOC_TCACHE
-
-/*
- * JEMALLOC_DSS enables use of sbrk(2) to allocate chunks from the data storage
- * segment (DSS).
- */
-#undef JEMALLOC_DSS
-
-/* Support memory filling (junk/zero/quarantine/redzone). */
-#undef JEMALLOC_FILL
 
 /* Support the experimental API. */
 #undef JEMALLOC_EXPERIMENTAL
 
-/* Support utrace(2)-based tracing. */
-#undef JEMALLOC_UTRACE
-
-/* Support Valgrind. */
-#undef JEMALLOC_VALGRIND
-
-/* Support optional abort() on OOM. */
-#undef JEMALLOC_XMALLOC
-
-/* Support lazy locking (avoid locking unless a second thread is launched). */
-#undef JEMALLOC_LAZY_LOCK
-
-/* One page is 2^STATIC_PAGE_SHIFT bytes. */
-#undef STATIC_PAGE_SHIFT
-
 /*
- * If defined, use munmap() to unmap freed chunks, rather than storing them for
- * later reuse.  This is disabled by default on Linux because common sequences
- * of mmap()/munmap() calls will cause virtual memory map holes.
- */
-#undef JEMALLOC_MUNMAP
-
-/*
- * If defined, use mremap(...MREMAP_FIXED...) for huge realloc().  This is
- * disabled by default because it is Linux-specific and it will cause virtual
- * memory map holes, much like munmap(2) does.
- */
-#undef JEMALLOC_MREMAP
-
-/* TLS is used to map arenas and magazine caches to threads. */
-#undef JEMALLOC_TLS
-
-/*
- * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside
- * within jemalloc-owned chunks before dereferencing them.
- */
-#undef JEMALLOC_IVSALLOC
-
-/*
- * Define overrides for non-standard allocator-related functions if they
- * are present on the system.
+ * Define overrides for non-standard allocator-related functions if they are
+ * present on the system.
  */
 #undef JEMALLOC_OVERRIDE_MEMALIGN
 #undef JEMALLOC_OVERRIDE_VALLOC
@@ -230,38 +20,5 @@
  */
 #undef JEMALLOC_USABLE_SIZE_CONST
 
-/*
- * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
- */
-#undef JEMALLOC_ZONE
-#undef JEMALLOC_ZONE_VERSION
-
-/*
- * Methods for purging unused pages differ between operating systems.
- *
- *   madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages,
- *                                 such that new pages will be demand-zeroed if
- *                                 the address region is later touched.
- *   madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being
- *                             unused, such that they will be discarded rather
- *                             than swapped out.
- */
-#undef JEMALLOC_PURGE_MADVISE_DONTNEED
-#undef JEMALLOC_PURGE_MADVISE_FREE
-
-/*
- * Define if operating system has alloca.h header.
- */
-#undef JEMALLOC_HAS_ALLOCA_H
-
 /* sizeof(void *) == 2^LG_SIZEOF_PTR. */
 #undef LG_SIZEOF_PTR
-
-/* sizeof(int) == 2^LG_SIZEOF_INT. */
-#undef LG_SIZEOF_INT
-
-/* sizeof(long) == 2^LG_SIZEOF_LONG. */
-#undef LG_SIZEOF_LONG
-
-/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */
-#undef LG_SIZEOF_INTMAX_T

include/jemalloc/jemalloc_macros.h.in

@@ -0,0 +1,61 @@
+#include <limits.h>
+#include <strings.h>
+
+#define	JEMALLOC_VERSION "@jemalloc_version@"
+#define	JEMALLOC_VERSION_MAJOR @jemalloc_version_major@
+#define	JEMALLOC_VERSION_MINOR @jemalloc_version_minor@
+#define	JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@
+#define	JEMALLOC_VERSION_NREV @jemalloc_version_nrev@
+#define	JEMALLOC_VERSION_GID "@jemalloc_version_gid@"
+
+#  define MALLOCX_LG_ALIGN(la)	(la)
+#  if LG_SIZEOF_PTR == 2
+#    define MALLOCX_ALIGN(a)	(ffs(a)-1)
+#  else
+#    define MALLOCX_ALIGN(a)						\
+	 ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31)
+#  endif
+#  define MALLOCX_ZERO	((int)0x40)
+/* Bias arena index bits so that 0 encodes "MALLOCX_ARENA() unspecified". */
+#  define MALLOCX_ARENA(a)	((int)(((a)+1) << 8))
+
+#ifdef JEMALLOC_EXPERIMENTAL
+#  define ALLOCM_LG_ALIGN(la)	(la)
+#  if LG_SIZEOF_PTR == 2
+#    define ALLOCM_ALIGN(a)	(ffs(a)-1)
+#  else
+#    define ALLOCM_ALIGN(a)						\
+	 ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31)
+#  endif
+#  define ALLOCM_ZERO	((int)0x40)
+#  define ALLOCM_NO_MOVE	((int)0x80)
+/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */
+#  define ALLOCM_ARENA(a)	((int)(((a)+1) << 8))
+#  define ALLOCM_SUCCESS	0
+#  define ALLOCM_ERR_OOM	1
+#  define ALLOCM_ERR_NOT_MOVED	2
+#endif
+
+#ifdef JEMALLOC_HAVE_ATTR
+#  define JEMALLOC_ATTR(s) __attribute__((s))
+#  define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
+#  define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s))
+#  define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s))
+#  define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline)
+#elif _MSC_VER
+#  define JEMALLOC_ATTR(s)
+#  ifdef DLLEXPORT
+#    define JEMALLOC_EXPORT __declspec(dllexport)
+#  else
+#    define JEMALLOC_EXPORT __declspec(dllimport)
+#  endif
+#  define JEMALLOC_ALIGNED(s) __declspec(align(s))
+#  define JEMALLOC_SECTION(s) __declspec(allocate(s))
+#  define JEMALLOC_NOINLINE __declspec(noinline)
+#else
+#  define JEMALLOC_ATTR(s)
+#  define JEMALLOC_EXPORT
+#  define JEMALLOC_ALIGNED(s)
+#  define JEMALLOC_SECTION(s)
+#  define JEMALLOC_NOINLINE
+#endif

include/jemalloc/jemalloc_mangle.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+public_symbols_txt=$1
+symbol_prefix=$2
+
+cat <<EOF
+/*
+ * By default application code must explicitly refer to mangled symbol names,
+ * so that it is possible to use jemalloc in conjunction with another allocator
+ * in the same application.  Define JEMALLOC_MANGLE in order to cause automatic
+ * name mangling that matches the API prefixing that happened as a result of
+ * --with-mangling and/or --with-jemalloc-prefix configuration settings.
+ */
+#ifdef JEMALLOC_MANGLE
+#  ifndef JEMALLOC_NO_DEMANGLE
+#    define JEMALLOC_NO_DEMANGLE
+#  endif
+EOF
+
+for nm in `cat ${public_symbols_txt}` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  echo "#  define ${n} ${symbol_prefix}${n}"
+done
+
+cat <<EOF
+#endif
+
+/*
+ * The ${symbol_prefix}* macros can be used as stable alternative names for the
+ * public jemalloc API if JEMALLOC_NO_DEMANGLE is defined.  This is primarily
+ * meant for use in jemalloc itself, but it can be used by application code to
+ * provide isolation from the name mangling specified via --with-mangling
+ * and/or --with-jemalloc-prefix.
+ */
+#ifndef JEMALLOC_NO_DEMANGLE
+EOF
+
+for nm in `cat ${public_symbols_txt}` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  echo "#  undef ${symbol_prefix}${n}"
+done
+
+cat <<EOF
+#endif
+EOF

include/jemalloc/jemalloc_protos.h.in

@@ -0,0 +1,58 @@
+/*
+ * The @je_@ prefix on the following public symbol declarations is an artifact
+ * of namespace management, and should be omitted in application code unless
+ * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle@install_suffix@.h).
+ */
+extern JEMALLOC_EXPORT const char	*@je_@malloc_conf;
+extern JEMALLOC_EXPORT void		(*@je_@malloc_message)(void *cbopaque,
+    const char *s);
+
+JEMALLOC_EXPORT void	*@je_@malloc(size_t size) JEMALLOC_ATTR(malloc);
+JEMALLOC_EXPORT void	*@je_@calloc(size_t num, size_t size)
+    JEMALLOC_ATTR(malloc);
+JEMALLOC_EXPORT int	@je_@posix_memalign(void **memptr, size_t alignment,
+    size_t size) JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT void	*@je_@aligned_alloc(size_t alignment, size_t size)
+    JEMALLOC_ATTR(malloc);
+JEMALLOC_EXPORT void	*@je_@realloc(void *ptr, size_t size);
+JEMALLOC_EXPORT void	@je_@free(void *ptr);
+
+JEMALLOC_EXPORT void	*@je_@mallocx(size_t size, int flags);
+JEMALLOC_EXPORT void	*@je_@rallocx(void *ptr, size_t size, int flags);
+JEMALLOC_EXPORT size_t	@je_@xallocx(void *ptr, size_t size, size_t extra,
+    int flags);
+JEMALLOC_EXPORT size_t	@je_@sallocx(const void *ptr, int flags);
+JEMALLOC_EXPORT void	@je_@dallocx(void *ptr, int flags);
+JEMALLOC_EXPORT size_t	@je_@nallocx(size_t size, int flags);
+
+JEMALLOC_EXPORT int	@je_@mallctl(const char *name, void *oldp,
+    size_t *oldlenp, void *newp, size_t newlen);
+JEMALLOC_EXPORT int	@je_@mallctlnametomib(const char *name, size_t *mibp,
+    size_t *miblenp);
+JEMALLOC_EXPORT int	@je_@mallctlbymib(const size_t *mib, size_t miblen,
+    void *oldp, size_t *oldlenp, void *newp, size_t newlen);
+JEMALLOC_EXPORT void	@je_@malloc_stats_print(void (*write_cb)(void *,
+    const char *), void *@je_@cbopaque, const char *opts);
+JEMALLOC_EXPORT size_t	@je_@malloc_usable_size(
+    JEMALLOC_USABLE_SIZE_CONST void *ptr);
+
+#ifdef JEMALLOC_OVERRIDE_MEMALIGN
+JEMALLOC_EXPORT void *	@je_@memalign(size_t alignment, size_t size)
+    JEMALLOC_ATTR(malloc);
+#endif
+
+#ifdef JEMALLOC_OVERRIDE_VALLOC
+JEMALLOC_EXPORT void *	@je_@valloc(size_t size) JEMALLOC_ATTR(malloc);
+#endif
+
+#ifdef JEMALLOC_EXPERIMENTAL
+JEMALLOC_EXPORT int	@je_@allocm(void **ptr, size_t *rsize, size_t size,
+    int flags) JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT int	@je_@rallocm(void **ptr, size_t *rsize, size_t size,
+    size_t extra, int flags) JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT int	@je_@sallocm(const void *ptr, size_t *rsize, int flags)
+    JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT int	@je_@dallocm(void *ptr, int flags)
+    JEMALLOC_ATTR(nonnull(1));
+JEMALLOC_EXPORT int	@je_@nallocm(size_t *rsize, size_t size, int flags);
+#endif

include/jemalloc/jemalloc_rename.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+public_symbols_txt=$1
+
+cat <<EOF
+/*
+ * Name mangling for public symbols is controlled by --with-mangling and
+ * --with-jemalloc-prefix.  With default settings the je_ prefix is stripped by
+ * these macro definitions.
+ */
+#ifndef JEMALLOC_NO_RENAME
+EOF
+
+for nm in `cat ${public_symbols_txt}` ; do
+  n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'`
+  m=`echo ${nm} |tr ':' ' ' |awk '{print $2}'`
+  echo "#  define je_${n} ${m}"
+done
+
+cat <<EOF
+#endif
+EOF

src/bitmap.c

@@ -1,4 +1,4 @@
-#define JEMALLOC_BITMAP_C_
+#define	JEMALLOC_BITMAP_C_
 #include "jemalloc/internal/jemalloc_internal.h"
 
 /******************************************************************************/

src/chunk.c

@@ -180,7 +180,7 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
 label_return:
 	if (ret != NULL) {
 		if (config_ivsalloc && base == false) {
-			if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) {
+			if (rtree_set(chunks_rtree, (uintptr_t)ret, 1)) {
 				chunk_dealloc(ret, size, true);
 				return (NULL);
 			}
@@ -321,7 +321,7 @@ chunk_dealloc(void *chunk, size_t size, bool unmap)
 	assert((size & chunksize_mask) == 0);
 
 	if (config_ivsalloc)
-		rtree_set(chunks_rtree, (uintptr_t)chunk, NULL);
+		rtree_set(chunks_rtree, (uintptr_t)chunk, 0);
 	if (config_stats || config_prof) {
 		malloc_mutex_lock(&chunks_mtx);
 		assert(stats_chunks.curchunks >= (size / chunksize));
@@ -356,7 +356,7 @@ chunk_boot(void)
 	extent_tree_ad_new(&chunks_ad_dss);
 	if (config_ivsalloc) {
 		chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) -
-		    opt_lg_chunk);
+		    opt_lg_chunk, base_alloc, NULL);
 		if (chunks_rtree == NULL)
 			return (true);
 	}
@@ -368,7 +368,7 @@ void
 chunk_prefork(void)
 {
 
-	malloc_mutex_lock(&chunks_mtx);
+	malloc_mutex_prefork(&chunks_mtx);
 	if (config_ivsalloc)
 		rtree_prefork(chunks_rtree);
 	chunk_dss_prefork();

src/chunk_dss.c

@@ -28,16 +28,17 @@ static void		*dss_max;
 
 /******************************************************************************/
 
-#ifndef JEMALLOC_HAVE_SBRK
 static void *
-sbrk(intptr_t increment)
+chunk_dss_sbrk(intptr_t increment)
 {
 
+#ifdef JEMALLOC_HAVE_SBRK
+	return (sbrk(increment));
+#else
 	not_implemented();
-
 	return (NULL);
-}
 #endif
+}
 
 dss_prec_t
 chunk_dss_prec_get(void)
@@ -93,7 +94,7 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero)
 		 */
 		do {
 			/* Get the current end of the DSS. */
-			dss_max = sbrk(0);
+			dss_max = chunk_dss_sbrk(0);
 			/*
 			 * Calculate how much padding is necessary to
 			 * chunk-align the end of the DSS.
@@ -117,7 +118,7 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero)
 				return (NULL);
 			}
 			incr = gap_size + cpad_size + size;
-			dss_prev = sbrk(incr);
+			dss_prev = chunk_dss_sbrk(incr);
 			if (dss_prev == dss_max) {
 				/* Success. */
 				dss_max = dss_next;
@@ -163,7 +164,7 @@ chunk_dss_boot(void)
 
 	if (malloc_mutex_init(&dss_mtx))
 		return (true);
-	dss_base = sbrk(0);
+	dss_base = chunk_dss_sbrk(0);
 	dss_prev = dss_base;
 	dss_max = dss_base;
 

src/chunk_mmap.c

@@ -43,7 +43,7 @@ pages_map(void *addr, size_t size)
 		if (munmap(ret, size) == -1) {
 			char buf[BUFERROR_BUF];
 
-			buferror(buf, sizeof(buf));
+			buferror(get_errno(), buf, sizeof(buf));
 			malloc_printf("<jemalloc: Error in munmap(): %s\n",
 			    buf);
 			if (opt_abort)
@@ -69,7 +69,7 @@ pages_unmap(void *addr, size_t size)
 	{
 		char buf[BUFERROR_BUF];
 
-		buferror(buf, sizeof(buf));
+		buferror(get_errno(), buf, sizeof(buf));
 		malloc_printf("<jemalloc>: Error in "
 #ifdef _WIN32
 		              "VirtualFree"

src/ckh.c

@@ -49,7 +49,7 @@ static void	ckh_shrink(ckh_t *ckh);
  * Search bucket for key and return the cell number if found; SIZE_T_MAX
  * otherwise.
  */
-JEMALLOC_INLINE size_t
+JEMALLOC_INLINE_C size_t
 ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key)
 {
 	ckhc_t *cell;
@@ -67,7 +67,7 @@ ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key)
 /*
  * Search table for key and return cell number if found; SIZE_T_MAX otherwise.
  */
-JEMALLOC_INLINE size_t
+JEMALLOC_INLINE_C size_t
 ckh_isearch(ckh_t *ckh, const void *key)
 {
 	size_t hashes[2], bucket, cell;
@@ -88,7 +88,7 @@ ckh_isearch(ckh_t *ckh, const void *key)
 	return (cell);
 }
 
-JEMALLOC_INLINE bool
+JEMALLOC_INLINE_C bool
 ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
     const void *data)
 {
@@ -120,7 +120,7 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
  * eviction/relocation procedure until either success or detection of an
  * eviction/relocation bucket cycle.
  */
-JEMALLOC_INLINE bool
+JEMALLOC_INLINE_C bool
 ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
     void const **argdata)
 {
@@ -190,7 +190,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
 	}
 }
 
-JEMALLOC_INLINE bool
+JEMALLOC_INLINE_C bool
 ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
 {
 	size_t hashes[2], bucket;
@@ -219,7 +219,7 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
  * Try to rebuild the hash table from scratch by inserting all items from the
  * old table into the new.
  */
-JEMALLOC_INLINE bool
+JEMALLOC_INLINE_C bool
 ckh_rebuild(ckh_t *ckh, ckhc_t *aTab)
 {
 	size_t count, i, nins;

src/ctl.c

@@ -929,7 +929,7 @@ void
 ctl_prefork(void)
 {
 
-	malloc_mutex_lock(&ctl_mtx);
+	malloc_mutex_prefork(&ctl_mtx);
 }
 
 void
@@ -1110,6 +1110,8 @@ label_return:								\
 	return (ret);							\
 }
 
+/******************************************************************************/
+
 CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *)
 
 static int
@@ -1131,49 +1133,52 @@ label_return:
 	return (ret);
 }
 
-static int
-thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
-    size_t *oldlenp, void *newp, size_t newlen)
-{
-	int ret;
-	bool oldval;
+/******************************************************************************/
 
-	if (config_tcache == false)
-		return (ENOENT);
+CTL_RO_BOOL_CONFIG_GEN(config_debug)
+CTL_RO_BOOL_CONFIG_GEN(config_dss)
+CTL_RO_BOOL_CONFIG_GEN(config_fill)
+CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock)
+CTL_RO_BOOL_CONFIG_GEN(config_mremap)
+CTL_RO_BOOL_CONFIG_GEN(config_munmap)
+CTL_RO_BOOL_CONFIG_GEN(config_prof)
+CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc)
+CTL_RO_BOOL_CONFIG_GEN(config_prof_libunwind)
+CTL_RO_BOOL_CONFIG_GEN(config_stats)
+CTL_RO_BOOL_CONFIG_GEN(config_tcache)
+CTL_RO_BOOL_CONFIG_GEN(config_tls)
+CTL_RO_BOOL_CONFIG_GEN(config_utrace)
+CTL_RO_BOOL_CONFIG_GEN(config_valgrind)
+CTL_RO_BOOL_CONFIG_GEN(config_xmalloc)
 
-	oldval = tcache_enabled_get();
-	if (newp != NULL) {
-		if (newlen != sizeof(bool)) {
-			ret = EINVAL;
-			goto label_return;
-		}
-		tcache_enabled_set(*(bool *)newp);
-	}
-	READ(oldval, bool);
+/******************************************************************************/
 
-	ret = 0;
-label_return:
-	return (ret);
-}
+CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
+CTL_RO_NL_GEN(opt_dss, opt_dss, const char *)
+CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t)
+CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t)
+CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t)
+CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
+CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool)
+CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t)
+CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool)
+CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
+CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
+CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool)
+CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
+CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool)
+CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
+CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
+CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
+CTL_RO_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) /* Mutable. */
+CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t)
+CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
+CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t)
+CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool)
+CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool)
+CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool)
 
-static int
-thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
-    size_t *oldlenp, void *newp, size_t newlen)
-{
-	int ret;
-
-	if (config_tcache == false)
-		return (ENOENT);
-
-	READONLY();
-	WRITEONLY();
-
-	tcache_flush();
-
-	ret = 0;
-label_return:
-	return (ret);
-}
+/******************************************************************************/
 
 static int
 thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
@@ -1235,50 +1240,49 @@ CTL_RO_NL_CGEN(config_stats, thread_deallocated,
 CTL_RO_NL_CGEN(config_stats, thread_deallocatedp,
     &thread_allocated_tsd_get()->deallocated, uint64_t *)
 
-/******************************************************************************/
+static int
+thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
+    size_t *oldlenp, void *newp, size_t newlen)
+{
+	int ret;
+	bool oldval;
 
-CTL_RO_BOOL_CONFIG_GEN(config_debug)
-CTL_RO_BOOL_CONFIG_GEN(config_dss)
-CTL_RO_BOOL_CONFIG_GEN(config_fill)
-CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock)
-CTL_RO_BOOL_CONFIG_GEN(config_mremap)
-CTL_RO_BOOL_CONFIG_GEN(config_munmap)
-CTL_RO_BOOL_CONFIG_GEN(config_prof)
-CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc)
-CTL_RO_BOOL_CONFIG_GEN(config_prof_libunwind)
-CTL_RO_BOOL_CONFIG_GEN(config_stats)
-CTL_RO_BOOL_CONFIG_GEN(config_tcache)
-CTL_RO_BOOL_CONFIG_GEN(config_tls)
-CTL_RO_BOOL_CONFIG_GEN(config_utrace)
-CTL_RO_BOOL_CONFIG_GEN(config_valgrind)
-CTL_RO_BOOL_CONFIG_GEN(config_xmalloc)
+	if (config_tcache == false)
+		return (ENOENT);
 
-/******************************************************************************/
+	oldval = tcache_enabled_get();
+	if (newp != NULL) {
+		if (newlen != sizeof(bool)) {
+			ret = EINVAL;
+			goto label_return;
+		}
+		tcache_enabled_set(*(bool *)newp);
+	}
+	READ(oldval, bool);
 
-CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
-CTL_RO_NL_GEN(opt_dss, opt_dss, const char *)
-CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t)
-CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t)
-CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t)
-CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
-CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool)
-CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
-CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t)
-CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool)
-CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
-CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool)
-CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
-CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool)
-CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
-CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
-CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
-CTL_RO_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) /* Mutable. */
-CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t)
-CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t)
-CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool)
-CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool)
-CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool)
-CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
+	ret = 0;
+label_return:
+	return (ret);
+}
+
+static int
+thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
+    size_t *oldlenp, void *newp, size_t newlen)
+{
+	int ret;
+
+	if (config_tcache == false)
+		return (ENOENT);
+
+	READONLY();
+	WRITEONLY();
+
+	tcache_flush();
+
+	ret = 0;
+label_return:
+	return (ret);
+}
 
 /******************************************************************************/
 
@@ -1390,31 +1394,8 @@ label_return:
 	return (ret);
 }
 
-
 /******************************************************************************/
 
-CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t)
-CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t)
-CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t)
-static const ctl_named_node_t *
-arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
-{
-
-	if (i > NBINS)
-		return (NULL);
-	return (super_arenas_bin_i_node);
-}
-
-CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t)
-static const ctl_named_node_t *
-arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i)
-{
-
-	if (i > nlclasses)
-		return (NULL);
-	return (super_arenas_lrun_i_node);
-}
-
 static int
 arenas_narenas_ctl(const size_t *mib, size_t miblen, void *oldp,
     size_t *oldlenp, void *newp, size_t newlen)
@@ -1468,7 +1449,28 @@ CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
 CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t)
 CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned)
 CTL_RO_NL_CGEN(config_tcache, arenas_nhbins, nhbins, unsigned)
+CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t)
+CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t)
+CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t)
+static const ctl_named_node_t *
+arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
+{
+
+	if (i > NBINS)
+		return (NULL);
+	return (super_arenas_bin_i_node);
+}
+
 CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t)
+CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t)
+static const ctl_named_node_t *
+arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i)
+{
+
+	if (i > nlclasses)
+		return (NULL);
+	return (super_arenas_lrun_i_node);
+}
 
 static int
 arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
@@ -1575,6 +1577,11 @@ CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t)
 
 /******************************************************************************/
 
+CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *)
+CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t)
+CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t)
+CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t)
+
 CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current,
     size_t)
 CTL_RO_CGEN(config_stats, stats_chunks_total, ctl_stats.chunks.total, uint64_t)
@@ -1582,6 +1589,20 @@ CTL_RO_CGEN(config_stats, stats_chunks_high, ctl_stats.chunks.high, size_t)
 CTL_RO_CGEN(config_stats, stats_huge_allocated, huge_allocated, size_t)
 CTL_RO_CGEN(config_stats, stats_huge_nmalloc, huge_nmalloc, uint64_t)
 CTL_RO_CGEN(config_stats, stats_huge_ndalloc, huge_ndalloc, uint64_t)
+
+CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *)
+CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned)
+CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t)
+CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
+    ctl_stats.arenas[mib[2]].astats.mapped, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_npurge,
+    ctl_stats.arenas[mib[2]].astats.npurge, uint64_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise,
+    ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_purged,
+    ctl_stats.arenas[mib[2]].astats.purged, uint64_t)
+
 CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
     ctl_stats.arenas[mib[2]].allocated_small, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc,
@@ -1645,19 +1666,6 @@ stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j)
 	return (super_stats_arenas_i_lruns_j_node);
 }
 
-CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned)
-CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *)
-CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t)
-CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t)
-CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
-    ctl_stats.arenas[mib[2]].astats.mapped, size_t)
-CTL_RO_CGEN(config_stats, stats_arenas_i_npurge,
-    ctl_stats.arenas[mib[2]].astats.npurge, uint64_t)
-CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise,
-    ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t)
-CTL_RO_CGEN(config_stats, stats_arenas_i_purged,
-    ctl_stats.arenas[mib[2]].astats.purged, uint64_t)
-
 static const ctl_named_node_t *
 stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i)
 {
@@ -1674,8 +1682,3 @@ label_return:
 	malloc_mutex_unlock(&ctl_mtx);
 	return (ret);
 }
-
-CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *)
-CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t)
-CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t)
-CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t)

src/huge.c

@@ -78,7 +78,7 @@ huge_palloc(size_t size, size_t alignment, bool zero)
 	return (ret);
 }
 
-void *
+bool
 huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
 {
 
@@ -89,15 +89,11 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
 	    && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
 	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
 		assert(CHUNK_CEILING(oldsize) == oldsize);
-		if (config_fill && opt_junk && size < oldsize) {
-			memset((void *)((uintptr_t)ptr + size), 0x5a,
-			    oldsize - size);
-		}
-		return (ptr);
+		return (false);
 	}
 
 	/* Reallocation would require a move. */
-	return (NULL);
+	return (true);
 }
 
 void *
@@ -108,9 +104,8 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 	size_t copysize;
 
 	/* Try to avoid moving the allocation. */
-	ret = huge_ralloc_no_move(ptr, oldsize, size, extra);
-	if (ret != NULL)
-		return (ret);
+	if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false)
+		return (ptr);
 
 	/*
 	 * size and oldsize are different enough that we need to use a
@@ -169,7 +164,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 			 */
 			char buf[BUFERROR_BUF];
 
-			buferror(buf, sizeof(buf));
+			buferror(get_errno(), buf, sizeof(buf));
 			malloc_printf("<jemalloc>: Error in mremap(): %s\n",
 			    buf);
 			if (opt_abort)
@@ -181,11 +176,34 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 #endif
 	{
 		memcpy(ret, ptr, copysize);
-		iqallocx(ptr, try_tcache_dalloc);
+		iqalloct(ptr, try_tcache_dalloc);
 	}
 	return (ret);
 }
 
+#ifdef JEMALLOC_JET
+#undef huge_dalloc_junk
+#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
+#endif
+static void
+huge_dalloc_junk(void *ptr, size_t usize)
+{
+
+	if (config_fill && config_dss && opt_junk) {
+		/*
+		 * Only bother junk filling if the chunk isn't about to be
+		 * unmapped.
+		 */
+		if (config_munmap == false || (config_dss && chunk_in_dss(ptr)))
+			memset(ptr, 0x5a, usize);
+	}
+}
+#ifdef JEMALLOC_JET
+#undef huge_dalloc_junk
+#define	huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
+huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
+#endif
+
 void
 huge_dalloc(void *ptr, bool unmap)
 {
@@ -208,8 +226,8 @@ huge_dalloc(void *ptr, bool unmap)
 
 	malloc_mutex_unlock(&huge_mtx);
 
-	if (unmap && config_fill && config_dss && opt_junk)
-		memset(node->addr, 0x5a, node->size);
+	if (unmap)
+		huge_dalloc_junk(node->addr, node->size);
 
 	chunk_dealloc(node->addr, node->size, unmap);
 

src/mutex.c

@@ -6,7 +6,7 @@
 #endif
 
 #ifndef _CRT_SPINCOUNT
-#define _CRT_SPINCOUNT 4000
+#define	_CRT_SPINCOUNT 4000
 #endif
 
 /******************************************************************************/

src/quarantine.c

@@ -141,8 +141,17 @@ quarantine(void *ptr)
 		obj->usize = usize;
 		quarantine->curbytes += usize;
 		quarantine->curobjs++;
-		if (opt_junk)
-			memset(ptr, 0x5a, usize);
+		if (config_fill && opt_junk) {
+			/*
+			 * Only do redzone validation if Valgrind isn't in
+			 * operation.
+			 */
+			if ((config_valgrind == false || opt_valgrind == false)
+			    && usize <= SMALL_MAXCLASS)
+				arena_quarantine_junk_small(ptr, usize);
+			else
+				memset(ptr, 0x5a, usize);
+		}
 	} else {
 		assert(quarantine->curbytes == 0);
 		idalloc(ptr);

src/rtree.c

@@ -2,42 +2,55 @@
 #include "jemalloc/internal/jemalloc_internal.h"
 
 rtree_t *
-rtree_new(unsigned bits)
+rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc)
 {
 	rtree_t *ret;
-	unsigned bits_per_level, height, i;
+	unsigned bits_per_level, bits_in_leaf, height, i;
+
+	assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3));
 
 	bits_per_level = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1;
-	height = bits / bits_per_level;
-	if (height * bits_per_level != bits)
-		height++;
-	assert(height * bits_per_level >= bits);
+	bits_in_leaf = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1;
+	if (bits > bits_in_leaf) {
+		height = 1 + (bits - bits_in_leaf) / bits_per_level;
+		if ((height-1) * bits_per_level + bits_in_leaf != bits)
+			height++;
+	} else {
+		height = 1;
+	}
+	assert((height-1) * bits_per_level + bits_in_leaf >= bits);
 
-	ret = (rtree_t*)base_alloc(offsetof(rtree_t, level2bits) +
+	ret = (rtree_t*)alloc(offsetof(rtree_t, level2bits) +
 	    (sizeof(unsigned) * height));
 	if (ret == NULL)
 		return (NULL);
 	memset(ret, 0, offsetof(rtree_t, level2bits) + (sizeof(unsigned) *
 	    height));
 
+	ret->alloc = alloc;
+	ret->dalloc = dalloc;
 	if (malloc_mutex_init(&ret->mutex)) {
-		/* Leak the rtree. */
+		if (dalloc != NULL)
+			dalloc(ret);
 		return (NULL);
 	}
 	ret->height = height;
-	if (bits_per_level * height > bits)
-		ret->level2bits[0] = bits % bits_per_level;
-	else
-		ret->level2bits[0] = bits_per_level;
-	for (i = 1; i < height; i++)
-		ret->level2bits[i] = bits_per_level;
+	if (height > 1) {
+		if ((height-1) * bits_per_level + bits_in_leaf > bits) {
+			ret->level2bits[0] = (bits - bits_in_leaf) %
+			    bits_per_level;
+		} else
+			ret->level2bits[0] = bits_per_level;
+		for (i = 1; i < height-1; i++)
+			ret->level2bits[i] = bits_per_level;
+		ret->level2bits[height-1] = bits_in_leaf;
+	} else
+		ret->level2bits[0] = bits;
 
-	ret->root = (void**)base_alloc(sizeof(void *) << ret->level2bits[0]);
+	ret->root = (void**)alloc(sizeof(void *) << ret->level2bits[0]);
 	if (ret->root == NULL) {
-		/*
-		 * We leak the rtree here, since there's no generic base
-		 * deallocation.
-		 */
+		if (dalloc != NULL)
+			dalloc(ret);
 		return (NULL);
 	}
 	memset(ret->root, 0, sizeof(void *) << ret->level2bits[0]);
@@ -45,6 +58,31 @@ rtree_new(unsigned bits)
 	return (ret);
 }
 
+static void
+rtree_delete_subtree(rtree_t *rtree, void **node, unsigned level)
+{
+
+	if (level < rtree->height - 1) {
+		size_t nchildren, i;
+
+		nchildren = ZU(1) << rtree->level2bits[level];
+		for (i = 0; i < nchildren; i++) {
+			void **child = (void **)node[i];
+			if (child != NULL)
+				rtree_delete_subtree(rtree, child, level + 1);
+		}
+	}
+	rtree->dalloc(node);
+}
+
+void
+rtree_delete(rtree_t *rtree)
+{
+
+	rtree_delete_subtree(rtree, rtree->root, 0);
+	rtree->dalloc(rtree);
+}
+
 void
 rtree_prefork(rtree_t *rtree)
 {

src/stats.c

@@ -345,25 +345,25 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		malloc_cprintf(write_cb, cbopaque, "Assertions %s\n",
 		    bv ? "enabled" : "disabled");
 
-#define OPT_WRITE_BOOL(n)						\
+#define	OPT_WRITE_BOOL(n)						\
 		if ((err = je_mallctl("opt."#n, &bv, &bsz, NULL, 0))	\
 		    == 0) {						\
 			malloc_cprintf(write_cb, cbopaque,		\
 			    "  opt."#n": %s\n", bv ? "true" : "false");	\
 		}
-#define OPT_WRITE_SIZE_T(n)						\
+#define	OPT_WRITE_SIZE_T(n)						\
 		if ((err = je_mallctl("opt."#n, &sv, &ssz, NULL, 0))	\
 		    == 0) {						\
 			malloc_cprintf(write_cb, cbopaque,		\
 			"  opt."#n": %zu\n", sv);			\
 		}
-#define OPT_WRITE_SSIZE_T(n)						\
+#define	OPT_WRITE_SSIZE_T(n)						\
 		if ((err = je_mallctl("opt."#n, &ssv, &sssz, NULL, 0))	\
 		    == 0) {						\
 			malloc_cprintf(write_cb, cbopaque,		\
 			    "  opt."#n": %zd\n", ssv);			\
 		}
-#define OPT_WRITE_CHAR_P(n)						\
+#define	OPT_WRITE_CHAR_P(n)						\
 		if ((err = je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0))	\
 		    == 0) {						\
 			malloc_cprintf(write_cb, cbopaque,		\

src/tcache.c

@@ -260,8 +260,8 @@ tcache_arena_dissociate(tcache_t *tcache)
 		/* Unlink from list of extant tcaches. */
 		malloc_mutex_lock(&tcache->arena->lock);
 		ql_remove(&tcache->arena->tcache_ql, tcache, link);
-		malloc_mutex_unlock(&tcache->arena->lock);
 		tcache_stats_merge(tcache, tcache->arena);
+		malloc_mutex_unlock(&tcache->arena->lock);
 	}
 }
 
@@ -292,7 +292,7 @@ tcache_create(arena_t *arena)
 	else if (size <= tcache_maxclass)
 		tcache = (tcache_t *)arena_malloc_large(arena, size, true);
 	else
-		tcache = (tcache_t *)icallocx(size, false, arena);
+		tcache = (tcache_t *)icalloct(size, false, arena);
 
 	if (tcache == NULL)
 		return (NULL);
@@ -366,7 +366,7 @@ tcache_destroy(tcache_t *tcache)
 
 		arena_dalloc_large(arena, chunk, tcache);
 	} else
-		idallocx(tcache, false);
+		idalloct(tcache, false);
 }
 
 void
@@ -399,11 +399,14 @@ tcache_thread_cleanup(void *arg)
 	}
 }
 
+/* Caller must own arena->lock. */
 void
 tcache_stats_merge(tcache_t *tcache, arena_t *arena)
 {
 	unsigned i;
 
+	cassert(config_stats);
+
 	/* Merge and reset tcache stats. */
 	for (i = 0; i < NBINS; i++) {
 		arena_bin_t *bin = &arena->bins[i];

src/tsd.c

@@ -21,7 +21,7 @@ void
 malloc_tsd_dalloc(void *wrapper)
 {
 
-	idalloc(wrapper);
+	idalloct(wrapper, false);
 }
 
 void
@@ -105,3 +105,37 @@ JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used)
 static const BOOL	(WINAPI *tls_callback)(HINSTANCE hinstDLL,
     DWORD fdwReason, LPVOID lpvReserved) = _tls_callback;
 #endif
+
+#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
+    !defined(_WIN32))
+void *
+tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block)
+{
+	pthread_t self = pthread_self();
+	tsd_init_block_t *iter;
+
+	/* Check whether this thread has already inserted into the list. */
+	malloc_mutex_lock(&head->lock);
+	ql_foreach(iter, &head->blocks, link) {
+		if (iter->thread == self) {
+			malloc_mutex_unlock(&head->lock);
+			return (iter->data);
+		}
+	}
+	/* Insert block into list. */
+	ql_elm_new(block, link);
+	block->thread = self;
+	ql_tail_insert(&head->blocks, block, link);
+	malloc_mutex_unlock(&head->lock);
+	return (NULL);
+}
+
+void
+tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block)
+{
+
+	malloc_mutex_lock(&head->lock);
+	ql_remove(&head->blocks, block, link);
+	malloc_mutex_unlock(&head->lock);
+}
+#endif

src/util.c

@@ -77,7 +77,7 @@ malloc_write(const char *s)
  * provide a wrapper.
  */
 int
-buferror(char *buf, size_t buflen)
+buferror(int err, char *buf, size_t buflen)
 {
 
 #ifdef _WIN32
@@ -85,34 +85,36 @@ buferror(char *buf, size_t buflen)
 	    (LPSTR)buf, buflen, NULL);
 	return (0);
 #elif defined(_GNU_SOURCE)
-	char *b = strerror_r(errno, buf, buflen);
+	char *b = strerror_r(err, buf, buflen);
 	if (b != buf) {
 		strncpy(buf, b, buflen);
 		buf[buflen-1] = '\0';
 	}
 	return (0);
 #else
-	return (strerror_r(errno, buf, buflen));
+	return (strerror_r(err, buf, buflen));
 #endif
 }
 
 uintmax_t
-malloc_strtoumax(const char *nptr, char **endptr, int base)
+malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base)
 {
 	uintmax_t ret, digit;
 	int b;
 	bool neg;
 	const char *p, *ns;
 
+	p = nptr;
 	if (base < 0 || base == 1 || base > 36) {
+		ns = p;
 		set_errno(EINVAL);
-		return (UINTMAX_MAX);
+		ret = UINTMAX_MAX;
+		goto label_return;
 	}
 	b = base;
 
 	/* Swallow leading whitespace and get sign, if any. */
 	neg = false;
-	p = nptr;
 	while (true) {
 		switch (*p) {
 		case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
@@ -146,7 +148,7 @@ malloc_strtoumax(const char *nptr, char **endptr, int base)
 			if (b == 8)
 				p++;
 			break;
-		case 'x':
+		case 'X': case 'x':
 			switch (p[2]) {
 			case '0': case '1': case '2': case '3': case '4':
 			case '5': case '6': case '7': case '8': case '9':
@@ -164,7 +166,9 @@ malloc_strtoumax(const char *nptr, char **endptr, int base)
 			}
 			break;
 		default:
-			break;
+			p++;
+			ret = 0;
+			goto label_return;
 		}
 	}
 	if (b == 0)
@@ -181,13 +185,22 @@ malloc_strtoumax(const char *nptr, char **endptr, int base)
 		if (ret < pret) {
 			/* Overflow. */
 			set_errno(ERANGE);
-			return (UINTMAX_MAX);
+			ret = UINTMAX_MAX;
+			goto label_return;
 		}
 		p++;
 	}
 	if (neg)
 		ret = -ret;
 
+	if (p == ns) {
+		/* No conversion performed. */
+		set_errno(EINVAL);
+		ret = UINTMAX_MAX;
+		goto label_return;
+	}
+
+label_return:
 	if (endptr != NULL) {
 		if (p == ns) {
 			/* No characters were converted. */
@@ -195,7 +208,6 @@ malloc_strtoumax(const char *nptr, char **endptr, int base)
 		} else
 			*endptr = (char *)p;
 	}
-
 	return (ret);
 }
 
@@ -331,7 +343,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 			APPEND_C(' ');					\
 	}								\
 } while (0)
-#define GET_ARG_NUMERIC(val, len) do {					\
+#define	GET_ARG_NUMERIC(val, len) do {					\
 	switch (len) {							\
 	case '?':							\
 		val = va_arg(ap, int);					\
@@ -354,6 +366,9 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 	case 'j':							\
 		val = va_arg(ap, intmax_t);				\
 		break;							\
+	case 'j' | 0x80:						\
+		val = va_arg(ap, uintmax_t);				\
+		break;							\
 	case 't':							\
 		val = va_arg(ap, ptrdiff_t);				\
 		break;							\
@@ -385,11 +400,6 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 			unsigned char len = '?';
 
 			f++;
-			if (*f == '%') {
-				/* %% */
-				APPEND_C(*f);
-				break;
-			}
 			/* Flags. */
 			while (true) {
 				switch (*f) {
@@ -419,6 +429,10 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 			case '*':
 				width = va_arg(ap, int);
 				f++;
+				if (width < 0) {
+					left_justify = true;
+					width = -width;
+				}
 				break;
 			case '0': case '1': case '2': case '3': case '4':
 			case '5': case '6': case '7': case '8': case '9': {
@@ -428,19 +442,16 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 				assert(uwidth != UINTMAX_MAX || get_errno() !=
 				    ERANGE);
 				width = (int)uwidth;
-				if (*f == '.') {
-					f++;
-					goto label_precision;
-				} else
-					goto label_length;
 				break;
-			} case '.':
-				f++;
-				goto label_precision;
-			default: goto label_length;
+			} default:
+				break;
 			}
+			/* Width/precision separator. */
+			if (*f == '.')
+				f++;
+			else
+				goto label_length;
 			/* Precision. */
-			label_precision:
 			switch (*f) {
 			case '*':
 				prec = va_arg(ap, int);
@@ -469,16 +480,8 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 				} else
 					len = 'l';
 				break;
-			case 'j':
-				len = 'j';
-				f++;
-				break;
-			case 't':
-				len = 't';
-				f++;
-				break;
-			case 'z':
-				len = 'z';
+			case 'q': case 'j': case 't': case 'z':
+				len = *f;
 				f++;
 				break;
 			default: break;
@@ -487,6 +490,11 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 			switch (*f) {
 				char *s;
 				size_t slen;
+			case '%':
+				/* %% */
+				APPEND_C(*f);
+				f++;
+				break;
 			case 'd': case 'i': {
 				intmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 				char buf[D2S_BUFSIZE];
@@ -540,7 +548,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 				assert(len == '?' || len == 'l');
 				assert_not_implemented(len != 'l');
 				s = va_arg(ap, char *);
-				slen = (prec == -1) ? strlen(s) : prec;
+				slen = (prec < 0) ? strlen(s) : prec;
 				APPEND_PADDED_S(s, slen, width, left_justify);
 				f++;
 				break;
@@ -553,8 +561,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 				APPEND_PADDED_S(s, slen, width, left_justify);
 				f++;
 				break;
-			}
-			default: not_implemented();
+			} default: not_reached();
 			}
 			break;
 		} default: {

src/zone.c

@@ -137,7 +137,7 @@ zone_destroy(malloc_zone_t *zone)
 {
 
 	/* This function should never be called. */
-	assert(false);
+	not_reached();
 	return (NULL);
 }
 

test/ALLOCM_ARENA.c

@@ -1,68 +0,0 @@
-#define	JEMALLOC_MANGLE
-#include "jemalloc_test.h"
-
-#define	NTHREADS 10
-
-void *
-je_thread_start(void *arg)
-{
-	unsigned thread_ind = (unsigned)(uintptr_t)arg;
-	unsigned arena_ind;
-	int r;
-	void *p;
-	size_t rsz, sz;
-
-	sz = sizeof(arena_ind);
-	if (mallctl("arenas.extend", &arena_ind, &sz, NULL, 0)
-	    != 0) {
-		malloc_printf("Error in arenas.extend\n");
-		abort();
-	}
-
-	if (thread_ind % 4 != 3) {
-		size_t mib[3];
-		size_t miblen = sizeof(mib) / sizeof(size_t);
-		const char *dss_precs[] = {"disabled", "primary", "secondary"};
-		const char *dss = dss_precs[thread_ind %
-		    (sizeof(dss_precs)/sizeof(char*))];
-		if (mallctlnametomib("arena.0.dss", mib, &miblen) != 0) {
-			malloc_printf("Error in mallctlnametomib()\n");
-			abort();
-		}
-		mib[1] = arena_ind;
-		if (mallctlbymib(mib, miblen, NULL, NULL, (void *)&dss,
-		    sizeof(const char *))) {
-			malloc_printf("Error in mallctlbymib()\n");
-			abort();
-		}
-	}
-
-	r = allocm(&p, &rsz, 1, ALLOCM_ARENA(arena_ind));
-	if (r != ALLOCM_SUCCESS) {
-		malloc_printf("Unexpected allocm() error\n");
-		abort();
-	}
-	dallocm(p, 0);
-
-	return (NULL);
-}
-
-int
-main(void)
-{
-	je_thread_t threads[NTHREADS];
-	unsigned i;
-
-	malloc_printf("Test begin\n");
-
-	for (i = 0; i < NTHREADS; i++) {
-		je_thread_create(&threads[i], je_thread_start,
-		    (void *)(uintptr_t)i);
-	}
-
-	for (i = 0; i < NTHREADS; i++)
-		je_thread_join(threads[i], NULL);
-
-	malloc_printf("Test end\n");
-	return (0);
-}

test/ALLOCM_ARENA.exp

@@ -1,2 +0,0 @@
-Test begin
-Test end

test/aligned_alloc.exp

@@ -1,25 +0,0 @@
-Test begin
-Alignment: 8
-Alignment: 16
-Alignment: 32
-Alignment: 64
-Alignment: 128
-Alignment: 256
-Alignment: 512
-Alignment: 1024
-Alignment: 2048
-Alignment: 4096
-Alignment: 8192
-Alignment: 16384
-Alignment: 32768
-Alignment: 65536
-Alignment: 131072
-Alignment: 262144
-Alignment: 524288
-Alignment: 1048576
-Alignment: 2097152
-Alignment: 4194304
-Alignment: 8388608
-Alignment: 16777216
-Alignment: 33554432
-Test end

test/allocated.c

@@ -1,118 +0,0 @@
-#define	JEMALLOC_MANGLE
-#include "jemalloc_test.h"
-
-void *
-je_thread_start(void *arg)
-{
-	int err;
-	void *p;
-	uint64_t a0, a1, d0, d1;
-	uint64_t *ap0, *ap1, *dp0, *dp1;
-	size_t sz, usize;
-
-	sz = sizeof(a0);
-	if ((err = mallctl("thread.allocated", &a0, &sz, NULL, 0))) {
-		if (err == ENOENT) {
-#ifdef JEMALLOC_STATS
-			assert(false);
-#endif
-			goto label_return;
-		}
-		malloc_printf("%s(): Error in mallctl(): %s\n", __func__,
-		    strerror(err));
-		exit(1);
-	}
-	sz = sizeof(ap0);
-	if ((err = mallctl("thread.allocatedp", &ap0, &sz, NULL, 0))) {
-		if (err == ENOENT) {
-#ifdef JEMALLOC_STATS
-			assert(false);
-#endif
-			goto label_return;
-		}
-		malloc_printf("%s(): Error in mallctl(): %s\n", __func__,
-		    strerror(err));
-		exit(1);
-	}
-	assert(*ap0 == a0);
-
-	sz = sizeof(d0);
-	if ((err = mallctl("thread.deallocated", &d0, &sz, NULL, 0))) {
-		if (err == ENOENT) {
-#ifdef JEMALLOC_STATS
-			assert(false);
-#endif
-			goto label_return;
-		}
-		malloc_printf("%s(): Error in mallctl(): %s\n", __func__,
-		    strerror(err));
-		exit(1);
-	}
-	sz = sizeof(dp0);
-	if ((err = mallctl("thread.deallocatedp", &dp0, &sz, NULL, 0))) {
-		if (err == ENOENT) {
-#ifdef JEMALLOC_STATS
-			assert(false);
-#endif
-			goto label_return;
-		}
-		malloc_printf("%s(): Error in mallctl(): %s\n", __func__,
-		    strerror(err));
-		exit(1);
-	}
-	assert(*dp0 == d0);
-
-	p = malloc(1);
-	if (p == NULL) {
-		malloc_printf("%s(): Error in malloc()\n", __func__);
-		exit(1);
-	}
-
-	sz = sizeof(a1);
-	mallctl("thread.allocated", &a1, &sz, NULL, 0);
-	sz = sizeof(ap1);
-	mallctl("thread.allocatedp", &ap1, &sz, NULL, 0);
-	assert(*ap1 == a1);
-	assert(ap0 == ap1);
-
-	usize = malloc_usable_size(p);
-	assert(a0 + usize <= a1);
-
-	free(p);
-
-	sz = sizeof(d1);
-	mallctl("thread.deallocated", &d1, &sz, NULL, 0);
-	sz = sizeof(dp1);
-	mallctl("thread.deallocatedp", &dp1, &sz, NULL, 0);
-	assert(*dp1 == d1);
-	assert(dp0 == dp1);
-
-	assert(d0 + usize <= d1);
-
-label_return:
-	return (NULL);
-}
-
-int
-main(void)
-{
-	int ret = 0;
-	je_thread_t thread;
-
-	malloc_printf("Test begin\n");
-
-	je_thread_start(NULL);
-
-	je_thread_create(&thread, je_thread_start, NULL);
-	je_thread_join(thread, NULL);
-
-	je_thread_start(NULL);
-
-	je_thread_create(&thread, je_thread_start, NULL);
-	je_thread_join(thread, NULL);
-
-	je_thread_start(NULL);
-
-	malloc_printf("Test end\n");
-	return (ret);
-}

test/allocated.exp

@@ -1,2 +0,0 @@
-Test begin
-Test end

test/allocm.c

@@ -1,194 +0,0 @@
-#define	JEMALLOC_MANGLE
-#include "jemalloc_test.h"
-
-#define CHUNK 0x400000
-/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */
-#define MAXALIGN ((size_t)0x2000000LU)
-#define NITER 4
-
-int
-main(void)
-{
-	int r;
-	void *p;
-	size_t nsz, rsz, sz, alignment, total;
-	unsigned i;
-	void *ps[NITER];
-
-	malloc_printf("Test begin\n");
-
-	sz = 42;
-	nsz = 0;
-	r = nallocm(&nsz, sz, 0);
-	if (r != ALLOCM_SUCCESS) {
-		malloc_printf("Unexpected nallocm() error\n");
-		abort();
-	}
-	rsz = 0;
-	r = allocm(&p, &rsz, sz, 0);
-	if (r != ALLOCM_SUCCESS) {
-		malloc_printf("Unexpected allocm() error\n");
-		abort();
-	}
-	if (rsz < sz)
-		malloc_printf("Real size smaller than expected\n");
-	if (nsz != rsz)
-		malloc_printf("nallocm()/allocm() rsize mismatch\n");
-	if (dallocm(p, 0) != ALLOCM_SUCCESS)
-		malloc_printf("Unexpected dallocm() error\n");
-
-	r = allocm(&p, NULL, sz, 0);
-	if (r != ALLOCM_SUCCESS) {
-		malloc_printf("Unexpected allocm() error\n");
-		abort();
-	}
-	if (dallocm(p, 0) != ALLOCM_SUCCESS)
-		malloc_printf("Unexpected dallocm() error\n");
-
-	nsz = 0;
-	r = nallocm(&nsz, sz, ALLOCM_ZERO);
-	if (r != ALLOCM_SUCCESS) {
-		malloc_printf("Unexpected nallocm() error\n");
-		abort();
-	}
-	rsz = 0;
-	r = allocm(&p, &rsz, sz, ALLOCM_ZERO);
-	if (r != ALLOCM_SUCCESS) {
-		malloc_printf("Unexpected allocm() error\n");
-		abort();
-	}
-	if (nsz != rsz)
-		malloc_printf("nallocm()/allocm() rsize mismatch\n");
-	if (dallocm(p, 0) != ALLOCM_SUCCESS)
-		malloc_printf("Unexpected dallocm() error\n");
-
-#if LG_SIZEOF_PTR == 3
-	alignment = UINT64_C(0x8000000000000000);
-	sz        = UINT64_C(0x8000000000000000);
-#else
-	alignment = 0x80000000LU;
-	sz        = 0x80000000LU;
-#endif
-	nsz = 0;
-	r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment));
-	if (r == ALLOCM_SUCCESS) {
-		malloc_printf(
-		    "Expected error for nallocm(&nsz, %zu, %#x)\n",
-		    sz, ALLOCM_ALIGN(alignment));
-	}
-	rsz = 0;
-	r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment));
-	if (r == ALLOCM_SUCCESS) {
-		malloc_printf(
-		    "Expected error for allocm(&p, %zu, %#x)\n",
-		    sz, ALLOCM_ALIGN(alignment));
-	}
-	if (nsz != rsz)
-		malloc_printf("nallocm()/allocm() rsize mismatch\n");
-
-#if LG_SIZEOF_PTR == 3
-	alignment = UINT64_C(0x4000000000000000);
-	sz        = UINT64_C(0x8400000000000001);
-#else
-	alignment = 0x40000000LU;
-	sz        = 0x84000001LU;
-#endif
-	nsz = 0;
-	r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment));
-	if (r != ALLOCM_SUCCESS)
-		malloc_printf("Unexpected nallocm() error\n");
-	rsz = 0;
-	r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment));
-	if (r == ALLOCM_SUCCESS) {
-		malloc_printf(
-		    "Expected error for allocm(&p, %zu, %#x)\n",
-		    sz, ALLOCM_ALIGN(alignment));
-	}
-
-	alignment = 0x10LU;
-#if LG_SIZEOF_PTR == 3
-	sz = UINT64_C(0xfffffffffffffff0);
-#else
-	sz = 0xfffffff0LU;
-#endif
-	nsz = 0;
-	r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment));
-	if (r == ALLOCM_SUCCESS) {
-		malloc_printf(
-		    "Expected error for nallocm(&nsz, %zu, %#x)\n",
-		    sz, ALLOCM_ALIGN(alignment));
-	}
-	rsz = 0;
-	r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment));
-	if (r == ALLOCM_SUCCESS) {
-		malloc_printf(
-		    "Expected error for allocm(&p, %zu, %#x)\n",
-		    sz, ALLOCM_ALIGN(alignment));
-	}
-	if (nsz != rsz)
-		malloc_printf("nallocm()/allocm() rsize mismatch\n");
-
-	for (i = 0; i < NITER; i++)
-		ps[i] = NULL;
-
-	for (alignment = 8;
-	    alignment <= MAXALIGN;
-	    alignment <<= 1) {
-		total = 0;
-		malloc_printf("Alignment: %zu\n", alignment);
-		for (sz = 1;
-		    sz < 3 * alignment && sz < (1U << 31);
-		    sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) {
-			for (i = 0; i < NITER; i++) {
-				nsz = 0;
-				r = nallocm(&nsz, sz,
-				    ALLOCM_ALIGN(alignment) | ALLOCM_ZERO);
-				if (r != ALLOCM_SUCCESS) {
-					malloc_printf(
-					    "nallocm() error for size %zu"
-					    " (%#zx): %d\n",
-					    sz, sz, r);
-					exit(1);
-				}
-				rsz = 0;
-				r = allocm(&ps[i], &rsz, sz,
-				    ALLOCM_ALIGN(alignment) | ALLOCM_ZERO);
-				if (r != ALLOCM_SUCCESS) {
-					malloc_printf(
-					    "allocm() error for size %zu"
-					    " (%#zx): %d\n",
-					    sz, sz, r);
-					exit(1);
-				}
-				if (rsz < sz) {
-					malloc_printf(
-					    "Real size smaller than"
-					    " expected\n");
-				}
-				if (nsz != rsz) {
-					malloc_printf(
-					    "nallocm()/allocm() rsize"
-					    " mismatch\n");
-				}
-				if ((uintptr_t)p & (alignment-1)) {
-					malloc_printf(
-					    "%p inadequately aligned for"
-					    " alignment: %zu\n", p, alignment);
-				}
-				sallocm(ps[i], &rsz, 0);
-				total += rsz;
-				if (total >= (MAXALIGN << 1))
-					break;
-			}
-			for (i = 0; i < NITER; i++) {
-				if (ps[i] != NULL) {
-					dallocm(ps[i], 0);
-					ps[i] = NULL;
-				}
-			}
-		}
-	}
-
-	malloc_printf("Test end\n");
-	return (0);
-}

test/allocm.exp

@@ -1,25 +0,0 @@
-Test begin
-Alignment: 8
-Alignment: 16
-Alignment: 32
-Alignment: 64
-Alignment: 128
-Alignment: 256
-Alignment: 512
-Alignment: 1024
-Alignment: 2048
-Alignment: 4096
-Alignment: 8192
-Alignment: 16384
-Alignment: 32768
-Alignment: 65536
-Alignment: 131072
-Alignment: 262144
-Alignment: 524288
-Alignment: 1048576
-Alignment: 2097152
-Alignment: 4194304
-Alignment: 8388608
-Alignment: 16777216
-Alignment: 33554432
-Test end

test/bitmap.exp

@@ -1,2 +0,0 @@
-Test begin
-Test end

test/include/test/SFMT-alti.h

@@ -0,0 +1,186 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/** 
+ * @file SFMT-alti.h 
+ *
+ * @brief SIMD oriented Fast Mersenne Twister(SFMT)
+ * pseudorandom number generator
+ *
+ * @author Mutsuo Saito (Hiroshima University)
+ * @author Makoto Matsumoto (Hiroshima University)
+ *
+ * Copyright (C) 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ * University. All rights reserved.
+ *
+ * The new BSD License is applied to this software.
+ * see LICENSE.txt
+ */
+
+#ifndef SFMT_ALTI_H
+#define SFMT_ALTI_H
+
+/**
+ * This function represents the recursion formula in AltiVec and BIG ENDIAN.
+ * @param a a 128-bit part of the interal state array
+ * @param b a 128-bit part of the interal state array
+ * @param c a 128-bit part of the interal state array
+ * @param d a 128-bit part of the interal state array
+ * @return output
+ */
+JEMALLOC_ALWAYS_INLINE
+static vector unsigned int vec_recursion(vector unsigned int a,
+						vector unsigned int b,
+						vector unsigned int c,
+						vector unsigned int d) {
+
+    const vector unsigned int sl1 = ALTI_SL1;
+    const vector unsigned int sr1 = ALTI_SR1;
+#ifdef ONLY64
+    const vector unsigned int mask = ALTI_MSK64;
+    const vector unsigned char perm_sl = ALTI_SL2_PERM64;
+    const vector unsigned char perm_sr = ALTI_SR2_PERM64;
+#else
+    const vector unsigned int mask = ALTI_MSK;
+    const vector unsigned char perm_sl = ALTI_SL2_PERM;
+    const vector unsigned char perm_sr = ALTI_SR2_PERM;
+#endif
+    vector unsigned int v, w, x, y, z;
+    x = vec_perm(a, (vector unsigned int)perm_sl, perm_sl);
+    v = a;
+    y = vec_sr(b, sr1);
+    z = vec_perm(c, (vector unsigned int)perm_sr, perm_sr);
+    w = vec_sl(d, sl1);
+    z = vec_xor(z, w);
+    y = vec_and(y, mask);
+    v = vec_xor(v, x);
+    z = vec_xor(z, y);
+    z = vec_xor(z, v);
+    return z;
+}
+
+/**
+ * This function fills the internal state array with pseudorandom
+ * integers.
+ */
+JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) {
+    int i;
+    vector unsigned int r, r1, r2;
+
+    r1 = ctx->sfmt[N - 2].s;
+    r2 = ctx->sfmt[N - 1].s;
+    for (i = 0; i < N - POS1; i++) {
+	r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1].s, r1, r2);
+	ctx->sfmt[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+    for (; i < N; i++) {
+	r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1 - N].s, r1, r2);
+	ctx->sfmt[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+}
+
+/**
+ * This function fills the user-specified array with pseudorandom
+ * integers.
+ *
+ * @param array an 128-bit array to be filled by pseudorandom numbers.  
+ * @param size number of 128-bit pesudorandom numbers to be generated.
+ */
+JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
+    int i, j;
+    vector unsigned int r, r1, r2;
+
+    r1 = ctx->sfmt[N - 2].s;
+    r2 = ctx->sfmt[N - 1].s;
+    for (i = 0; i < N - POS1; i++) {
+	r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1].s, r1, r2);
+	array[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+    for (; i < N; i++) {
+	r = vec_recursion(ctx->sfmt[i].s, array[i + POS1 - N].s, r1, r2);
+	array[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+    /* main loop */
+    for (; i < size - N; i++) {
+	r = vec_recursion(array[i - N].s, array[i + POS1 - N].s, r1, r2);
+	array[i].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+    for (j = 0; j < 2 * N - size; j++) {
+	ctx->sfmt[j].s = array[j + size - N].s;
+    }
+    for (; i < size; i++) {
+	r = vec_recursion(array[i - N].s, array[i + POS1 - N].s, r1, r2);
+	array[i].s = r;
+	ctx->sfmt[j++].s = r;
+	r1 = r2;
+	r2 = r;
+    }
+}
+
+#ifndef ONLY64
+#if defined(__APPLE__)
+#define ALTI_SWAP (vector unsigned char) \
+	(4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11)
+#else
+#define ALTI_SWAP {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11}
+#endif
+/**
+ * This function swaps high and low 32-bit of 64-bit integers in user
+ * specified array.
+ *
+ * @param array an 128-bit array to be swaped.
+ * @param size size of 128-bit array.
+ */
+JEMALLOC_INLINE void swap(w128_t *array, int size) {
+    int i;
+    const vector unsigned char perm = ALTI_SWAP;
+
+    for (i = 0; i < size; i++) {
+	array[i].s = vec_perm(array[i].s, (vector unsigned int)perm, perm);
+    }
+}
+#endif
+
+#endif

test/include/test/SFMT-params.h

@@ -0,0 +1,132 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS_H
+#define SFMT_PARAMS_H
+
+#if !defined(MEXP)
+#ifdef __GNUC__
+  #warning "MEXP is not defined. I assume MEXP is 19937."
+#endif
+  #define MEXP 19937
+#endif
+/*-----------------
+  BASIC DEFINITIONS
+  -----------------*/
+/** Mersenne Exponent. The period of the sequence 
+ *  is a multiple of 2^MEXP-1.
+ * #define MEXP 19937 */
+/** SFMT generator has an internal state array of 128-bit integers,
+ * and N is its size. */
+#define N (MEXP / 128 + 1)
+/** N32 is the size of internal state array when regarded as an array
+ * of 32-bit integers.*/
+#define N32 (N * 4)
+/** N64 is the size of internal state array when regarded as an array
+ * of 64-bit integers.*/
+#define N64 (N * 2)
+
+/*----------------------
+  the parameters of SFMT
+  following definitions are in paramsXXXX.h file.
+  ----------------------*/
+/** the pick up position of the array.
+#define POS1 122 
+*/
+
+/** the parameter of shift left as four 32-bit registers.
+#define SL1 18
+ */
+
+/** the parameter of shift left as one 128-bit register. 
+ * The 128-bit integer is shifted by (SL2 * 8) bits. 
+#define SL2 1 
+*/
+
+/** the parameter of shift right as four 32-bit registers.
+#define SR1 11
+*/
+
+/** the parameter of shift right as one 128-bit register. 
+ * The 128-bit integer is shifted by (SL2 * 8) bits. 
+#define SR2 1 
+*/
+
+/** A bitmask, used in the recursion.  These parameters are introduced
+ * to break symmetry of SIMD.
+#define MSK1 0xdfffffefU
+#define MSK2 0xddfecb7fU
+#define MSK3 0xbffaffffU
+#define MSK4 0xbffffff6U 
+*/
+
+/** These definitions are part of a 128-bit period certification vector.
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0xc98e126aU
+*/
+
+#if MEXP == 607
+  #include "test/SFMT-params607.h"
+#elif MEXP == 1279
+  #include "test/SFMT-params1279.h"
+#elif MEXP == 2281
+  #include "test/SFMT-params2281.h"
+#elif MEXP == 4253
+  #include "test/SFMT-params4253.h"
+#elif MEXP == 11213
+  #include "test/SFMT-params11213.h"
+#elif MEXP == 19937
+  #include "test/SFMT-params19937.h"
+#elif MEXP == 44497
+  #include "test/SFMT-params44497.h"
+#elif MEXP == 86243
+  #include "test/SFMT-params86243.h"
+#elif MEXP == 132049
+  #include "test/SFMT-params132049.h"
+#elif MEXP == 216091
+  #include "test/SFMT-params216091.h"
+#else
+#ifdef __GNUC__
+  #error "MEXP is not valid."
+  #undef MEXP
+#else
+  #undef MEXP
+#endif
+
+#endif
+
+#endif /* SFMT_PARAMS_H */

test/include/test/SFMT-params11213.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS11213_H
+#define SFMT_PARAMS11213_H
+
+#define POS1	68
+#define SL1	14
+#define SL2	3
+#define SR1	7
+#define SR2	3
+#define MSK1	0xeffff7fbU
+#define MSK2	0xffffffefU
+#define MSK3	0xdfdfbfffU
+#define MSK4	0x7fffdbfdU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0xe8148000U
+#define PARITY4	0xd0c7afa3U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12}
+    #define ALTI_SR2_PERM64	{13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-11213:68-14-3-7-3:effff7fb-ffffffef-dfdfbfff-7fffdbfd"
+
+#endif /* SFMT_PARAMS11213_H */

test/include/test/SFMT-params1279.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS1279_H
+#define SFMT_PARAMS1279_H
+
+#define POS1	7
+#define SL1	14
+#define SL2	3
+#define SR1	5
+#define SR2	1
+#define MSK1	0xf7fefffdU
+#define MSK2	0x7fefcfffU
+#define MSK3	0xaff3ef3fU
+#define MSK4	0xb5ffff7fU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0x20000000U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-1279:7-14-3-5-1:f7fefffd-7fefcfff-aff3ef3f-b5ffff7f"
+
+#endif /* SFMT_PARAMS1279_H */

test/include/test/SFMT-params132049.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS132049_H
+#define SFMT_PARAMS132049_H
+
+#define POS1	110
+#define SL1	19
+#define SL2	1
+#define SR1	21
+#define SR2	1
+#define MSK1	0xffffbb5fU
+#define MSK2	0xfb6ebf95U
+#define MSK3	0xfffefffaU
+#define MSK4	0xcff77fffU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0xcb520000U
+#define PARITY4	0xc7e91c7dU
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8}
+    #define ALTI_SL2_PERM64	{1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-132049:110-19-1-21-1:ffffbb5f-fb6ebf95-fffefffa-cff77fff"
+
+#endif /* SFMT_PARAMS132049_H */

test/include/test/SFMT-params19937.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS19937_H
+#define SFMT_PARAMS19937_H
+
+#define POS1	122
+#define SL1	18
+#define SL2	1
+#define SR1	11
+#define SR2	1
+#define MSK1	0xdfffffefU
+#define MSK2	0xddfecb7fU
+#define MSK3	0xbffaffffU
+#define MSK4	0xbffffff6U
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0x13c9e684U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8}
+    #define ALTI_SL2_PERM64	{1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-19937:122-18-1-11-1:dfffffef-ddfecb7f-bffaffff-bffffff6"
+
+#endif /* SFMT_PARAMS19937_H */

test/include/test/SFMT-params216091.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS216091_H
+#define SFMT_PARAMS216091_H
+
+#define POS1	627
+#define SL1	11
+#define SL2	3
+#define SR1	10
+#define SR2	1
+#define MSK1	0xbff7bff7U
+#define MSK2	0xbfffffffU
+#define MSK3	0xbffffa7fU
+#define MSK4	0xffddfbfbU
+#define PARITY1	0xf8000001U
+#define PARITY2	0x89e80709U
+#define PARITY3	0x3bd2b64bU
+#define PARITY4	0x0c64b1e4U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-216091:627-11-3-10-1:bff7bff7-bfffffff-bffffa7f-ffddfbfb"
+
+#endif /* SFMT_PARAMS216091_H */

test/include/test/SFMT-params2281.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS2281_H
+#define SFMT_PARAMS2281_H
+
+#define POS1	12
+#define SL1	19
+#define SL2	1
+#define SR1	5
+#define SR2	1
+#define MSK1	0xbff7ffbfU
+#define MSK2	0xfdfffffeU
+#define MSK3	0xf7ffef7fU
+#define MSK4	0xf2f7cbbfU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0x41dfa600U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8}
+    #define ALTI_SL2_PERM64	{1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-2281:12-19-1-5-1:bff7ffbf-fdfffffe-f7ffef7f-f2f7cbbf"
+
+#endif /* SFMT_PARAMS2281_H */

test/include/test/SFMT-params4253.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS4253_H
+#define SFMT_PARAMS4253_H
+
+#define POS1	17
+#define SL1	20
+#define SL2	1
+#define SR1	7
+#define SR2	1
+#define MSK1	0x9f7bffffU
+#define MSK2	0x9fffff5fU
+#define MSK3	0x3efffffbU
+#define MSK4	0xfffff7bbU
+#define PARITY1	0xa8000001U
+#define PARITY2	0xaf5390a3U
+#define PARITY3	0xb740b3f8U
+#define PARITY4	0x6c11486dU
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8}
+    #define ALTI_SL2_PERM64	{1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-4253:17-20-1-7-1:9f7bffff-9fffff5f-3efffffb-fffff7bb"
+
+#endif /* SFMT_PARAMS4253_H */

test/include/test/SFMT-params44497.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS44497_H
+#define SFMT_PARAMS44497_H
+
+#define POS1	330
+#define SL1	5
+#define SL2	3
+#define SR1	9
+#define SR2	3
+#define MSK1	0xeffffffbU
+#define MSK2	0xdfbebfffU
+#define MSK3	0xbfbf7befU
+#define MSK4	0x9ffd7bffU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0xa3ac4000U
+#define PARITY4	0xecc1327aU
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12}
+    #define ALTI_SR2_PERM64	{13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-44497:330-5-3-9-3:effffffb-dfbebfff-bfbf7bef-9ffd7bff"
+
+#endif /* SFMT_PARAMS44497_H */

test/include/test/SFMT-params607.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS607_H
+#define SFMT_PARAMS607_H
+
+#define POS1	2
+#define SL1	15
+#define SL2	3
+#define SR1	13
+#define SR2	3
+#define MSK1	0xfdff37ffU
+#define MSK2	0xef7f3f7dU
+#define MSK3	0xff777b7dU
+#define MSK4	0x7ff7fb2fU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0x5986f054U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10}
+    #define ALTI_SL2_PERM64	{3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2}
+    #define ALTI_SR2_PERM	{5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12}
+    #define ALTI_SR2_PERM64	{13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-607:2-15-3-13-3:fdff37ff-ef7f3f7d-ff777b7d-7ff7fb2f"
+
+#endif /* SFMT_PARAMS607_H */

test/include/test/SFMT-params86243.h

@@ -0,0 +1,81 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef SFMT_PARAMS86243_H
+#define SFMT_PARAMS86243_H
+
+#define POS1	366
+#define SL1	6
+#define SL2	7
+#define SR1	19
+#define SR2	1
+#define MSK1	0xfdbffbffU
+#define MSK2	0xbff7ff3fU
+#define MSK3	0xfd77efffU
+#define MSK4	0xbf9ff3ffU
+#define PARITY1	0x00000001U
+#define PARITY2	0x00000000U
+#define PARITY3	0x00000000U
+#define PARITY4	0xe9528d85U
+
+
+/* PARAMETERS FOR ALTIVEC */
+#if defined(__APPLE__)	/* For OSX */
+    #define ALTI_SL1	(vector unsigned int)(SL1, SL1, SL1, SL1)
+    #define ALTI_SR1	(vector unsigned int)(SR1, SR1, SR1, SR1)
+    #define ALTI_MSK	(vector unsigned int)(MSK1, MSK2, MSK3, MSK4)
+    #define ALTI_MSK64 \
+	(vector unsigned int)(MSK2, MSK1, MSK4, MSK3)
+    #define ALTI_SL2_PERM \
+	(vector unsigned char)(25,25,25,25,3,25,25,25,7,0,1,2,11,4,5,6)
+    #define ALTI_SL2_PERM64 \
+	(vector unsigned char)(7,25,25,25,25,25,25,25,15,0,1,2,3,4,5,6)
+    #define ALTI_SR2_PERM \
+	(vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14)
+    #define ALTI_SR2_PERM64 \
+	(vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14)
+#else	/* For OTHER OSs(Linux?) */
+    #define ALTI_SL1	{SL1, SL1, SL1, SL1}
+    #define ALTI_SR1	{SR1, SR1, SR1, SR1}
+    #define ALTI_MSK	{MSK1, MSK2, MSK3, MSK4}
+    #define ALTI_MSK64	{MSK2, MSK1, MSK4, MSK3}
+    #define ALTI_SL2_PERM	{25,25,25,25,3,25,25,25,7,0,1,2,11,4,5,6}
+    #define ALTI_SL2_PERM64	{7,25,25,25,25,25,25,25,15,0,1,2,3,4,5,6}
+    #define ALTI_SR2_PERM	{7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14}
+    #define ALTI_SR2_PERM64	{15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14}
+#endif	/* For OSX */
+#define IDSTR	"SFMT-86243:366-6-7-19-1:fdbffbff-bff7ff3f-fd77efff-bf9ff3ff"
+
+#endif /* SFMT_PARAMS86243_H */

test/include/test/SFMT-sse2.h

@@ -0,0 +1,157 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/** 
+ * @file  SFMT-sse2.h
+ * @brief SIMD oriented Fast Mersenne Twister(SFMT) for Intel SSE2
+ *
+ * @author Mutsuo Saito (Hiroshima University)
+ * @author Makoto Matsumoto (Hiroshima University)
+ *
+ * @note We assume LITTLE ENDIAN in this file
+ *
+ * Copyright (C) 2006, 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ * University. All rights reserved.
+ *
+ * The new BSD License is applied to this software, see LICENSE.txt
+ */
+
+#ifndef SFMT_SSE2_H
+#define SFMT_SSE2_H
+
+/**
+ * This function represents the recursion formula.
+ * @param a a 128-bit part of the interal state array
+ * @param b a 128-bit part of the interal state array
+ * @param c a 128-bit part of the interal state array
+ * @param d a 128-bit part of the interal state array
+ * @param mask 128-bit mask
+ * @return output
+ */
+JEMALLOC_ALWAYS_INLINE __m128i mm_recursion(__m128i *a, __m128i *b, 
+				   __m128i c, __m128i d, __m128i mask) {
+    __m128i v, x, y, z;
+    
+    x = _mm_load_si128(a);
+    y = _mm_srli_epi32(*b, SR1);
+    z = _mm_srli_si128(c, SR2);
+    v = _mm_slli_epi32(d, SL1);
+    z = _mm_xor_si128(z, x);
+    z = _mm_xor_si128(z, v);
+    x = _mm_slli_si128(x, SL2);
+    y = _mm_and_si128(y, mask);
+    z = _mm_xor_si128(z, x);
+    z = _mm_xor_si128(z, y);
+    return z;
+}
+
+/**
+ * This function fills the internal state array with pseudorandom
+ * integers.
+ */
+JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) {
+    int i;
+    __m128i r, r1, r2, mask;
+    mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1);
+
+    r1 = _mm_load_si128(&ctx->sfmt[N - 2].si);
+    r2 = _mm_load_si128(&ctx->sfmt[N - 1].si);
+    for (i = 0; i < N - POS1; i++) {
+	r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1].si, r1, r2,
+	  mask);
+	_mm_store_si128(&ctx->sfmt[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+    for (; i < N; i++) {
+	r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1 - N].si, r1, r2,
+	  mask);
+	_mm_store_si128(&ctx->sfmt[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+}
+
+/**
+ * This function fills the user-specified array with pseudorandom
+ * integers.
+ *
+ * @param array an 128-bit array to be filled by pseudorandom numbers.  
+ * @param size number of 128-bit pesudorandom numbers to be generated.
+ */
+JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
+    int i, j;
+    __m128i r, r1, r2, mask;
+    mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1);
+
+    r1 = _mm_load_si128(&ctx->sfmt[N - 2].si);
+    r2 = _mm_load_si128(&ctx->sfmt[N - 1].si);
+    for (i = 0; i < N - POS1; i++) {
+	r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1].si, r1, r2,
+	  mask);
+	_mm_store_si128(&array[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+    for (; i < N; i++) {
+	r = mm_recursion(&ctx->sfmt[i].si, &array[i + POS1 - N].si, r1, r2,
+	  mask);
+	_mm_store_si128(&array[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+    /* main loop */
+    for (; i < size - N; i++) {
+	r = mm_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2,
+			 mask);
+	_mm_store_si128(&array[i].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+    for (j = 0; j < 2 * N - size; j++) {
+	r = _mm_load_si128(&array[j + size - N].si);
+	_mm_store_si128(&ctx->sfmt[j].si, r);
+    }
+    for (; i < size; i++) {
+	r = mm_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2,
+			 mask);
+	_mm_store_si128(&array[i].si, r);
+	_mm_store_si128(&ctx->sfmt[j++].si, r);
+	r1 = r2;
+	r2 = r;
+    }
+}
+
+#endif

test/include/test/SFMT.h

@@ -0,0 +1,171 @@
+/*
+ * This file derives from SFMT 1.3.3
+ * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was
+ * released under the terms of the following license:
+ *
+ *   Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ *   University. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions are
+ *   met:
+ *
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials provided
+ *         with the distribution.
+ *       * Neither the name of the Hiroshima University nor the names of
+ *         its contributors may be used to endorse or promote products
+ *         derived from this software without specific prior written
+ *         permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/** 
+ * @file SFMT.h 
+ *
+ * @brief SIMD oriented Fast Mersenne Twister(SFMT) pseudorandom
+ * number generator
+ *
+ * @author Mutsuo Saito (Hiroshima University)
+ * @author Makoto Matsumoto (Hiroshima University)
+ *
+ * Copyright (C) 2006, 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ * University. All rights reserved.
+ *
+ * The new BSD License is applied to this software.
+ * see LICENSE.txt
+ *
+ * @note We assume that your system has inttypes.h.  If your system
+ * doesn't have inttypes.h, you have to typedef uint32_t and uint64_t,
+ * and you have to define PRIu64 and PRIx64 in this file as follows:
+ * @verbatim
+ typedef unsigned int uint32_t
+ typedef unsigned long long uint64_t  
+ #define PRIu64 "llu"
+ #define PRIx64 "llx"
+@endverbatim
+ * uint32_t must be exactly 32-bit unsigned integer type (no more, no
+ * less), and uint64_t must be exactly 64-bit unsigned integer type.
+ * PRIu64 and PRIx64 are used for printf function to print 64-bit
+ * unsigned int and 64-bit unsigned int in hexadecimal format.
+ */
+
+#ifndef SFMT_H
+#define SFMT_H
+
+typedef struct sfmt_s sfmt_t;
+
+uint32_t gen_rand32(sfmt_t *ctx);
+uint32_t gen_rand32_range(sfmt_t *ctx, uint32_t limit);
+uint64_t gen_rand64(sfmt_t *ctx);
+uint64_t gen_rand64_range(sfmt_t *ctx, uint64_t limit);
+void fill_array32(sfmt_t *ctx, uint32_t *array, int size);
+void fill_array64(sfmt_t *ctx, uint64_t *array, int size);
+sfmt_t *init_gen_rand(uint32_t seed);
+sfmt_t *init_by_array(uint32_t *init_key, int key_length);
+void fini_gen_rand(sfmt_t *ctx);
+const char *get_idstring(void);
+int get_min_array_size32(void);
+int get_min_array_size64(void);
+
+#ifndef JEMALLOC_ENABLE_INLINE
+double to_real1(uint32_t v);
+double genrand_real1(sfmt_t *ctx);
+double to_real2(uint32_t v);
+double genrand_real2(sfmt_t *ctx);
+double to_real3(uint32_t v);
+double genrand_real3(sfmt_t *ctx);
+double to_res53(uint64_t v);
+double to_res53_mix(uint32_t x, uint32_t y);
+double genrand_res53(sfmt_t *ctx);
+double genrand_res53_mix(sfmt_t *ctx);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(SFMT_C_))
+/* These real versions are due to Isaku Wada */
+/** generates a random number on [0,1]-real-interval */
+JEMALLOC_INLINE double to_real1(uint32_t v)
+{
+    return v * (1.0/4294967295.0); 
+    /* divided by 2^32-1 */ 
+}
+
+/** generates a random number on [0,1]-real-interval */
+JEMALLOC_INLINE double genrand_real1(sfmt_t *ctx)
+{
+    return to_real1(gen_rand32(ctx));
+}
+
+/** generates a random number on [0,1)-real-interval */
+JEMALLOC_INLINE double to_real2(uint32_t v)
+{
+    return v * (1.0/4294967296.0); 
+    /* divided by 2^32 */
+}
+
+/** generates a random number on [0,1)-real-interval */
+JEMALLOC_INLINE double genrand_real2(sfmt_t *ctx)
+{
+    return to_real2(gen_rand32(ctx));
+}
+
+/** generates a random number on (0,1)-real-interval */
+JEMALLOC_INLINE double to_real3(uint32_t v)
+{
+    return (((double)v) + 0.5)*(1.0/4294967296.0); 
+    /* divided by 2^32 */
+}
+
+/** generates a random number on (0,1)-real-interval */
+JEMALLOC_INLINE double genrand_real3(sfmt_t *ctx)
+{
+    return to_real3(gen_rand32(ctx));
+}
+/** These real versions are due to Isaku Wada */
+
+/** generates a random number on [0,1) with 53-bit resolution*/
+JEMALLOC_INLINE double to_res53(uint64_t v) 
+{ 
+    return v * (1.0/18446744073709551616.0L);
+}
+
+/** generates a random number on [0,1) with 53-bit resolution from two
+ * 32 bit integers */
+JEMALLOC_INLINE double to_res53_mix(uint32_t x, uint32_t y) 
+{ 
+    return to_res53(x | ((uint64_t)y << 32));
+}
+
+/** generates a random number on [0,1) with 53-bit resolution
+ */
+JEMALLOC_INLINE double genrand_res53(sfmt_t *ctx) 
+{ 
+    return to_res53(gen_rand64(ctx));
+} 
+
+/** generates a random number on [0,1) with 53-bit resolution
+    using 32bit integer.
+ */
+JEMALLOC_INLINE double genrand_res53_mix(sfmt_t *ctx) 
+{ 
+    uint32_t x, y;
+
+    x = gen_rand32(ctx);
+    y = gen_rand32(ctx);
+    return to_res53_mix(x, y);
+} 
+#endif
+#endif

test/include/test/jemalloc_test.h.in

@@ -0,0 +1,141 @@
+#include <stdlib.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <math.h>
+#include <string.h>
+
+#ifdef _WIN32
+#  include <windows.h>
+#else
+#  include <pthread.h>
+#endif
+
+/******************************************************************************/
+/*
+ * Define always-enabled assertion macros, so that test assertions execute even
+ * if assertions are disabled in the library code.  These definitions must
+ * exist prior to including "jemalloc/internal/util.h".
+ */
+#define	assert(e) do {							\
+	if (!(e)) {							\
+		malloc_printf(						\
+		    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n",	\
+		    __FILE__, __LINE__, #e);				\
+		abort();						\
+	}								\
+} while (0)
+
+#define	not_reached() do {						\
+	malloc_printf(							\
+	    "<jemalloc>: %s:%d: Unreachable code reached\n",		\
+	    __FILE__, __LINE__);					\
+	abort();							\
+} while (0)
+
+#define	not_implemented() do {						\
+	malloc_printf("<jemalloc>: %s:%d: Not implemented\n",		\
+	    __FILE__, __LINE__);					\
+	abort();							\
+} while (0)
+
+#define	assert_not_implemented(e) do {					\
+	if (!(e))							\
+		not_implemented();					\
+} while (0)
+
+#include "test/jemalloc_test_defs.h"
+
+#ifdef JEMALLOC_OSSPIN
+#  include <libkern/OSAtomic.h>
+#endif
+
+#if defined(HAVE_ALTIVEC) && !defined(__APPLE__)
+#  include <altivec.h>
+#endif
+#ifdef HAVE_SSE2
+#  include <emmintrin.h>
+#endif
+
+/******************************************************************************/
+/*
+ * For unit tests, expose all public and private interfaces.
+ */
+#ifdef JEMALLOC_UNIT_TEST
+#  define JEMALLOC_JET
+#  define JEMALLOC_MANGLE
+#  include "jemalloc/internal/jemalloc_internal.h"
+
+/******************************************************************************/
+/*
+ * For integration tests, expose the public jemalloc interfaces, but only
+ * expose the minimum necessary internal utility code (to avoid re-implementing
+ * essentially identical code within the test infrastructure).
+ */
+#elif defined(JEMALLOC_INTEGRATION_TEST)
+#  define JEMALLOC_MANGLE
+#  include "jemalloc/jemalloc@install_suffix@.h"
+#  include "jemalloc/internal/jemalloc_internal_defs.h"
+#  include "jemalloc/internal/jemalloc_internal_macros.h"
+
+#  define JEMALLOC_N(n) @private_namespace@##n
+#  include "jemalloc/internal/private_namespace.h"
+
+#  define JEMALLOC_H_TYPES
+#  define JEMALLOC_H_STRUCTS
+#  define JEMALLOC_H_EXTERNS
+#  define JEMALLOC_H_INLINES
+#  include "jemalloc/internal/util.h"
+#  include "jemalloc/internal/qr.h"
+#  include "jemalloc/internal/ql.h"
+#  undef JEMALLOC_H_TYPES
+#  undef JEMALLOC_H_STRUCTS
+#  undef JEMALLOC_H_EXTERNS
+#  undef JEMALLOC_H_INLINES
+
+/******************************************************************************/
+/*
+ * For stress tests, expose the public jemalloc interfaces with name mangling
+ * so that they can be tested as e.g. malloc() and free().  Also expose the
+ * public jemalloc interfaces with jet_ prefixes, so that stress tests can use
+ * a separate allocator for their internal data structures.
+ */
+#elif defined(JEMALLOC_STRESS_TEST)
+#  include "jemalloc/jemalloc@install_suffix@.h"
+
+#  include "jemalloc/jemalloc_protos_jet.h"
+
+#  define JEMALLOC_JET
+#  include "jemalloc/internal/jemalloc_internal.h"
+#  include "jemalloc/internal/public_unnamespace.h"
+#  undef JEMALLOC_JET
+
+#  include "jemalloc/jemalloc_rename.h"
+#  define JEMALLOC_MANGLE
+#  ifdef JEMALLOC_STRESS_TESTLIB
+#    include "jemalloc/jemalloc_mangle_jet.h"
+#  else
+#    include "jemalloc/jemalloc_mangle.h"
+#  endif
+
+/******************************************************************************/
+/*
+ * This header does dangerous things, the effects of which only test code
+ * should be subject to.
+ */
+#else
+#  error "This header cannot be included outside a testing context"
+#endif
+
+/******************************************************************************/
+/*
+ * Common test utilities.
+ */
+#include "test/math.h"
+#include "test/mtx.h"
+#include "test/mq.h"
+#include "test/test.h"
+#include "test/thd.h"
+#define	MEXP 19937
+#include "test/SFMT.h"

test/include/test/jemalloc_test_defs.h.in

@@ -0,0 +1,5 @@
+#include "jemalloc/internal/jemalloc_internal_defs.h"
+
+/* For use by SFMT. */
+#undef HAVE_SSE2
+#undef HAVE_ALTIVEC

test/include/test/math.h

@@ -0,0 +1,311 @@
+#ifndef JEMALLOC_ENABLE_INLINE
+double	ln_gamma(double x);
+double	i_gamma(double x, double p, double ln_gamma_p);
+double	pt_norm(double p);
+double	pt_chi2(double p, double df, double ln_gamma_df_2);
+double	pt_gamma(double p, double shape, double scale, double ln_gamma_shape);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(MATH_C_))
+/*
+ * Compute the natural log of Gamma(x), accurate to 10 decimal places.
+ *
+ * This implementation is based on:
+ *
+ *   Pike, M.C., I.D. Hill (1966) Algorithm 291: Logarithm of Gamma function
+ *   [S14].  Communications of the ACM 9(9):684.
+ */
+JEMALLOC_INLINE double
+ln_gamma(double x)
+{
+	double f, z;
+
+	assert(x > 0.0);
+
+	if (x < 7.0) {
+		f = 1.0;
+		z = x;
+		while (z < 7.0) {
+			f *= z;
+			z += 1.0;
+		}
+		x = z;
+		f = -log(f);
+	} else
+		f = 0.0;
+
+	z = 1.0 / (x * x);
+
+	return (f + (x-0.5) * log(x) - x + 0.918938533204673 +
+	    (((-0.000595238095238 * z + 0.000793650793651) * z -
+	    0.002777777777778) * z + 0.083333333333333) / x);
+}
+
+/*
+ * Compute the incomplete Gamma ratio for [0..x], where p is the shape
+ * parameter, and ln_gamma_p is ln_gamma(p).
+ *
+ * This implementation is based on:
+ *
+ *   Bhattacharjee, G.P. (1970) Algorithm AS 32: The incomplete Gamma integral.
+ *   Applied Statistics 19:285-287.
+ */
+JEMALLOC_INLINE double
+i_gamma(double x, double p, double ln_gamma_p)
+{
+	double acu, factor, oflo, gin, term, rn, a, b, an, dif;
+	double pn[6];
+	unsigned i;
+
+	assert(p > 0.0);
+	assert(x >= 0.0);
+
+	if (x == 0.0)
+		return (0.0);
+
+	acu = 1.0e-10;
+	oflo = 1.0e30;
+	gin = 0.0;
+	factor = exp(p * log(x) - x - ln_gamma_p);
+
+	if (x <= 1.0 || x < p) {
+		/* Calculation by series expansion. */
+		gin = 1.0;
+		term = 1.0;
+		rn = p;
+
+		while (true) {
+			rn += 1.0;
+			term *= x / rn;
+			gin += term;
+			if (term <= acu) {
+				gin *= factor / p;
+				return (gin);
+			}
+		}
+	} else {
+		/* Calculation by continued fraction. */
+		a = 1.0 - p;
+		b = a + x + 1.0;
+		term = 0.0;
+		pn[0] = 1.0;
+		pn[1] = x;
+		pn[2] = x + 1.0;
+		pn[3] = x * b;
+		gin = pn[2] / pn[3];
+
+		while (true) {
+			a += 1.0;
+			b += 2.0;
+			term += 1.0;
+			an = a * term;
+			for (i = 0; i < 2; i++)
+				pn[i+4] = b * pn[i+2] - an * pn[i];
+			if (pn[5] != 0.0) {
+				rn = pn[4] / pn[5];
+				dif = fabs(gin - rn);
+				if (dif <= acu && dif <= acu * rn) {
+					gin = 1.0 - factor * gin;
+					return (gin);
+				}
+				gin = rn;
+			}
+			for (i = 0; i < 4; i++)
+				pn[i] = pn[i+2];
+
+			if (fabs(pn[4]) >= oflo) {
+				for (i = 0; i < 4; i++)
+					pn[i] /= oflo;
+			}
+		}
+	}
+}
+
+/*
+ * Given a value p in [0..1] of the lower tail area of the normal distribution,
+ * compute the limit on the definite integral from [-inf..z] that satisfies p,
+ * accurate to 16 decimal places.
+ *
+ * This implementation is based on:
+ *
+ *   Wichura, M.J. (1988) Algorithm AS 241: The percentage points of the normal
+ *   distribution.  Applied Statistics 37(3):477-484.
+ */
+JEMALLOC_INLINE double
+pt_norm(double p)
+{
+	double q, r, ret;
+
+	assert(p > 0.0 && p < 1.0);
+
+	q = p - 0.5;
+	if (fabs(q) <= 0.425) {
+		/* p close to 1/2. */
+		r = 0.180625 - q * q;
+		return (q * (((((((2.5090809287301226727e3 * r +
+		    3.3430575583588128105e4) * r + 6.7265770927008700853e4) * r
+		    + 4.5921953931549871457e4) * r + 1.3731693765509461125e4) *
+		    r + 1.9715909503065514427e3) * r + 1.3314166789178437745e2)
+		    * r + 3.3871328727963666080e0) /
+		    (((((((5.2264952788528545610e3 * r +
+		    2.8729085735721942674e4) * r + 3.9307895800092710610e4) * r
+		    + 2.1213794301586595867e4) * r + 5.3941960214247511077e3) *
+		    r + 6.8718700749205790830e2) * r + 4.2313330701600911252e1)
+		    * r + 1.0));
+	} else {
+		if (q < 0.0)
+			r = p;
+		else
+			r = 1.0 - p;
+		assert(r > 0.0);
+
+		r = sqrt(-log(r));
+		if (r <= 5.0) {
+			/* p neither close to 1/2 nor 0 or 1. */
+			r -= 1.6;
+			ret = ((((((((7.74545014278341407640e-4 * r +
+			    2.27238449892691845833e-2) * r +
+			    2.41780725177450611770e-1) * r +
+			    1.27045825245236838258e0) * r +
+			    3.64784832476320460504e0) * r +
+			    5.76949722146069140550e0) * r +
+			    4.63033784615654529590e0) * r +
+			    1.42343711074968357734e0) /
+			    (((((((1.05075007164441684324e-9 * r +
+			    5.47593808499534494600e-4) * r +
+			    1.51986665636164571966e-2)
+			    * r + 1.48103976427480074590e-1) * r +
+			    6.89767334985100004550e-1) * r +
+			    1.67638483018380384940e0) * r +
+			    2.05319162663775882187e0) * r + 1.0));
+		} else {
+			/* p near 0 or 1. */
+			r -= 5.0;
+			ret = ((((((((2.01033439929228813265e-7 * r +
+			    2.71155556874348757815e-5) * r +
+			    1.24266094738807843860e-3) * r +
+			    2.65321895265761230930e-2) * r +
+			    2.96560571828504891230e-1) * r +
+			    1.78482653991729133580e0) * r +
+			    5.46378491116411436990e0) * r +
+			    6.65790464350110377720e0) /
+			    (((((((2.04426310338993978564e-15 * r +
+			    1.42151175831644588870e-7) * r +
+			    1.84631831751005468180e-5) * r +
+			    7.86869131145613259100e-4) * r +
+			    1.48753612908506148525e-2) * r +
+			    1.36929880922735805310e-1) * r +
+			    5.99832206555887937690e-1)
+			    * r + 1.0));
+		}
+		if (q < 0.0)
+			ret = -ret;
+		return (ret);
+	}
+}
+
+/*
+ * Given a value p in [0..1] of the lower tail area of the Chi^2 distribution
+ * with df degrees of freedom, where ln_gamma_df_2 is ln_gamma(df/2.0), compute
+ * the upper limit on the definite integral from [0..z] that satisfies p,
+ * accurate to 12 decimal places.
+ *
+ * This implementation is based on:
+ *
+ *   Best, D.J., D.E. Roberts (1975) Algorithm AS 91: The percentage points of
+ *   the Chi^2 distribution.  Applied Statistics 24(3):385-388.
+ *
+ *   Shea, B.L. (1991) Algorithm AS R85: A remark on AS 91: The percentage
+ *   points of the Chi^2 distribution.  Applied Statistics 40(1):233-235.
+ */
+JEMALLOC_INLINE double
+pt_chi2(double p, double df, double ln_gamma_df_2)
+{
+	double e, aa, xx, c, ch, a, q, p1, p2, t, x, b, s1, s2, s3, s4, s5, s6;
+	unsigned i;
+
+	assert(p >= 0.0 && p < 1.0);
+	assert(df > 0.0);
+
+	e = 5.0e-7;
+	aa = 0.6931471805;
+
+	xx = 0.5 * df;
+	c = xx - 1.0;
+
+	if (df < -1.24 * log(p)) {
+		/* Starting approximation for small Chi^2. */
+		ch = pow(p * xx * exp(ln_gamma_df_2 + xx * aa), 1.0 / xx);
+		if (ch - e < 0.0)
+			return (ch);
+	} else {
+		if (df > 0.32) {
+			x = pt_norm(p);
+			/*
+			 * Starting approximation using Wilson and Hilferty
+			 * estimate.
+			 */
+			p1 = 0.222222 / df;
+			ch = df * pow(x * sqrt(p1) + 1.0 - p1, 3.0);
+			/* Starting approximation for p tending to 1. */
+			if (ch > 2.2 * df + 6.0) {
+				ch = -2.0 * (log(1.0 - p) - c * log(0.5 * ch) +
+				    ln_gamma_df_2);
+			}
+		} else {
+			ch = 0.4;
+			a = log(1.0 - p);
+			while (true) {
+				q = ch;
+				p1 = 1.0 + ch * (4.67 + ch);
+				p2 = ch * (6.73 + ch * (6.66 + ch));
+				t = -0.5 + (4.67 + 2.0 * ch) / p1 - (6.73 + ch
+				    * (13.32 + 3.0 * ch)) / p2;
+				ch -= (1.0 - exp(a + ln_gamma_df_2 + 0.5 * ch +
+				    c * aa) * p2 / p1) / t;
+				if (fabs(q / ch - 1.0) - 0.01 <= 0.0)
+					break;
+			}
+		}
+	}
+
+	for (i = 0; i < 20; i++) {
+		/* Calculation of seven-term Taylor series. */
+		q = ch;
+		p1 = 0.5 * ch;
+		if (p1 < 0.0)
+			return (-1.0);
+		p2 = p - i_gamma(p1, xx, ln_gamma_df_2);
+		t = p2 * exp(xx * aa + ln_gamma_df_2 + p1 - c * log(ch));
+		b = t / ch;
+		a = 0.5 * t - b * c;
+		s1 = (210.0 + a * (140.0 + a * (105.0 + a * (84.0 + a * (70.0 +
+		    60.0 * a))))) / 420.0;
+		s2 = (420.0 + a * (735.0 + a * (966.0 + a * (1141.0 + 1278.0 *
+		    a)))) / 2520.0;
+		s3 = (210.0 + a * (462.0 + a * (707.0 + 932.0 * a))) / 2520.0;
+		s4 = (252.0 + a * (672.0 + 1182.0 * a) + c * (294.0 + a *
+		    (889.0 + 1740.0 * a))) / 5040.0;
+		s5 = (84.0 + 264.0 * a + c * (175.0 + 606.0 * a)) / 2520.0;
+		s6 = (120.0 + c * (346.0 + 127.0 * c)) / 5040.0;
+		ch += t * (1.0 + 0.5 * t * s1 - b * c * (s1 - b * (s2 - b * (s3
+		    - b * (s4 - b * (s5 - b * s6))))));
+		if (fabs(q / ch - 1.0) <= e)
+			break;
+	}
+
+	return (ch);
+}
+
+/*
+ * Given a value p in [0..1] and Gamma distribution shape and scale parameters,
+ * compute the upper limit on the definite integeral from [0..z] that satisfies
+ * p.
+ */
+JEMALLOC_INLINE double
+pt_gamma(double p, double shape, double scale, double ln_gamma_shape)
+{
+
+	return (pt_chi2(p, shape * 2.0, ln_gamma_shape) * 0.5 * scale);
+}
+#endif

test/include/test/mq.h

@@ -0,0 +1,110 @@
+/*
+ * Simple templated message queue implementation that relies on only mutexes for
+ * synchronization (which reduces portability issues).  Given the following
+ * setup:
+ *
+ *   typedef struct mq_msg_s mq_msg_t;
+ *   struct mq_msg_s {
+ *           mq_msg(mq_msg_t) link;
+ *           [message data]
+ *   };
+ *   mq_gen(, mq_, mq_t, mq_msg_t, link)
+ *
+ * The API is as follows:
+ *
+ *   bool mq_init(mq_t *mq);
+ *   void mq_fini(mq_t *mq);
+ *   unsigned mq_count(mq_t *mq);
+ *   mq_msg_t *mq_tryget(mq_t *mq);
+ *   mq_msg_t *mq_get(mq_t *mq);
+ *   void mq_put(mq_t *mq, mq_msg_t *msg);
+ *
+ * The message queue linkage embedded in each message is to be treated as
+ * externally opaque (no need to initialize or clean up externally).  mq_fini()
+ * does not perform any cleanup of messages, since it knows nothing of their
+ * payloads.
+ */
+#define	mq_msg(a_mq_msg_type)	ql_elm(a_mq_msg_type)
+
+#define	mq_gen(a_attr, a_prefix, a_mq_type, a_mq_msg_type, a_field)	\
+typedef struct {							\
+	mtx_t			lock;					\
+	ql_head(a_mq_msg_type)	msgs;					\
+	unsigned		count;					\
+} a_mq_type;								\
+a_attr bool								\
+a_prefix##init(a_mq_type *mq) {						\
+									\
+	if (mtx_init(&mq->lock))					\
+		return (true);						\
+	ql_new(&mq->msgs);						\
+	mq->count = 0;							\
+	return (false);							\
+}									\
+a_attr void								\
+a_prefix##fini(a_mq_type *mq)						\
+{									\
+									\
+	mtx_fini(&mq->lock);						\
+}									\
+a_attr unsigned								\
+a_prefix##count(a_mq_type *mq)						\
+{									\
+	unsigned count;							\
+									\
+	mtx_lock(&mq->lock);						\
+	count = mq->count;						\
+	mtx_unlock(&mq->lock);						\
+	return (count);							\
+}									\
+a_attr a_mq_msg_type *							\
+a_prefix##tryget(a_mq_type *mq)						\
+{									\
+	a_mq_msg_type *msg;						\
+									\
+	mtx_lock(&mq->lock);						\
+	msg = ql_first(&mq->msgs);					\
+	if (msg != NULL) {						\
+		ql_head_remove(&mq->msgs, a_mq_msg_type, a_field);	\
+		mq->count--;						\
+	}								\
+	mtx_unlock(&mq->lock);						\
+	return (msg);							\
+}									\
+a_attr a_mq_msg_type *							\
+a_prefix##get(a_mq_type *mq)						\
+{									\
+	a_mq_msg_type *msg;						\
+	struct timespec timeout;					\
+									\
+	msg = a_prefix##tryget(mq);					\
+	if (msg != NULL)						\
+		return (msg);						\
+									\
+	timeout.tv_sec = 0;						\
+	timeout.tv_nsec = 1;						\
+	while (true) {							\
+		nanosleep(&timeout, NULL);				\
+		msg = a_prefix##tryget(mq);				\
+		if (msg != NULL)					\
+			return (msg);					\
+		if (timeout.tv_sec == 0) {				\
+			/* Double sleep time, up to max 1 second. */	\
+			timeout.tv_nsec <<= 1;				\
+			if (timeout.tv_nsec >= 1000*1000*1000) {	\
+				timeout.tv_sec = 1;			\
+				timeout.tv_nsec = 0;			\
+			}						\
+		}							\
+	}								\
+}									\
+a_attr void								\
+a_prefix##put(a_mq_type *mq, a_mq_msg_type *msg)			\
+{									\
+									\
+	mtx_lock(&mq->lock);						\
+	ql_elm_new(msg, a_field);					\
+	ql_tail_insert(&mq->msgs, msg, a_field);			\
+	mq->count++;							\
+	mtx_unlock(&mq->lock);						\
+}

test/include/test/mtx.h

@@ -0,0 +1,21 @@
+/*
+ * mtx is a slightly simplified version of malloc_mutex.  This code duplication
+ * is unfortunate, but there are allocator bootstrapping considerations that
+ * would leak into the test infrastructure if malloc_mutex were used directly
+ * in tests.
+ */
+
+typedef struct {
+#ifdef _WIN32
+	CRITICAL_SECTION	lock;
+#elif (defined(JEMALLOC_OSSPIN))
+	OSSpinLock		lock;
+#else
+	pthread_mutex_t		lock;
+#endif
+} mtx_t;
+
+bool	mtx_init(mtx_t *mtx);
+void	mtx_fini(mtx_t *mtx);
+void	mtx_lock(mtx_t *mtx);
+void	mtx_unlock(mtx_t *mtx);

test/include/test/test.h

@@ -0,0 +1,302 @@
+#define	assert_cmp(t, a, b, cmp, neg_cmp, pri, fmt...) do {		\
+	t a_ = (a);							\
+	t b_ = (b);							\
+	if (!(a_ cmp b_)) {						\
+		p_test_fail(						\
+		    "%s:%s:%d: Failed assertion: "			\
+		    "(%s) "#cmp" (%s) --> "				\
+		    "%"pri" "#neg_cmp" %"pri": ",			\
+		    __func__, __FILE__, __LINE__,			\
+		    #a, #b, a_, b_, fmt);				\
+	}								\
+} while (0)
+
+#define	assert_ptr_eq(a, b, fmt...)	assert_cmp(void *, a, b, ==,	\
+    !=, "p", fmt)
+#define	assert_ptr_ne(a, b, fmt...)	assert_cmp(void *, a, b, !=,	\
+    ==, "p", fmt)
+#define	assert_ptr_null(a, fmt...)	assert_cmp(void *, a, NULL, ==,	\
+    !=, "p", fmt)
+#define	assert_ptr_not_null(a, fmt...)	assert_cmp(void *, a, NULL, !=,	\
+    ==, "p", fmt)
+
+#define	assert_c_eq(a, b, fmt...)	assert_cmp(char, a, b, ==, !=, "c", fmt)
+#define	assert_c_ne(a, b, fmt...)	assert_cmp(char, a, b, !=, ==, "c", fmt)
+#define	assert_c_lt(a, b, fmt...)	assert_cmp(char, a, b, <, >=, "c", fmt)
+#define	assert_c_le(a, b, fmt...)	assert_cmp(char, a, b, <=, >, "c", fmt)
+#define	assert_c_ge(a, b, fmt...)	assert_cmp(char, a, b, >=, <, "c", fmt)
+#define	assert_c_gt(a, b, fmt...)	assert_cmp(char, a, b, >, <=, "c", fmt)
+
+#define	assert_x_eq(a, b, fmt...)	assert_cmp(int, a, b, ==, !=, "#x", fmt)
+#define	assert_x_ne(a, b, fmt...)	assert_cmp(int, a, b, !=, ==, "#x", fmt)
+#define	assert_x_lt(a, b, fmt...)	assert_cmp(int, a, b, <, >=, "#x", fmt)
+#define	assert_x_le(a, b, fmt...)	assert_cmp(int, a, b, <=, >, "#x", fmt)
+#define	assert_x_ge(a, b, fmt...)	assert_cmp(int, a, b, >=, <, "#x", fmt)
+#define	assert_x_gt(a, b, fmt...)	assert_cmp(int, a, b, >, <=, "#x", fmt)
+
+#define	assert_d_eq(a, b, fmt...)	assert_cmp(int, a, b, ==, !=, "d", fmt)
+#define	assert_d_ne(a, b, fmt...)	assert_cmp(int, a, b, !=, ==, "d", fmt)
+#define	assert_d_lt(a, b, fmt...)	assert_cmp(int, a, b, <, >=, "d", fmt)
+#define	assert_d_le(a, b, fmt...)	assert_cmp(int, a, b, <=, >, "d", fmt)
+#define	assert_d_ge(a, b, fmt...)	assert_cmp(int, a, b, >=, <, "d", fmt)
+#define	assert_d_gt(a, b, fmt...)	assert_cmp(int, a, b, >, <=, "d", fmt)
+
+#define	assert_u_eq(a, b, fmt...)	assert_cmp(int, a, b, ==, !=, "u", fmt)
+#define	assert_u_ne(a, b, fmt...)	assert_cmp(int, a, b, !=, ==, "u", fmt)
+#define	assert_u_lt(a, b, fmt...)	assert_cmp(int, a, b, <, >=, "u", fmt)
+#define	assert_u_le(a, b, fmt...)	assert_cmp(int, a, b, <=, >, "u", fmt)
+#define	assert_u_ge(a, b, fmt...)	assert_cmp(int, a, b, >=, <, "u", fmt)
+#define	assert_u_gt(a, b, fmt...)	assert_cmp(int, a, b, >, <=, "u", fmt)
+
+#define	assert_ld_eq(a, b, fmt...)	assert_cmp(long, a, b, ==,	\
+    !=, "ld", fmt)
+#define	assert_ld_ne(a, b, fmt...)	assert_cmp(long, a, b, !=,	\
+    ==, "ld", fmt)
+#define	assert_ld_lt(a, b, fmt...)	assert_cmp(long, a, b, <,	\
+    >=, "ld", fmt)
+#define	assert_ld_le(a, b, fmt...)	assert_cmp(long, a, b, <=,	\
+    >, "ld", fmt)
+#define	assert_ld_ge(a, b, fmt...)	assert_cmp(long, a, b, >=,	\
+    <, "ld", fmt)
+#define	assert_ld_gt(a, b, fmt...)	assert_cmp(long, a, b, >,	\
+    <=, "ld", fmt)
+
+#define	assert_lu_eq(a, b, fmt...)	assert_cmp(unsigned long,	\
+    a, b, ==, !=, "lu", fmt)
+#define	assert_lu_ne(a, b, fmt...)	assert_cmp(unsigned long,	\
+    a, b, !=, ==, "lu", fmt)
+#define	assert_lu_lt(a, b, fmt...)	assert_cmp(unsigned long,	\
+    a, b, <, >=, "lu", fmt)
+#define	assert_lu_le(a, b, fmt...)	assert_cmp(unsigned long,	\
+    a, b, <=, >, "lu", fmt)
+#define	assert_lu_ge(a, b, fmt...)	assert_cmp(unsigned long,	\
+    a, b, >=, <, "lu", fmt)
+#define	assert_lu_gt(a, b, fmt...)	assert_cmp(unsigned long,	\
+    a, b, >, <=, "lu", fmt)
+
+#define	assert_qd_eq(a, b, fmt...)	assert_cmp(long long, a, b, ==,	\
+    !=, "qd", fmt)
+#define	assert_qd_ne(a, b, fmt...)	assert_cmp(long long, a, b, !=,	\
+    ==, "qd", fmt)
+#define	assert_qd_lt(a, b, fmt...)	assert_cmp(long long, a, b, <,	\
+    >=, "qd", fmt)
+#define	assert_qd_le(a, b, fmt...)	assert_cmp(long long, a, b, <=,	\
+    >, "qd", fmt)
+#define	assert_qd_ge(a, b, fmt...)	assert_cmp(long long, a, b, >=,	\
+    <, "qd", fmt)
+#define	assert_qd_gt(a, b, fmt...)	assert_cmp(long long, a, b, >,	\
+    <=, "qd", fmt)
+
+#define	assert_qu_eq(a, b, fmt...)	assert_cmp(unsigned long long,	\
+    a, b, ==, !=, "qu", fmt)
+#define	assert_qu_ne(a, b, fmt...)	assert_cmp(unsigned long long,	\
+    a, b, !=, ==, "qu", fmt)
+#define	assert_qu_lt(a, b, fmt...)	assert_cmp(unsigned long long,	\
+    a, b, <, >=, "qu", fmt)
+#define	assert_qu_le(a, b, fmt...)	assert_cmp(unsigned long long,	\
+    a, b, <=, >, "qu", fmt)
+#define	assert_qu_ge(a, b, fmt...)	assert_cmp(unsigned long long,	\
+    a, b, >=, <, "qu", fmt)
+#define	assert_qu_gt(a, b, fmt...)	assert_cmp(unsigned long long,	\
+    a, b, >, <=, "qu", fmt)
+
+#define	assert_jd_eq(a, b, fmt...)	assert_cmp(intmax_t, a, b, ==,	\
+    !=, "jd", fmt)
+#define	assert_jd_ne(a, b, fmt...)	assert_cmp(intmax_t, a, b, !=,	\
+    ==, "jd", fmt)
+#define	assert_jd_lt(a, b, fmt...)	assert_cmp(intmax_t, a, b, <,	\
+    >=, "jd", fmt)
+#define	assert_jd_le(a, b, fmt...)	assert_cmp(intmax_t, a, b, <=,	\
+    >, "jd", fmt)
+#define	assert_jd_ge(a, b, fmt...)	assert_cmp(intmax_t, a, b, >=,	\
+    <, "jd", fmt)
+#define	assert_jd_gt(a, b, fmt...)	assert_cmp(intmax_t, a, b, >,	\
+    <=, "jd", fmt)
+
+#define	assert_ju_eq(a, b, fmt...)	assert_cmp(uintmax_t, a, b, ==,	\
+    !=, "ju", fmt)
+#define	assert_ju_ne(a, b, fmt...)	assert_cmp(uintmax_t, a, b, !=,	\
+    ==, "ju", fmt)
+#define	assert_ju_lt(a, b, fmt...)	assert_cmp(uintmax_t, a, b, <,	\
+    >=, "ju", fmt)
+#define	assert_ju_le(a, b, fmt...)	assert_cmp(uintmax_t, a, b, <=,	\
+    >, "ju", fmt)
+#define	assert_ju_ge(a, b, fmt...)	assert_cmp(uintmax_t, a, b, >=,	\
+    <, "ju", fmt)
+#define	assert_ju_gt(a, b, fmt...)	assert_cmp(uintmax_t, a, b, >,	\
+    <=, "ju", fmt)
+
+#define	assert_zd_eq(a, b, fmt...)	assert_cmp(ssize_t, a, b, ==,	\
+    !=, "zd", fmt)
+#define	assert_zd_ne(a, b, fmt...)	assert_cmp(ssize_t, a, b, !=,	\
+    ==, "zd", fmt)
+#define	assert_zd_lt(a, b, fmt...)	assert_cmp(ssize_t, a, b, <,	\
+    >=, "zd", fmt)
+#define	assert_zd_le(a, b, fmt...)	assert_cmp(ssize_t, a, b, <=,	\
+    >, "zd", fmt)
+#define	assert_zd_ge(a, b, fmt...)	assert_cmp(ssize_t, a, b, >=,	\
+    <, "zd", fmt)
+#define	assert_zd_gt(a, b, fmt...)	assert_cmp(ssize_t, a, b, >,	\
+    <=, "zd", fmt)
+
+#define	assert_zu_eq(a, b, fmt...)	assert_cmp(size_t, a, b, ==,	\
+    !=, "zu", fmt)
+#define	assert_zu_ne(a, b, fmt...)	assert_cmp(size_t, a, b, !=,	\
+    ==, "zu", fmt)
+#define	assert_zu_lt(a, b, fmt...)	assert_cmp(size_t, a, b, <,	\
+    >=, "zu", fmt)
+#define	assert_zu_le(a, b, fmt...)	assert_cmp(size_t, a, b, <=,	\
+    >, "zu", fmt)
+#define	assert_zu_ge(a, b, fmt...)	assert_cmp(size_t, a, b, >=,	\
+    <, "zu", fmt)
+#define	assert_zu_gt(a, b, fmt...)	assert_cmp(size_t, a, b, >,	\
+    <=, "zu", fmt)
+
+#define	assert_d32_eq(a, b, fmt...)	assert_cmp(int32_t, a, b, ==,	\
+    !=, PRId32, fmt)
+#define	assert_d32_ne(a, b, fmt...)	assert_cmp(int32_t, a, b, !=,	\
+    ==, PRId32, fmt)
+#define	assert_d32_lt(a, b, fmt...)	assert_cmp(int32_t, a, b, <,	\
+    >=, PRId32, fmt)
+#define	assert_d32_le(a, b, fmt...)	assert_cmp(int32_t, a, b, <=,	\
+    >, PRId32, fmt)
+#define	assert_d32_ge(a, b, fmt...)	assert_cmp(int32_t, a, b, >=,	\
+    <, PRId32, fmt)
+#define	assert_d32_gt(a, b, fmt...)	assert_cmp(int32_t, a, b, >,	\
+    <=, PRId32, fmt)
+
+#define	assert_u32_eq(a, b, fmt...)	assert_cmp(uint32_t, a, b, ==,	\
+    !=, PRIu32, fmt)
+#define	assert_u32_ne(a, b, fmt...)	assert_cmp(uint32_t, a, b, !=,	\
+    ==, PRIu32, fmt)
+#define	assert_u32_lt(a, b, fmt...)	assert_cmp(uint32_t, a, b, <,	\
+    >=, PRIu32, fmt)
+#define	assert_u32_le(a, b, fmt...)	assert_cmp(uint32_t, a, b, <=,	\
+    >, PRIu32, fmt)
+#define	assert_u32_ge(a, b, fmt...)	assert_cmp(uint32_t, a, b, >=,	\
+    <, PRIu32, fmt)
+#define	assert_u32_gt(a, b, fmt...)	assert_cmp(uint32_t, a, b, >,	\
+    <=, PRIu32, fmt)
+
+#define	assert_d64_eq(a, b, fmt...)	assert_cmp(int64_t, a, b, ==,	\
+    !=, PRId64, fmt)
+#define	assert_d64_ne(a, b, fmt...)	assert_cmp(int64_t, a, b, !=,	\
+    ==, PRId64, fmt)
+#define	assert_d64_lt(a, b, fmt...)	assert_cmp(int64_t, a, b, <,	\
+    >=, PRId64, fmt)
+#define	assert_d64_le(a, b, fmt...)	assert_cmp(int64_t, a, b, <=,	\
+    >, PRId64, fmt)
+#define	assert_d64_ge(a, b, fmt...)	assert_cmp(int64_t, a, b, >=,	\
+    <, PRId64, fmt)
+#define	assert_d64_gt(a, b, fmt...)	assert_cmp(int64_t, a, b, >,	\
+    <=, PRId64, fmt)
+
+#define	assert_u64_eq(a, b, fmt...)	assert_cmp(uint64_t, a, b, ==,	\
+    !=, PRIu64, fmt)
+#define	assert_u64_ne(a, b, fmt...)	assert_cmp(uint64_t, a, b, !=,	\
+    ==, PRIu64, fmt)
+#define	assert_u64_lt(a, b, fmt...)	assert_cmp(uint64_t, a, b, <,	\
+    >=, PRIu64, fmt)
+#define	assert_u64_le(a, b, fmt...)	assert_cmp(uint64_t, a, b, <=,	\
+    >, PRIu64, fmt)
+#define	assert_u64_ge(a, b, fmt...)	assert_cmp(uint64_t, a, b, >=,	\
+    <, PRIu64, fmt)
+#define	assert_u64_gt(a, b, fmt...)	assert_cmp(uint64_t, a, b, >,	\
+    <=, PRIu64, fmt)
+
+#define	assert_b_eq(a, b, fmt...) do {					\
+	bool a_ = (a);							\
+	bool b_ = (b);							\
+	if (!(a_ == b_)) {						\
+		p_test_fail(						\
+		    "%s:%s:%d: Failed assertion: "			\
+		    "(%s) == (%s) --> %s != %s: ",			\
+		    __func__, __FILE__, __LINE__,			\
+		    #a, #b, a_ ? "true" : "false",			\
+		    b_ ? "true" : "false", fmt);			\
+	}								\
+} while (0)
+#define	assert_b_ne(a, b, fmt...) do {					\
+	bool a_ = (a);							\
+	bool b_ = (b);							\
+	if (!(a_ != b_)) {						\
+		p_test_fail(						\
+		    "%s:%s:%d: Failed assertion: "			\
+		    "(%s) != (%s) --> %s == %s: ",			\
+		    __func__, __FILE__, __LINE__,			\
+		    #a, #b, a_ ? "true" : "false",			\
+		    b_ ? "true" : "false", fmt);			\
+	}								\
+} while (0)
+#define	assert_true(a, fmt...)	assert_b_eq(a, true, fmt)
+#define	assert_false(a, fmt...)	assert_b_eq(a, false, fmt)
+
+#define	assert_str_eq(a, b, fmt...) do {				\
+	if (strcmp((a), (b))) {						\
+		p_test_fail(						\
+		    "%s:%s:%d: Failed assertion: "			\
+		    "(%s) same as (%s) --> "				\
+		    "\"%s\" differs from \"%s\": ",			\
+		    __func__, __FILE__, __LINE__, #a, #b, a, b, fmt);	\
+	}								\
+} while (0)
+#define	assert_str_ne(a, b, fmt...) do {				\
+	if (!strcmp((a), (b))) {					\
+		p_test_fail(						\
+		    "%s:%s:%d: Failed assertion: "			\
+		    "(%s) differs from (%s) --> "			\
+		    "\"%s\" same as \"%s\": ",				\
+		    __func__, __FILE__, __LINE__, #a, #b, a, b, fmt);	\
+	}								\
+} while (0)
+
+#define	assert_not_reached(fmt...) do {					\
+	p_test_fail("%s:%s:%d: Unreachable code reached: ",		\
+	    __func__, __FILE__, __LINE__, fmt);				\
+} while (0)
+
+/*
+ * If this enum changes, corresponding changes in test/test.sh.in are also
+ * necessary.
+ */
+typedef enum {
+	test_status_pass = 0,
+	test_status_skip = 1,
+	test_status_fail = 2,
+
+	test_status_count = 3
+} test_status_t;
+
+typedef void (test_t)(void);
+
+#define	TEST_BEGIN(f)							\
+static void								\
+f(void)									\
+{									\
+	p_test_init(#f);
+
+#define	TEST_END							\
+	goto label_test_end;						\
+label_test_end:								\
+	p_test_fini();							\
+}
+
+#define	test(tests...)							\
+	p_test(tests, NULL)
+
+#define	test_skip_if(e) do {						\
+	if (e) {							\
+		test_skip("%s:%s:%d: Test skipped: (%s)",		\
+		    __func__, __FILE__, __LINE__, #e);			\
+		goto label_test_end;					\
+	}								\
+} while (0)
+
+void	test_skip(const char *format, ...) JEMALLOC_ATTR(format(printf, 1, 2));
+void	test_fail(const char *format, ...) JEMALLOC_ATTR(format(printf, 1, 2));
+
+/* For private use by macros. */
+test_status_t	p_test(test_t* t, ...);
+void	p_test_init(const char *name);
+void	p_test_fini(void);
+void	p_test_fail(const char *format, ...);

test/include/test/thd.h

@@ -0,0 +1,9 @@
+/* Abstraction layer for threading in tests */
+#ifdef _WIN32
+typedef HANDLE thd_t;
+#else
+typedef pthread_t thd_t;
+#endif
+
+void	thd_create(thd_t *thd, void *(*proc)(void *), void *arg);
+void	thd_join(thd_t thd, void **ret);

test/integration/ALLOCM_ARENA.c

@@ -0,0 +1,58 @@
+#include "test/jemalloc_test.h"
+
+#define	NTHREADS 10
+
+void *
+thd_start(void *arg)
+{
+	unsigned thread_ind = (unsigned)(uintptr_t)arg;
+	unsigned arena_ind;
+	void *p;
+	size_t rsz, sz;
+
+	sz = sizeof(arena_ind);
+	assert_d_eq(mallctl("arenas.extend", &arena_ind, &sz, NULL, 0), 0,
+	    "Error in arenas.extend");
+
+	if (thread_ind % 4 != 3) {
+		size_t mib[3];
+		size_t miblen = sizeof(mib) / sizeof(size_t);
+		const char *dss_precs[] = {"disabled", "primary", "secondary"};
+		const char *dss = dss_precs[thread_ind %
+		    (sizeof(dss_precs)/sizeof(char*))];
+		assert_d_eq(mallctlnametomib("arena.0.dss", mib, &miblen), 0,
+		    "Error in mallctlnametomib()");
+		mib[1] = arena_ind;
+		assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, (void *)&dss,
+		    sizeof(const char *)), 0, "Error in mallctlbymib()");
+	}
+
+	assert_d_eq(allocm(&p, &rsz, 1, ALLOCM_ARENA(arena_ind)),
+	    ALLOCM_SUCCESS, "Unexpected allocm() error");
+	dallocm(p, 0);
+
+	return (NULL);
+}
+
+TEST_BEGIN(test_ALLOCM_ARENA)
+{
+	thd_t thds[NTHREADS];
+	unsigned i;
+
+	for (i = 0; i < NTHREADS; i++) {
+		thd_create(&thds[i], thd_start,
+		    (void *)(uintptr_t)i);
+	}
+
+	for (i = 0; i < NTHREADS; i++)
+		thd_join(thds[i], NULL);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_ALLOCM_ARENA));
+}

test/integration/aligned_alloc.c

@@ -1,39 +1,36 @@
-#define	JEMALLOC_MANGLE
-#include "jemalloc_test.h"
+#include "test/jemalloc_test.h"
 
-#define CHUNK 0x400000
+#define	CHUNK 0x400000
 /* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */
-#define MAXALIGN ((size_t)0x2000000LU)
-#define NITER 4
+#define	MAXALIGN ((size_t)0x2000000LU)
+#define	NITER 4
 
-int
-main(void)
+TEST_BEGIN(test_alignment_errors)
 {
-	size_t alignment, size, total;
-	unsigned i;
-	void *p, *ps[NITER];
+	size_t alignment;
+	void *p;
 
-	malloc_printf("Test begin\n");
-
-	/* Test error conditions. */
 	alignment = 0;
 	set_errno(0);
 	p = aligned_alloc(alignment, 1);
-	if (p != NULL || get_errno() != EINVAL) {
-		malloc_printf(
-		    "Expected error for invalid alignment %zu\n", alignment);
-	}
+	assert_false(p != NULL || get_errno() != EINVAL,
+	    "Expected error for invalid alignment %zu", alignment);
 
 	for (alignment = sizeof(size_t); alignment < MAXALIGN;
 	    alignment <<= 1) {
 		set_errno(0);
 		p = aligned_alloc(alignment + 1, 1);
-		if (p != NULL || get_errno() != EINVAL) {
-			malloc_printf(
-			    "Expected error for invalid alignment %zu\n",
-			    alignment + 1);
-		}
+		assert_false(p != NULL || get_errno() != EINVAL,
+		    "Expected error for invalid alignment %zu",
+		    alignment + 1);
 	}
+}
+TEST_END
+
+TEST_BEGIN(test_oom_errors)
+{
+	size_t alignment, size;
+	void *p;
 
 #if LG_SIZEOF_PTR == 3
 	alignment = UINT64_C(0x8000000000000000);
@@ -44,11 +41,9 @@ main(void)
 #endif
 	set_errno(0);
 	p = aligned_alloc(alignment, size);
-	if (p != NULL || get_errno() != ENOMEM) {
-		malloc_printf(
-		    "Expected error for aligned_alloc(%zu, %zu)\n",
-		    alignment, size);
-	}
+	assert_false(p != NULL || get_errno() != ENOMEM,
+	    "Expected error for aligned_alloc(%zu, %zu)",
+	    alignment, size);
 
 #if LG_SIZEOF_PTR == 3
 	alignment = UINT64_C(0x4000000000000000);
@@ -59,11 +54,9 @@ main(void)
 #endif
 	set_errno(0);
 	p = aligned_alloc(alignment, size);
-	if (p != NULL || get_errno() != ENOMEM) {
-		malloc_printf(
-		    "Expected error for aligned_alloc(%zu, %zu)\n",
-		    alignment, size);
-	}
+	assert_false(p != NULL || get_errno() != ENOMEM,
+	    "Expected error for aligned_alloc(%zu, %zu)",
+	    alignment, size);
 
 	alignment = 0x10LU;
 #if LG_SIZEOF_PTR == 3
@@ -73,11 +66,17 @@ main(void)
 #endif
 	set_errno(0);
 	p = aligned_alloc(alignment, size);
-	if (p != NULL || get_errno() != ENOMEM) {
-		malloc_printf(
-		    "Expected error for aligned_alloc(&p, %zu, %zu)\n",
-		    alignment, size);
-	}
+	assert_false(p != NULL || get_errno() != ENOMEM,
+	    "Expected error for aligned_alloc(&p, %zu, %zu)",
+	    alignment, size);
+}
+TEST_END
+
+TEST_BEGIN(test_alignment_and_size)
+{
+	size_t alignment, size, total;
+	unsigned i;
+	void *ps[NITER];
 
 	for (i = 0; i < NITER; i++)
 		ps[i] = NULL;
@@ -86,7 +85,6 @@ main(void)
 	    alignment <= MAXALIGN;
 	    alignment <<= 1) {
 		total = 0;
-		malloc_printf("Alignment: %zu\n", alignment);
 		for (size = 1;
 		    size < 3 * alignment && size < (1U << 31);
 		    size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) {
@@ -95,11 +93,11 @@ main(void)
 				if (ps[i] == NULL) {
 					char buf[BUFERROR_BUF];
 
-					buferror(buf, sizeof(buf));
-					malloc_printf(
-					    "Error for size %zu (%#zx): %s\n",
-					    size, size, buf);
-					exit(1);
+					buferror(get_errno(), buf, sizeof(buf));
+					test_fail(
+					    "Error for alignment=%zu, "
+					    "size=%zu (%#zx): %s",
+					    alignment, size, size, buf);
 				}
 				total += malloc_usable_size(ps[i]);
 				if (total >= (MAXALIGN << 1))
@@ -113,7 +111,15 @@ main(void)
 			}
 		}
 	}
-
-	malloc_printf("Test end\n");
-	return (0);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_alignment_errors,
+	    test_oom_errors,
+	    test_alignment_and_size));
 }

test/integration/allocated.c

@@ -0,0 +1,125 @@
+#include "test/jemalloc_test.h"
+
+static const bool config_stats =
+#ifdef JEMALLOC_STATS
+    true
+#else
+    false
+#endif
+    ;
+
+void *
+thd_start(void *arg)
+{
+	int err;
+	void *p;
+	uint64_t a0, a1, d0, d1;
+	uint64_t *ap0, *ap1, *dp0, *dp1;
+	size_t sz, usize;
+
+	sz = sizeof(a0);
+	if ((err = mallctl("thread.allocated", &a0, &sz, NULL, 0))) {
+		if (err == ENOENT)
+			goto label_ENOENT;
+		test_fail("%s(): Error in mallctl(): %s", __func__,
+		    strerror(err));
+	}
+	sz = sizeof(ap0);
+	if ((err = mallctl("thread.allocatedp", &ap0, &sz, NULL, 0))) {
+		if (err == ENOENT)
+			goto label_ENOENT;
+		test_fail("%s(): Error in mallctl(): %s", __func__,
+		    strerror(err));
+	}
+	assert_u64_eq(*ap0, a0,
+	    "\"thread.allocatedp\" should provide a pointer to internal "
+	    "storage");
+
+	sz = sizeof(d0);
+	if ((err = mallctl("thread.deallocated", &d0, &sz, NULL, 0))) {
+		if (err == ENOENT)
+			goto label_ENOENT;
+		test_fail("%s(): Error in mallctl(): %s", __func__,
+		    strerror(err));
+	}
+	sz = sizeof(dp0);
+	if ((err = mallctl("thread.deallocatedp", &dp0, &sz, NULL, 0))) {
+		if (err == ENOENT)
+			goto label_ENOENT;
+		test_fail("%s(): Error in mallctl(): %s", __func__,
+		    strerror(err));
+	}
+	assert_u64_eq(*dp0, d0,
+	    "\"thread.deallocatedp\" should provide a pointer to internal "
+	    "storage");
+
+	p = malloc(1);
+	assert_ptr_not_null(p, "Unexpected malloc() error");
+
+	sz = sizeof(a1);
+	mallctl("thread.allocated", &a1, &sz, NULL, 0);
+	sz = sizeof(ap1);
+	mallctl("thread.allocatedp", &ap1, &sz, NULL, 0);
+	assert_u64_eq(*ap1, a1,
+	    "Dereferenced \"thread.allocatedp\" value should equal "
+	    "\"thread.allocated\" value");
+	assert_ptr_eq(ap0, ap1,
+	    "Pointer returned by \"thread.allocatedp\" should not change");
+
+	usize = malloc_usable_size(p);
+	assert_u64_le(a0 + usize, a1,
+	    "Allocated memory counter should increase by at least the amount "
+	    "explicitly allocated");
+
+	free(p);
+
+	sz = sizeof(d1);
+	mallctl("thread.deallocated", &d1, &sz, NULL, 0);
+	sz = sizeof(dp1);
+	mallctl("thread.deallocatedp", &dp1, &sz, NULL, 0);
+	assert_u64_eq(*dp1, d1,
+	    "Dereferenced \"thread.deallocatedp\" value should equal "
+	    "\"thread.deallocated\" value");
+	assert_ptr_eq(dp0, dp1,
+	    "Pointer returned by \"thread.deallocatedp\" should not change");
+
+	assert_u64_le(d0 + usize, d1,
+	    "Deallocated memory counter should increase by at least the amount "
+	    "explicitly deallocated");
+
+	return (NULL);
+label_ENOENT:
+	assert_false(config_stats,
+	    "ENOENT should only be returned if stats are disabled");
+	test_skip("\"thread.allocated\" mallctl not available");
+	return (NULL);
+}
+
+TEST_BEGIN(test_main_thread)
+{
+
+	thd_start(NULL);
+}
+TEST_END
+
+TEST_BEGIN(test_subthread)
+{
+	thd_t thd;
+
+	thd_create(&thd, thd_start, NULL);
+	thd_join(thd, NULL);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	/* Run tests multiple times to check for bad interactions. */
+	return (test(
+	    test_main_thread,
+	    test_subthread,
+	    test_main_thread,
+	    test_subthread,
+	    test_main_thread));
+}

test/integration/allocm.c

@@ -0,0 +1,131 @@
+#include "test/jemalloc_test.h"
+
+#define	CHUNK 0x400000
+/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */
+#define	MAXALIGN ((size_t)0x2000000LU)
+#define	NITER 4
+
+TEST_BEGIN(test_basic)
+{
+	size_t nsz, rsz, sz;
+	void *p;
+
+	sz = 42;
+	nsz = 0;
+	assert_d_eq(nallocm(&nsz, sz, 0), ALLOCM_SUCCESS,
+	    "Unexpected nallocm() error");
+	rsz = 0;
+	assert_d_eq(allocm(&p, &rsz, sz, 0), ALLOCM_SUCCESS,
+	    "Unexpected allocm() error");
+	assert_zu_ge(rsz, sz, "Real size smaller than expected");
+	assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch");
+	assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS,
+	    "Unexpected dallocm() error");
+
+	assert_d_eq(allocm(&p, NULL, sz, 0), ALLOCM_SUCCESS,
+	    "Unexpected allocm() error");
+	assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS,
+	    "Unexpected dallocm() error");
+
+	nsz = 0;
+	assert_d_eq(nallocm(&nsz, sz, ALLOCM_ZERO), ALLOCM_SUCCESS,
+	    "Unexpected nallocm() error");
+	rsz = 0;
+	assert_d_eq(allocm(&p, &rsz, sz, ALLOCM_ZERO), ALLOCM_SUCCESS,
+	    "Unexpected allocm() error");
+	assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch");
+	assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS,
+	    "Unexpected dallocm() error");
+}
+TEST_END
+
+TEST_BEGIN(test_alignment_errors)
+{
+	void *p;
+	size_t nsz, rsz, sz, alignment;
+
+#if LG_SIZEOF_PTR == 3
+	alignment = UINT64_C(0x4000000000000000);
+	sz        = UINT64_C(0x8400000000000001);
+#else
+	alignment = 0x40000000LU;
+	sz        = 0x84000001LU;
+#endif
+	nsz = 0;
+	assert_d_eq(nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)), ALLOCM_SUCCESS,
+	    "Unexpected nallocm() error");
+	rsz = 0;
+	assert_d_ne(allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)),
+	    ALLOCM_SUCCESS, "Expected error for allocm(&p, %zu, %#x)",
+	    sz, ALLOCM_ALIGN(alignment));
+}
+TEST_END
+
+TEST_BEGIN(test_alignment_and_size)
+{
+	int r;
+	size_t nsz, rsz, sz, alignment, total;
+	unsigned i;
+	void *ps[NITER];
+
+	for (i = 0; i < NITER; i++)
+		ps[i] = NULL;
+
+	for (alignment = 8;
+	    alignment <= MAXALIGN;
+	    alignment <<= 1) {
+		total = 0;
+		for (sz = 1;
+		    sz < 3 * alignment && sz < (1U << 31);
+		    sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) {
+			for (i = 0; i < NITER; i++) {
+				nsz = 0;
+				r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment) |
+				    ALLOCM_ZERO);
+				assert_d_eq(r, ALLOCM_SUCCESS,
+				    "nallocm() error for alignment=%zu, "
+				    "size=%zu (%#zx): %d",
+				    alignment, sz, sz, r);
+				rsz = 0;
+				r = allocm(&ps[i], &rsz, sz,
+				    ALLOCM_ALIGN(alignment) | ALLOCM_ZERO);
+				assert_d_eq(r, ALLOCM_SUCCESS,
+				    "allocm() error for alignment=%zu, "
+				    "size=%zu (%#zx): %d",
+				    alignment, sz, sz, r);
+				assert_zu_ge(rsz, sz,
+				    "Real size smaller than expected for "
+				    "alignment=%zu, size=%zu", alignment, sz);
+				assert_zu_eq(nsz, rsz,
+				    "nallocm()/allocm() rsize mismatch for "
+				    "alignment=%zu, size=%zu", alignment, sz);
+				assert_ptr_null(
+				    (void *)((uintptr_t)ps[i] & (alignment-1)),
+				    "%p inadequately aligned for"
+				    " alignment=%zu, size=%zu", ps[i],
+				    alignment, sz);
+				sallocm(ps[i], &rsz, 0);
+				total += rsz;
+				if (total >= (MAXALIGN << 1))
+					break;
+			}
+			for (i = 0; i < NITER; i++) {
+				if (ps[i] != NULL) {
+					dallocm(ps[i], 0);
+					ps[i] = NULL;
+				}
+			}
+		}
+	}
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_basic,
+	    test_alignment_errors,
+	    test_alignment_and_size));
+}

test/integration/mallocx.c

@@ -0,0 +1,119 @@
+#include "test/jemalloc_test.h"
+
+#define	CHUNK 0x400000
+/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */
+#define	MAXALIGN ((size_t)0x2000000LU)
+#define	NITER 4
+
+TEST_BEGIN(test_basic)
+{
+	size_t nsz, rsz, sz;
+	void *p;
+
+	sz = 42;
+	nsz = nallocx(sz, 0);
+	assert_zu_ne(nsz, 0, "Unexpected nallocx() error");
+	p = mallocx(sz, 0);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+	rsz = sallocx(p, 0);
+	assert_zu_ge(rsz, sz, "Real size smaller than expected");
+	assert_zu_eq(nsz, rsz, "nallocx()/sallocx() size mismatch");
+	dallocx(p, 0);
+
+	p = mallocx(sz, 0);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+	dallocx(p, 0);
+
+	nsz = nallocx(sz, MALLOCX_ZERO);
+	assert_zu_ne(nsz, 0, "Unexpected nallocx() error");
+	p = mallocx(sz, MALLOCX_ZERO);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+	rsz = sallocx(p, 0);
+	assert_zu_eq(nsz, rsz, "nallocx()/sallocx() rsize mismatch");
+	dallocx(p, 0);
+}
+TEST_END
+
+TEST_BEGIN(test_alignment_errors)
+{
+	void *p;
+	size_t nsz, sz, alignment;
+
+#if LG_SIZEOF_PTR == 3
+	alignment = UINT64_C(0x4000000000000000);
+	sz        = UINT64_C(0x8400000000000001);
+#else
+	alignment = 0x40000000LU;
+	sz        = 0x84000001LU;
+#endif
+	nsz = nallocx(sz, MALLOCX_ALIGN(alignment));
+	assert_zu_ne(nsz, 0, "Unexpected nallocx() error");
+	p = mallocx(sz, MALLOCX_ALIGN(alignment));
+	assert_ptr_null(p, "Expected error for mallocx(%zu, %#x)", sz,
+	    MALLOCX_ALIGN(alignment));
+}
+TEST_END
+
+TEST_BEGIN(test_alignment_and_size)
+{
+	size_t nsz, rsz, sz, alignment, total;
+	unsigned i;
+	void *ps[NITER];
+
+	for (i = 0; i < NITER; i++)
+		ps[i] = NULL;
+
+	for (alignment = 8;
+	    alignment <= MAXALIGN;
+	    alignment <<= 1) {
+		total = 0;
+		for (sz = 1;
+		    sz < 3 * alignment && sz < (1U << 31);
+		    sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) {
+			for (i = 0; i < NITER; i++) {
+				nsz = nallocx(sz, MALLOCX_ALIGN(alignment) |
+				    MALLOCX_ZERO);
+				assert_zu_ne(nsz, 0,
+				    "nallocx() error for alignment=%zu, "
+				    "size=%zu (%#zx)", alignment, sz, sz);
+				ps[i] = mallocx(sz, MALLOCX_ALIGN(alignment) |
+				    MALLOCX_ZERO);
+				assert_ptr_not_null(ps[i],
+				    "mallocx() error for alignment=%zu, "
+				    "size=%zu (%#zx)", alignment, sz, sz);
+				rsz = sallocx(ps[i], 0);
+				assert_zu_ge(rsz, sz,
+				    "Real size smaller than expected for "
+				    "alignment=%zu, size=%zu", alignment, sz);
+				assert_zu_eq(nsz, rsz,
+				    "nallocx()/sallocx() size mismatch for "
+				    "alignment=%zu, size=%zu", alignment, sz);
+				assert_ptr_null(
+				    (void *)((uintptr_t)ps[i] & (alignment-1)),
+				    "%p inadequately aligned for"
+				    " alignment=%zu, size=%zu", ps[i],
+				    alignment, sz);
+				total += rsz;
+				if (total >= (MAXALIGN << 1))
+					break;
+			}
+			for (i = 0; i < NITER; i++) {
+				if (ps[i] != NULL) {
+					dallocx(ps[i], 0);
+					ps[i] = NULL;
+				}
+			}
+		}
+	}
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_basic,
+	    test_alignment_errors,
+	    test_alignment_and_size));
+}

test/integration/mremap.c

@@ -0,0 +1,45 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_mremap)
+{
+	int err;
+	size_t sz, lg_chunk, chunksize, i;
+	char *p, *q;
+
+	sz = sizeof(lg_chunk);
+	err = mallctl("opt.lg_chunk", &lg_chunk, &sz, NULL, 0);
+	assert_d_eq(err, 0, "Error in mallctl(): %s", strerror(err));
+	chunksize = ((size_t)1U) << lg_chunk;
+
+	p = (char *)malloc(chunksize);
+	assert_ptr_not_null(p, "malloc(%zu) --> %p", chunksize, p);
+	memset(p, 'a', chunksize);
+
+	q = (char *)realloc(p, chunksize * 2);
+	assert_ptr_not_null(q, "realloc(%p, %zu) --> %p", p, chunksize * 2,
+	    q);
+	for (i = 0; i < chunksize; i++) {
+		assert_c_eq(q[i], 'a',
+		    "realloc() should preserve existing bytes across copies");
+	}
+
+	p = q;
+
+	q = (char *)realloc(p, chunksize);
+	assert_ptr_not_null(q, "realloc(%p, %zu) --> %p", p, chunksize, q);
+	for (i = 0; i < chunksize; i++) {
+		assert_c_eq(q[i], 'a',
+		    "realloc() should preserve existing bytes across copies");
+	}
+
+	free(q);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_mremap));
+}

test/integration/posix_memalign.c

@@ -1,40 +1,34 @@
-#define	JEMALLOC_MANGLE
-#include "jemalloc_test.h"
+#include "test/jemalloc_test.h"
 
-#define CHUNK 0x400000
+#define	CHUNK 0x400000
 /* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */
-#define MAXALIGN ((size_t)0x2000000LU)
-#define NITER 4
+#define	MAXALIGN ((size_t)0x2000000LU)
+#define	NITER 4
 
-int
-main(void)
+TEST_BEGIN(test_alignment_errors)
 {
-	size_t alignment, size, total;
-	unsigned i;
-	int err;
-	void *p, *ps[NITER];
+	size_t alignment;
+	void *p;
 
-	malloc_printf("Test begin\n");
-
-	/* Test error conditions. */
 	for (alignment = 0; alignment < sizeof(void *); alignment++) {
-		err = posix_memalign(&p, alignment, 1);
-		if (err != EINVAL) {
-			malloc_printf(
-			    "Expected error for invalid alignment %zu\n",
-			    alignment);
-		}
+		assert_d_eq(posix_memalign(&p, alignment, 1), EINVAL,
+		    "Expected error for invalid alignment %zu",
+		    alignment);
 	}
 
 	for (alignment = sizeof(size_t); alignment < MAXALIGN;
 	    alignment <<= 1) {
-		err = posix_memalign(&p, alignment + 1, 1);
-		if (err == 0) {
-			malloc_printf(
-			    "Expected error for invalid alignment %zu\n",
-			    alignment + 1);
-		}
+		assert_d_ne(posix_memalign(&p, alignment + 1, 1), 0,
+		    "Expected error for invalid alignment %zu",
+		    alignment + 1);
 	}
+}
+TEST_END
+
+TEST_BEGIN(test_oom_errors)
+{
+	size_t alignment, size;
+	void *p;
 
 #if LG_SIZEOF_PTR == 3
 	alignment = UINT64_C(0x8000000000000000);
@@ -43,12 +37,9 @@ main(void)
 	alignment = 0x80000000LU;
 	size      = 0x80000000LU;
 #endif
-	err = posix_memalign(&p, alignment, size);
-	if (err == 0) {
-		malloc_printf(
-		    "Expected error for posix_memalign(&p, %zu, %zu)\n",
-		    alignment, size);
-	}
+	assert_d_ne(posix_memalign(&p, alignment, size), 0,
+	    "Expected error for posix_memalign(&p, %zu, %zu)",
+	    alignment, size);
 
 #if LG_SIZEOF_PTR == 3
 	alignment = UINT64_C(0x4000000000000000);
@@ -57,12 +48,9 @@ main(void)
 	alignment = 0x40000000LU;
 	size      = 0x84000001LU;
 #endif
-	err = posix_memalign(&p, alignment, size);
-	if (err == 0) {
-		malloc_printf(
-		    "Expected error for posix_memalign(&p, %zu, %zu)\n",
-		    alignment, size);
-	}
+	assert_d_ne(posix_memalign(&p, alignment, size), 0,
+	    "Expected error for posix_memalign(&p, %zu, %zu)",
+	    alignment, size);
 
 	alignment = 0x10LU;
 #if LG_SIZEOF_PTR == 3
@@ -70,12 +58,18 @@ main(void)
 #else
 	size = 0xfffffff0LU;
 #endif
-	err = posix_memalign(&p, alignment, size);
-	if (err == 0) {
-		malloc_printf(
-		    "Expected error for posix_memalign(&p, %zu, %zu)\n",
-		    alignment, size);
-	}
+	assert_d_ne(posix_memalign(&p, alignment, size), 0,
+	    "Expected error for posix_memalign(&p, %zu, %zu)",
+	    alignment, size);
+}
+TEST_END
+
+TEST_BEGIN(test_alignment_and_size)
+{
+	size_t alignment, size, total;
+	unsigned i;
+	int err;
+	void *ps[NITER];
 
 	for (i = 0; i < NITER; i++)
 		ps[i] = NULL;
@@ -84,7 +78,6 @@ main(void)
 	    alignment <= MAXALIGN;
 	    alignment <<= 1) {
 		total = 0;
-		malloc_printf("Alignment: %zu\n", alignment);
 		for (size = 1;
 		    size < 3 * alignment && size < (1U << 31);
 		    size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) {
@@ -92,10 +85,13 @@ main(void)
 				err = posix_memalign(&ps[i],
 				    alignment, size);
 				if (err) {
-					malloc_printf(
-					    "Error for size %zu (%#zx): %s\n",
-					    size, size, strerror(err));
-					exit(1);
+					char buf[BUFERROR_BUF];
+
+					buferror(get_errno(), buf, sizeof(buf));
+					test_fail(
+					    "Error for alignment=%zu, "
+					    "size=%zu (%#zx): %s",
+					    alignment, size, size, buf);
 				}
 				total += malloc_usable_size(ps[i]);
 				if (total >= (MAXALIGN << 1))
@@ -109,7 +105,15 @@ main(void)
 			}
 		}
 	}
-
-	malloc_printf("Test end\n");
-	return (0);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_alignment_errors,
+	    test_oom_errors,
+	    test_alignment_and_size));
 }

test/integration/rallocm.c

@@ -0,0 +1,111 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_same_size)
+{
+	void *p, *q;
+	size_t sz, tsz;
+
+	assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS,
+	    "Unexpected allocm() error");
+
+	q = p;
+	assert_d_eq(rallocm(&q, &tsz, sz, 0, ALLOCM_NO_MOVE), ALLOCM_SUCCESS,
+	    "Unexpected rallocm() error");
+	assert_ptr_eq(q, p, "Unexpected object move");
+	assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz);
+
+	assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS,
+	    "Unexpected dallocm() error");
+}
+TEST_END
+
+TEST_BEGIN(test_extra_no_move)
+{
+	void *p, *q;
+	size_t sz, tsz;
+
+	assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS,
+	    "Unexpected allocm() error");
+
+	q = p;
+	assert_d_eq(rallocm(&q, &tsz, sz, sz-42, ALLOCM_NO_MOVE),
+	    ALLOCM_SUCCESS, "Unexpected rallocm() error");
+	assert_ptr_eq(q, p, "Unexpected object move");
+	assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz);
+
+	assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS,
+	    "Unexpected dallocm() error");
+}
+TEST_END
+
+TEST_BEGIN(test_no_move_fail)
+{
+	void *p, *q;
+	size_t sz, tsz;
+
+	assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS,
+	    "Unexpected allocm() error");
+
+	q = p;
+	assert_d_eq(rallocm(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE),
+	    ALLOCM_ERR_NOT_MOVED, "Unexpected rallocm() result");
+	assert_ptr_eq(q, p, "Unexpected object move");
+	assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz);
+
+	assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS,
+	    "Unexpected dallocm() error");
+}
+TEST_END
+
+TEST_BEGIN(test_grow_and_shrink)
+{
+	void *p, *q;
+	size_t tsz;
+#define	NCYCLES 3
+	unsigned i, j;
+#define	NSZS 2500
+	size_t szs[NSZS];
+#define	MAXSZ ZU(12 * 1024 * 1024)
+
+	assert_d_eq(allocm(&p, &szs[0], 1, 0), ALLOCM_SUCCESS,
+	    "Unexpected allocm() error");
+
+	for (i = 0; i < NCYCLES; i++) {
+		for (j = 1; j < NSZS && szs[j-1] < MAXSZ; j++) {
+			q = p;
+			assert_d_eq(rallocm(&q, &szs[j], szs[j-1]+1, 0, 0),
+			    ALLOCM_SUCCESS,
+			    "Unexpected rallocm() error for size=%zu-->%zu",
+			    szs[j-1], szs[j-1]+1);
+			assert_zu_ne(szs[j], szs[j-1]+1,
+			    "Expected size to at least: %zu", szs[j-1]+1);
+			p = q;
+		}
+
+		for (j--; j > 0; j--) {
+			q = p;
+			assert_d_eq(rallocm(&q, &tsz, szs[j-1], 0, 0),
+			    ALLOCM_SUCCESS,
+			    "Unexpected rallocm() error for size=%zu-->%zu",
+			    szs[j], szs[j-1]);
+			assert_zu_eq(tsz, szs[j-1],
+			    "Expected size=%zu, got size=%zu", szs[j-1], tsz);
+			p = q;
+		}
+	}
+
+	assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS,
+	    "Unexpected dallocm() error");
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_same_size,
+	    test_extra_no_move,
+	    test_no_move_fail,
+	    test_grow_and_shrink));
+}

test/integration/rallocx.c

@@ -0,0 +1,182 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_grow_and_shrink)
+{
+	void *p, *q;
+	size_t tsz;
+#define	NCYCLES 3
+	unsigned i, j;
+#define	NSZS 2500
+	size_t szs[NSZS];
+#define	MAXSZ ZU(12 * 1024 * 1024)
+
+	p = mallocx(1, 0);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+	szs[0] = sallocx(p, 0);
+
+	for (i = 0; i < NCYCLES; i++) {
+		for (j = 1; j < NSZS && szs[j-1] < MAXSZ; j++) {
+			q = rallocx(p, szs[j-1]+1, 0);
+			assert_ptr_not_null(q,
+			    "Unexpected rallocx() error for size=%zu-->%zu",
+			    szs[j-1], szs[j-1]+1);
+			szs[j] = sallocx(q, 0);
+			assert_zu_ne(szs[j], szs[j-1]+1,
+			    "Expected size to at least: %zu", szs[j-1]+1);
+			p = q;
+		}
+
+		for (j--; j > 0; j--) {
+			q = rallocx(p, szs[j-1], 0);
+			assert_ptr_not_null(q,
+			    "Unexpected rallocx() error for size=%zu-->%zu",
+			    szs[j], szs[j-1]);
+			tsz = sallocx(q, 0);
+			assert_zu_eq(tsz, szs[j-1],
+			    "Expected size=%zu, got size=%zu", szs[j-1], tsz);
+			p = q;
+		}
+	}
+
+	dallocx(p, 0);
+#undef MAXSZ
+#undef NSZS
+#undef NCYCLES
+}
+TEST_END
+
+static bool
+validate_fill(const void *p, uint8_t c, size_t offset, size_t len)
+{
+	bool ret = false;
+	const uint8_t *buf = (const uint8_t *)p;
+	size_t i;
+
+	for (i = 0; i < len; i++) {
+		uint8_t b = buf[offset+i];
+		if (b != c) {
+			test_fail("Allocation at %p contains %#x rather than "
+			    "%#x at offset %zu", p, b, c, offset+i);
+			ret = true;
+		}
+	}
+
+	return (ret);
+}
+
+TEST_BEGIN(test_zero)
+{
+	void *p, *q;
+	size_t psz, qsz, i, j;
+	size_t start_sizes[] = {1, 3*1024, 63*1024, 4095*1024};
+#define	FILL_BYTE 0xaaU
+#define	RANGE 2048
+
+	for (i = 0; i < sizeof(start_sizes)/sizeof(size_t); i++) {
+		size_t start_size = start_sizes[i];
+		p = mallocx(start_size, MALLOCX_ZERO);
+		assert_ptr_not_null(p, "Unexpected mallocx() error");
+		psz = sallocx(p, 0);
+
+		assert_false(validate_fill(p, 0, 0, psz),
+		    "Expected zeroed memory");
+		memset(p, FILL_BYTE, psz);
+		assert_false(validate_fill(p, FILL_BYTE, 0, psz),
+		    "Expected filled memory");
+
+		for (j = 1; j < RANGE; j++) {
+			q = rallocx(p, start_size+j, MALLOCX_ZERO);
+			assert_ptr_not_null(q, "Unexpected rallocx() error");
+			qsz = sallocx(q, 0);
+			if (q != p || qsz != psz) {
+				assert_false(validate_fill(q, FILL_BYTE, 0,
+				    psz), "Expected filled memory");
+				assert_false(validate_fill(q, 0, psz, qsz-psz),
+				    "Expected zeroed memory");
+			}
+			if (psz != qsz) {
+				memset(q+psz, FILL_BYTE, qsz-psz);
+				psz = qsz;
+			}
+			p = q;
+		}
+		assert_false(validate_fill(p, FILL_BYTE, 0, psz),
+		    "Expected filled memory");
+		dallocx(p, 0);
+	}
+#undef FILL_BYTE
+}
+TEST_END
+
+TEST_BEGIN(test_align)
+{
+	void *p, *q;
+	size_t align;
+#define	MAX_ALIGN (ZU(1) << 29)
+
+	align = ZU(1);
+	p = mallocx(1, MALLOCX_ALIGN(align));
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+
+	for (align <<= 1; align <= MAX_ALIGN; align <<= 1) {
+		q = rallocx(p, 1, MALLOCX_ALIGN(align));
+		assert_ptr_not_null(q,
+		    "Unexpected rallocx() error for align=%zu", align);
+		assert_ptr_null(
+		    (void *)((uintptr_t)q & (align-1)),
+		    "%p inadequately aligned for align=%zu",
+		    q, align);
+		p = q;
+	}
+	dallocx(p, 0);
+#undef MAX_ALIGN
+}
+TEST_END
+
+TEST_BEGIN(test_lg_align_and_zero)
+{
+	void *p, *q;
+	size_t lg_align, sz;
+#define	MAX_LG_ALIGN 29
+#define	MAX_VALIDATE (ZU(1) << 22)
+
+	lg_align = ZU(0);
+	p = mallocx(1, MALLOCX_LG_ALIGN(lg_align)|MALLOCX_ZERO);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+
+	for (lg_align++; lg_align <= MAX_LG_ALIGN; lg_align++) {
+		q = rallocx(p, 1, MALLOCX_LG_ALIGN(lg_align)|MALLOCX_ZERO);
+		assert_ptr_not_null(q,
+		    "Unexpected rallocx() error for lg_align=%zu", lg_align);
+		assert_ptr_null(
+		    (void *)((uintptr_t)q & ((ZU(1) << lg_align)-1)),
+		    "%p inadequately aligned for lg_align=%zu",
+		    q, lg_align);
+		sz = sallocx(q, 0);
+		if ((sz << 1) <= MAX_VALIDATE) {
+			assert_false(validate_fill(q, 0, 0, sz),
+			    "Expected zeroed memory");
+		} else {
+			assert_false(validate_fill(q, 0, 0, MAX_VALIDATE),
+			    "Expected zeroed memory");
+			assert_false(validate_fill(q+sz-MAX_VALIDATE, 0, 0,
+			    MAX_VALIDATE), "Expected zeroed memory");
+		}
+		p = q;
+	}
+	dallocx(p, 0);
+#undef MAX_VALIDATE
+#undef MAX_LG_ALIGN
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_grow_and_shrink,
+	    test_zero,
+	    test_align,
+	    test_lg_align_and_zero));
+}

test/integration/thread_arena.c

@@ -0,0 +1,79 @@
+#include "test/jemalloc_test.h"
+
+#define	NTHREADS 10
+
+void *
+thd_start(void *arg)
+{
+	unsigned main_arena_ind = *(unsigned *)arg;
+	void *p;
+	unsigned arena_ind;
+	size_t size;
+	int err;
+
+	p = malloc(1);
+	assert_ptr_not_null(p, "Error in malloc()");
+	free(p);
+
+	size = sizeof(arena_ind);
+	if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind,
+	    sizeof(main_arena_ind)))) {
+		char buf[BUFERROR_BUF];
+
+		buferror(err, buf, sizeof(buf));
+		test_fail("Error in mallctl(): %s", buf);
+	}
+
+	size = sizeof(arena_ind);
+	if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) {
+		char buf[BUFERROR_BUF];
+
+		buferror(err, buf, sizeof(buf));
+		test_fail("Error in mallctl(): %s", buf);
+	}
+	assert_u_eq(arena_ind, main_arena_ind,
+	    "Arena index should be same as for main thread");
+
+	return (NULL);
+}
+
+TEST_BEGIN(test_thread_arena)
+{
+	void *p;
+	unsigned arena_ind;
+	size_t size;
+	int err;
+	thd_t thds[NTHREADS];
+	unsigned i;
+
+	p = malloc(1);
+	assert_ptr_not_null(p, "Error in malloc()");
+
+	size = sizeof(arena_ind);
+	if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) {
+		char buf[BUFERROR_BUF];
+
+		buferror(err, buf, sizeof(buf));
+		test_fail("Error in mallctl(): %s", buf);
+	}
+
+	for (i = 0; i < NTHREADS; i++) {
+		thd_create(&thds[i], thd_start,
+		    (void *)&arena_ind);
+	}
+
+	for (i = 0; i < NTHREADS; i++) {
+		intptr_t join_ret;
+		thd_join(thds[i], (void *)&join_ret);
+		assert_zd_eq(join_ret, 0, "Unexpected thread join error");
+	}
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_thread_arena));
+}

test/integration/thread_tcache_enabled.c

@@ -0,0 +1,113 @@
+#include "test/jemalloc_test.h"
+
+static const bool config_tcache =
+#ifdef JEMALLOC_TCACHE
+    true
+#else
+    false
+#endif
+    ;
+
+void *
+thd_start(void *arg)
+{
+	int err;
+	size_t sz;
+	bool e0, e1;
+
+	sz = sizeof(bool);
+	if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) {
+		if (err == ENOENT) {
+			assert_false(config_tcache,
+			    "ENOENT should only be returned if tcache is "
+			    "disabled");
+		}
+		goto label_ENOENT;
+	}
+
+	if (e0) {
+		e1 = false;
+		assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz),
+		    0, "Unexpected mallctl() error");
+		assert_true(e0, "tcache should be enabled");
+	}
+
+	e1 = true;
+	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
+	    "Unexpected mallctl() error");
+	assert_false(e0, "tcache should be disabled");
+
+	e1 = true;
+	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
+	    "Unexpected mallctl() error");
+	assert_true(e0, "tcache should be enabled");
+
+	e1 = false;
+	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
+	    "Unexpected mallctl() error");
+	assert_true(e0, "tcache should be enabled");
+
+	e1 = false;
+	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
+	    "Unexpected mallctl() error");
+	assert_false(e0, "tcache should be disabled");
+
+	free(malloc(1));
+	e1 = true;
+	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
+	    "Unexpected mallctl() error");
+	assert_false(e0, "tcache should be disabled");
+
+	free(malloc(1));
+	e1 = true;
+	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
+	    "Unexpected mallctl() error");
+	assert_true(e0, "tcache should be enabled");
+
+	free(malloc(1));
+	e1 = false;
+	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
+	    "Unexpected mallctl() error");
+	assert_true(e0, "tcache should be enabled");
+
+	free(malloc(1));
+	e1 = false;
+	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
+	    "Unexpected mallctl() error");
+	assert_false(e0, "tcache should be disabled");
+
+	free(malloc(1));
+	return (NULL);
+label_ENOENT:
+	test_skip("\"thread.tcache.enabled\" mallctl not available");
+	return (NULL);
+}
+
+TEST_BEGIN(test_main_thread)
+{
+
+	thd_start(NULL);
+}
+TEST_END
+
+TEST_BEGIN(test_subthread)
+{
+	thd_t thd;
+
+	thd_create(&thd, thd_start, NULL);
+	thd_join(thd, NULL);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	/* Run tests multiple times to check for bad interactions. */
+	return (test(
+	    test_main_thread,
+	    test_subthread,
+	    test_main_thread,
+	    test_subthread,
+	    test_main_thread));
+}

test/integration/xallocx.c

@@ -0,0 +1,59 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_same_size)
+{
+	void *p;
+	size_t sz, tsz;
+
+	p = mallocx(42, 0);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+	sz = sallocx(p, 0);
+
+	tsz = xallocx(p, sz, 0, 0);
+	assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz);
+
+	dallocx(p, 0);
+}
+TEST_END
+
+TEST_BEGIN(test_extra_no_move)
+{
+	void *p;
+	size_t sz, tsz;
+
+	p = mallocx(42, 0);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+	sz = sallocx(p, 0);
+
+	tsz = xallocx(p, sz, sz-42, 0);
+	assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz);
+
+	dallocx(p, 0);
+}
+TEST_END
+
+TEST_BEGIN(test_no_move_fail)
+{
+	void *p;
+	size_t sz, tsz;
+
+	p = mallocx(42, 0);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+	sz = sallocx(p, 0);
+
+	tsz = xallocx(p, sz + 5, 0, 0);
+	assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz);
+
+	dallocx(p, 0);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_same_size,
+	    test_extra_no_move,
+	    test_no_move_fail));
+}

test/jemalloc_test.h.in

@@ -1,53 +0,0 @@
-/*
- * This header should be included by tests, rather than directly including
- * jemalloc/jemalloc.h, because --with-install-suffix may cause the header to
- * have a different name.
- */
-#include "jemalloc/jemalloc@install_suffix@.h"
-#include "jemalloc/internal/jemalloc_internal.h"
-
-/* Abstraction layer for threading in tests */
-#ifdef _WIN32
-#include <windows.h>
-
-typedef HANDLE je_thread_t;
-
-void
-je_thread_create(je_thread_t *thread, void *(*proc)(void *), void *arg)
-{
-	LPTHREAD_START_ROUTINE routine = (LPTHREAD_START_ROUTINE)proc;
-	*thread = CreateThread(NULL, 0, routine, arg, 0, NULL);
-	if (*thread == NULL) {
-		malloc_printf("Error in CreateThread()\n");
-		exit(1);
-	}
-}
-
-void
-je_thread_join(je_thread_t thread, void **ret)
-{
-	WaitForSingleObject(thread, INFINITE);
-}
-
-#else
-#include <pthread.h>
-
-typedef pthread_t je_thread_t;
-
-void
-je_thread_create(je_thread_t *thread, void *(*proc)(void *), void *arg)
-{
-
-	if (pthread_create(thread, NULL, proc, arg) != 0) {
-		malloc_printf("Error in pthread_create()\n");
-		exit(1);
-	}
-}
-
-void
-je_thread_join(je_thread_t thread, void **ret)
-{
-
-	pthread_join(thread, ret);
-}
-#endif