Implement malloc_vsnprintf().

Implement malloc_vsnprintf() (a subset of vsnprintf(3)) as well as several other printing functions based on it, so that formatted printing can be relied upon without concern for inducing a dependency on floating point runtime support. Replace malloc_write() calls with malloc_*printf() where doing so simplifies the code. Add name mangling for library-private symbols in the data and BSS sections. Adjust CONF_HANDLE_*() macros in malloc_conf_init() to expose all opt_* variable use to cpp so that proper mangling occurs.
2012-03-06 14:57:45 -08:00
parent 4507f34628
commit d81e4bdd5c
17 changed files with 958 additions and 570 deletions
--- a/include/jemalloc/internal/ctl.h
+++ b/include/jemalloc/internal/ctl.h
@@ -76,19 +76,17 @@ bool	ctl_boot(void);
 #define	xmallctl(name, oldp, oldlenp, newp, newlen) do {		\
 	if (je_mallctl(name, oldp, oldlenp, newp, newlen)		\
 	    != 0) {							\
-		malloc_write("<jemalloc>: Failure in xmallctl(\"");	\
-		malloc_write(name);					\
-		malloc_write("\", ...)\n");				\
+		malloc_printf(						\
+		    "<jemalloc>: Failure in xmallctl(\"%s\", ...)\n",	\
+		    name);						\
 		abort();						\
 	}								\
 } while (0)

 #define	xmallctlnametomib(name, mibp, miblenp) do {			\
 	if (je_mallctlnametomib(name, mibp, miblenp) != 0) {		\
-		malloc_write(						\
-		    "<jemalloc>: Failure in xmallctlnametomib(\"");	\
-		malloc_write(name);					\
-		malloc_write("\", ...)\n");				\
+		malloc_printf("<jemalloc>: Failure in "			\
+		    "xmallctlnametomib(\"%s\", ...)\n", name);		\
 		abort();						\
 	}								\
 } while (0)
--- a/include/jemalloc/internal/hash.h
+++ b/include/jemalloc/internal/hash.h
@@ -48,14 +48,14 @@ hash(const void *key, size_t len, uint64_t seed)

 	data2 = (const unsigned char *)data;
 	switch(len & 7) {
-		case 7: h ^= ((uint64_t)(data2[6])) << 48;
-		case 6: h ^= ((uint64_t)(data2[5])) << 40;
-		case 5: h ^= ((uint64_t)(data2[4])) << 32;
-		case 4: h ^= ((uint64_t)(data2[3])) << 24;
-		case 3: h ^= ((uint64_t)(data2[2])) << 16;
-		case 2: h ^= ((uint64_t)(data2[1])) << 8;
-		case 1: h ^= ((uint64_t)(data2[0]));
-			h *= m;
+	case 7: h ^= ((uint64_t)(data2[6])) << 48;
+	case 6: h ^= ((uint64_t)(data2[5])) << 40;
+	case 5: h ^= ((uint64_t)(data2[4])) << 32;
+	case 4: h ^= ((uint64_t)(data2[3])) << 24;
+	case 3: h ^= ((uint64_t)(data2[2])) << 16;
+	case 2: h ^= ((uint64_t)(data2[1])) << 8;
+	case 1: h ^= ((uint64_t)(data2[0]));
+		h *= m;
 	}

 	h ^= h >> r;
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -149,39 +149,6 @@ static const bool config_ivsalloc =
 #include "jemalloc/internal/qr.h"
 #include "jemalloc/internal/ql.h"

-extern void	(*je_malloc_message)(void *wcbopaque, const char *s);
-
-/*
- * Define a custom assert() in order to reduce the chances of deadlock during
- * assertion failure.
- */
-#ifndef assert
-#  ifdef JEMALLOC_DEBUG
-#    define assert(e) do {						\
-	if (!(e)) {							\
-		char line_buf[UMAX2S_BUFSIZE];				\
-		malloc_write("<jemalloc>: ");				\
-		malloc_write(__FILE__);					\
-		malloc_write(":");					\
-		malloc_write(u2s(__LINE__, 10, line_buf));		\
-		malloc_write(": Failed assertion: ");			\
-		malloc_write("\"");					\
-		malloc_write(#e);					\
-		malloc_write("\"\n");					\
-		abort();						\
-	}								\
-} while (0)
-#  else
-#    define assert(e)
-#  endif
-#endif
-
-/* Use to assert a particular configuration, e.g., cassert(config_debug). */
-#define	cassert(c) do {							\
-	if ((c) == false)						\
-		assert(false);						\
-} while (0)
-
 /*
 * jemalloc can conceptually be broken into components (arena, tcache, etc.),
 * but there are circular dependencies that cannot be broken without
@@ -215,9 +182,6 @@ extern void	(*je_malloc_message)(void *wcbopaque, const char *s);
 #  define JEMALLOC_INLINE static inline
 #endif

-/* Size of stack-allocated buffer passed to buferror(). */
-#define	BUFERROR_BUF		64
-
 /* Smallest size class to support. */
 #define	LG_TINY_MIN		3
 #define	TINY_MIN		(1U << LG_TINY_MIN)
@@ -318,6 +282,7 @@ extern void	(*je_malloc_message)(void *wcbopaque, const char *s);
 #define	PAGE_CEILING(s)							\
 	(((s) + PAGE_MASK) & ~PAGE_MASK)

+#include "jemalloc/internal/util.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/prng.h"
 #include "jemalloc/internal/ckh.h"
@@ -344,6 +309,7 @@ extern void	(*je_malloc_message)(void *wcbopaque, const char *s);
 /******************************************************************************/
 #define JEMALLOC_H_STRUCTS

+#include "jemalloc/internal/util.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/prng.h"
 #include "jemalloc/internal/ckh.h"
@@ -443,10 +409,10 @@ thread_allocated_t	*thread_allocated_get_hard(void);

 arena_t	*arenas_extend(unsigned ind);
 arena_t	*choose_arena_hard(void);
-int	buferror(int errnum, char *buf, size_t buflen);
 void	jemalloc_prefork(void);
 void	jemalloc_postfork(void);

+#include "jemalloc/internal/util.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/prng.h"
 #include "jemalloc/internal/ckh.h"
@@ -473,6 +439,7 @@ void	jemalloc_postfork(void);
 /******************************************************************************/
 #define JEMALLOC_H_INLINES

+#include "jemalloc/internal/util.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/prng.h"
 #include "jemalloc/internal/ckh.h"
@@ -487,33 +454,13 @@ void	jemalloc_postfork(void);
 #include "jemalloc/internal/huge.h"

 #ifndef JEMALLOC_ENABLE_INLINE
-size_t	pow2_ceil(size_t x);
 size_t	s2u(size_t size);
 size_t	sa2u(size_t size, size_t alignment, size_t *run_size_p);
-void	malloc_write(const char *s);
 arena_t	*choose_arena(void);
 thread_allocated_t	*thread_allocated_get(void);
 #endif

 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
-/* Compute the smallest power of 2 that is >= x. */
-JEMALLOC_INLINE size_t
-pow2_ceil(size_t x)
-{
-
-	x--;
-	x |= x >> 1;
-	x |= x >> 2;
-	x |= x >> 4;
-	x |= x >> 8;
-	x |= x >> 16;
-#if (LG_SIZEOF_PTR == 3)
-	x |= x >> 32;
-#endif
-	x++;
-	return (x);
-}
-
 /*
 * Compute usable size that would result from allocating an object with the
 * specified size.
@@ -619,17 +566,6 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p)
 	}
 }

-/*
- * Wrapper around malloc_message() that avoids the need for
- * je_malloc_message(...) throughout the code.
- */
-JEMALLOC_INLINE void
-malloc_write(const char *s)
-{
-
-	je_malloc_message(NULL, s);
-}
-
 /*
 * Choose an arena based on a per-thread value (fast-path code, calls slow-path
 * code if necessary).
--- a/include/jemalloc/internal/private_namespace.h
+++ b/include/jemalloc/internal/private_namespace.h
@@ -23,6 +23,7 @@
 #define	arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index)
 #define	arenas_extend JEMALLOC_N(arenas_extend)
 #define	arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index)
+#define	arenas_tls JEMALLOC_N(arenas_tls)
 #define	atomic_add_uint32 JEMALLOC_N(atomic_add_uint32)
 #define	atomic_add_uint64 JEMALLOC_N(atomic_add_uint64)
 #define	atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32)
@@ -137,8 +138,30 @@
 #define	malloc_mutex_trylock JEMALLOC_N(malloc_mutex_trylock)
 #define	malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock)
 #define	malloc_printf JEMALLOC_N(malloc_printf)
+#define	malloc_snprintf JEMALLOC_N(malloc_snprintf)
+#define	malloc_tprintf JEMALLOC_N(malloc_tprintf)
+#define	malloc_vcprintf JEMALLOC_N(malloc_vcprintf)
+#define	malloc_vsnprintf JEMALLOC_N(malloc_vsnprintf)
+#define	malloc_vtprintf JEMALLOC_N(malloc_vtprintf)
 #define	malloc_write JEMALLOC_N(malloc_write)
 #define	mb_write JEMALLOC_N(mb_write)
+#define	opt_abort JEMALLOC_N(opt_abort)
+#define	opt_junk JEMALLOC_N(opt_junk)
+#define	opt_lg_chunk JEMALLOC_N(opt_lg_chunk)
+#define	opt_lg_dirty_mult JEMALLOC_N(opt_lg_dirty_mult)
+#define	opt_lg_prof_interval JEMALLOC_N(opt_lg_prof_interval)
+#define	opt_lg_prof_sample JEMALLOC_N(opt_lg_prof_sample)
+#define	opt_lg_tcache_max JEMALLOC_N(opt_lg_tcache_max)
+#define	opt_narenas JEMALLOC_N(opt_narenas)
+#define	opt_prof JEMALLOC_N(opt_prof)
+#define	opt_prof_accum JEMALLOC_N(opt_prof_accum)
+#define	opt_prof_active JEMALLOC_N(opt_prof_active)
+#define	opt_prof_gdump JEMALLOC_N(opt_prof_gdump)
+#define	opt_prof_leak JEMALLOC_N(opt_prof_leak)
+#define	opt_stats_print JEMALLOC_N(opt_stats_print)
+#define	opt_tcache JEMALLOC_N(opt_tcache)
+#define	opt_xmalloc JEMALLOC_N(opt_xmalloc)
+#define	opt_zero JEMALLOC_N(opt_zero)
 #define	pow2_ceil JEMALLOC_N(pow2_ceil)
 #define	prof_backtrace JEMALLOC_N(prof_backtrace)
 #define	prof_boot0 JEMALLOC_N(prof_boot0)
@@ -156,6 +179,7 @@
 #define	prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update)
 #define	prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update)
 #define	prof_tdata_init JEMALLOC_N(prof_tdata_init)
+#define	prof_tdata_tls JEMALLOC_N(prof_tdata_tls)
 #define	pthread_create JEMALLOC_N(pthread_create)
 #define	rtree_get JEMALLOC_N(rtree_get)
 #define	rtree_get_locked JEMALLOC_N(rtree_get_locked)
@@ -166,6 +190,7 @@
 #define	stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index)
 #define	stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index)
 #define	stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index)
+#define	stats_cactive JEMALLOC_N(stats_cactive)
 #define	stats_cactive_add JEMALLOC_N(stats_cactive_add)
 #define	stats_cactive_get JEMALLOC_N(stats_cactive_get)
 #define	stats_cactive_sub JEMALLOC_N(stats_cactive_sub)
@@ -185,6 +210,7 @@
 #define	tcache_event JEMALLOC_N(tcache_event)
 #define	tcache_get JEMALLOC_N(tcache_get)
 #define	tcache_stats_merge JEMALLOC_N(tcache_stats_merge)
+#define	tcache_tls JEMALLOC_N(tcache_tls)
 #define	thread_allocated_get JEMALLOC_N(thread_allocated_get)
 #define	thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard)
-#define	u2s JEMALLOC_N(u2s)
+#define	thread_allocated_tls JEMALLOC_N(thread_allocated_tls)
--- a/include/jemalloc/internal/stats.h
+++ b/include/jemalloc/internal/stats.h
@@ -1,8 +1,6 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES

-#define	UMAX2S_BUFSIZE	65
-
 typedef struct tcache_bin_stats_s tcache_bin_stats_t;
 typedef struct malloc_bin_stats_s malloc_bin_stats_t;
 typedef struct malloc_large_stats_s malloc_large_stats_t;
@@ -135,11 +133,6 @@ extern bool	opt_stats_print;

 extern size_t	stats_cactive;

-char	*u2s(uint64_t x, unsigned base, char *s);
-void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
-    const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4));
-void	malloc_printf(const char *format, ...)
-    JEMALLOC_ATTR(format(printf, 1, 2));
 void	stats_print(void (*write)(void *, const char *), void *cbopaque,
    const char *opts);

--- a/include/jemalloc/internal/util.h
+++ b/include/jemalloc/internal/util.h
@@ -0,0 +1,130 @@
+/******************************************************************************/
+#ifdef JEMALLOC_H_TYPES
+
+/* Size of stack-allocated buffer passed to buferror(). */
+#define	BUFERROR_BUF		64
+
+/*
+ * Define a custom assert() in order to reduce the chances of deadlock during
+ * assertion failure.
+ */
+#ifndef assert
+#define	assert(e) do {							\
+	if (config_debug && !(e)) {					\
+		malloc_printf(						\
+		    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n",	\
+		    __FILE__, __LINE__, #e);				\
+		abort();						\
+	}								\
+} while (0)
+#endif
+
+/* Use to assert a particular configuration, e.g., cassert(config_debug). */
+#define	cassert(c) do {							\
+	if ((c) == false)						\
+		assert(false);						\
+} while (0)
+
+#ifndef not_reached
+#define	not_reached() do {						\
+	if (config_debug) {						\
+		malloc_printf(						\
+		    "<jemalloc>: %s:%d: Unreachable code reached\n",	\
+		    __FILE__, __LINE__);				\
+		abort();						\
+	}								\
+} while (0)
+#endif
+
+#ifndef not_implemented
+#define	not_implemented() do {						\
+	if (config_debug) {						\
+		malloc_printf("<jemalloc>: %s:%d: Not implemented\n",	\
+		    __FILE__, __LINE__);				\
+		abort();						\
+	}								\
+} while (0)
+#endif
+
+#define	assert_not_implemented(e) do {					\
+	if (config_debug && !(e))					\
+		not_implemented();					\
+} while (0)
+
+#endif /* JEMALLOC_H_TYPES */
+/******************************************************************************/
+#ifdef JEMALLOC_H_STRUCTS
+
+#endif /* JEMALLOC_H_STRUCTS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_EXTERNS
+
+extern void	(*je_malloc_message)(void *wcbopaque, const char *s);
+
+int	buferror(int errnum, char *buf, size_t buflen);
+
+/*
+ * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
+ * point math.
+ */
+int	malloc_vsnprintf(char *str, size_t size, const char *format,
+    va_list ap);
+int	malloc_snprintf(char *str, size_t size, const char *format, ...)
+    JEMALLOC_ATTR(format(printf, 3, 4));
+/*
+ * malloc_[v]tprintf() prints to a thread-local string buffer, so the result is
+ * overwritten by the next call to malloc_[v]{,c,t}printf().
+ */
+const char *	malloc_vtprintf(const char *format, va_list ap);
+const char *	malloc_tprintf(const char *format, ...)
+    JEMALLOC_ATTR(format(printf, 1, 2));
+void	malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
+    const char *format, va_list ap);
+void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
+    const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4));
+void	malloc_printf(const char *format, ...)
+    JEMALLOC_ATTR(format(printf, 1, 2));
+
+#endif /* JEMALLOC_H_EXTERNS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_INLINES
+
+#ifndef JEMALLOC_ENABLE_INLINE
+size_t	pow2_ceil(size_t x);
+void	malloc_write(const char *s);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
+/* Compute the smallest power of 2 that is >= x. */
+JEMALLOC_INLINE size_t
+pow2_ceil(size_t x)
+{
+
+	x--;
+	x |= x >> 1;
+	x |= x >> 2;
+	x |= x >> 4;
+	x |= x >> 8;
+	x |= x >> 16;
+#if (LG_SIZEOF_PTR == 3)
+	x |= x >> 32;
+#endif
+	x++;
+	return (x);
+}
+
+/*
+ * Wrapper around malloc_message() that avoids the need for
+ * je_malloc_message(...) throughout the code.
+ */
+JEMALLOC_INLINE void
+malloc_write(const char *s)
+{
+
+	je_malloc_message(NULL, s);
+}
+
+#endif
+
+#endif /* JEMALLOC_H_INLINES */
+/******************************************************************************/
--- a/include/jemalloc/jemalloc_defs.h.in
+++ b/include/jemalloc/jemalloc_defs.h.in
@@ -166,3 +166,6 @@

 /* sizeof(long) == 2^LG_SIZEOF_LONG. */
 #undef LG_SIZEOF_LONG
+
+/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */
+#undef LG_SIZEOF_INTMAX_T