Add fxp: A fixed-point math library.

This will be used in the next commit to allow non-integer values for
narenas_ratio.

Makefile.in

@@ -118,6 +118,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/extent.c \
 	$(srcroot)src/extent_dss.c \
 	$(srcroot)src/extent_mmap.c \
+	$(srcroot)src/fxp.c \
 	$(srcroot)src/hook.c \
 	$(srcroot)src/hpa.c \
 	$(srcroot)src/hpa_central.c \
@@ -212,6 +213,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/extent_quantize.c \
 	${srcroot}test/unit/flat_bitmap.c \
 	$(srcroot)test/unit/fork.c \
+	${srcroot}test/unit/fxp.c \
 	$(srcroot)test/unit/hash.c \
 	$(srcroot)test/unit/hook.c \
 	$(srcroot)test/unit/hpa.c \

include/jemalloc/internal/fxp.h

@@ -0,0 +1,100 @@
+#ifndef JEMALLOC_INTERNAL_FXP_H
+#define JEMALLOC_INTERNAL_FXP_H
+
+/*
+ * A simple fixed-point math implementation, supporting only unsigned values
+ * (with overflow being an error).
+ *
+ * It's not in general safe to use floating point in core code, because various
+ * libc implementations we get linked against can assume that malloc won't touch
+ * floating point state and call it with an unusual calling convention.
+ */
+
+/*
+ * High 16 bits are the integer part, low 16 are the fractional part.  Or
+ * equivalently, repr == 2**16 * val, where we use "val" to refer to the
+ * (imaginary) fractional representation of the true value.
+ *
+ * We pick a uint32_t here since it's convenient in some places to
+ * double the representation size (i.e. multiplication and division use
+ * 64-bit integer types), and a uint64_t is the largest type we're
+ * certain is available.
+ */
+typedef uint32_t fxp_t;
+#define FXP_INIT_INT(x) ((x) << 16)
+
+/*
+ * Amount of precision used in parsing and printing numbers.  The integer bound
+ * is simply because the integer part of the number gets 16 bits, and so is
+ * bounded by 65536.
+ *
+ * We use a lot of precision for the fractional part, even though most of it
+ * gets rounded off; this lets us get exact values for the important special
+ * case where the denominator is a small power of 2 (for instance,
+ * 1/512 == 0.001953125 is exactly representable even with only 16 bits of
+ * fractional precision).  We need to left-shift by 16 before dividing by
+ * 10**precision, so we pick precision to be floor(log(2**48)) = 14.
+ */
+#define FXP_INTEGER_PART_DIGITS 5
+#define FXP_FRACTIONAL_PART_DIGITS 14
+
+/*
+ * In addition to the integer and fractional parts of the number, we need to
+ * include a null character and (possibly) a decimal point.
+ */
+#define FXP_BUF_SIZE (FXP_INTEGER_PART_DIGITS + FXP_FRACTIONAL_PART_DIGITS + 2)
+
+static inline fxp_t
+fxp_add(fxp_t a, fxp_t b) {
+	return a + b;
+}
+
+static inline fxp_t
+fxp_sub(fxp_t a, fxp_t b) {
+	assert(a >= b);
+	return a - b;
+}
+
+static inline fxp_t
+fxp_mul(fxp_t a, fxp_t b) {
+	uint64_t unshifted = (uint64_t)a * (uint64_t)b;
+	/*
+	 * Unshifted is (a.val * 2**16) * (b.val * 2**16)
+	 *   == (a.val * b.val) * 2**32, but we want
+	 * (a.val * b.val) * 2 ** 16.
+	 */
+	return (uint32_t)(unshifted >> 16);
+}
+
+static inline fxp_t
+fxp_div(fxp_t a, fxp_t b) {
+	assert(b != 0);
+	uint64_t unshifted = ((uint64_t)a << 32) / (uint64_t)b;
+	/*
+	 * Unshifted is (a.val * 2**16) * (2**32) / (b.val * 2**16)
+	 *   == (a.val / b.val) * (2 ** 32), which again corresponds to a right
+	 *   shift of 16.
+	 */
+	return (uint32_t)(unshifted >> 16);
+}
+
+static inline uint32_t
+fxp_round_down(fxp_t a) {
+	return a >> 16;
+}
+
+static inline uint32_t
+fxp_round_nearest(fxp_t a) {
+	uint32_t fractional_part = (a  & ((1U << 16) - 1));
+	uint32_t increment = (uint32_t)(fractional_part >= (1U << 15));
+	return (a >> 16) + increment;
+}
+
+/*
+ * Returns true on error.  Otherwise, returns false and updates *ptr to point to
+ * the first character not parsed (because it wasn't a digit).
+ */
+bool fxp_parse(fxp_t *a, const char *ptr, char **end);
+void fxp_print(fxp_t a, char buf[FXP_BUF_SIZE]);
+
+#endif /* JEMALLOC_INTERNAL_FXP_H */

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -58,6 +58,7 @@
     <ClCompile Include="..\..\..\..\src\extent.c" />
     <ClCompile Include="..\..\..\..\src\extent_dss.c" />
     <ClCompile Include="..\..\..\..\src\extent_mmap.c" />
+    <ClCompile Include="..\..\..\..\src\fxp.c" />
     <ClCompile Include="..\..\..\..\src\hook.c" />
     <ClCompile Include="..\..\..\..\src\hpa.c" />
     <ClCompile Include="..\..\..\..\src\hpa_central.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -58,6 +58,9 @@
     <ClCompile Include="..\..\..\..\src\extent_mmap.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\fxp.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\hook.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -58,6 +58,7 @@
     <ClCompile Include="..\..\..\..\src\extent.c" />
     <ClCompile Include="..\..\..\..\src\extent_dss.c" />
     <ClCompile Include="..\..\..\..\src\extent_mmap.c" />
+    <ClCompile Include="..\..\..\..\src\fxp.c" />
     <ClCompile Include="..\..\..\..\src\hook.c" />
     <ClCompile Include="..\..\..\..\src\hpa.c" />
     <ClCompile Include="..\..\..\..\src\hpa_central.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters

@@ -58,6 +58,9 @@
     <ClCompile Include="..\..\..\..\src\extent_mmap.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\fxp.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\hook.c">
       <Filter>Source Files</Filter>
     </ClCompile>

src/fxp.c

@@ -0,0 +1,124 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/fxp.h"
+
+static bool
+fxp_isdigit(char c) {
+	return '0' <= c && c <= '9';
+}
+
+bool
+fxp_parse(fxp_t *result, const char *str, char **end) {
+	/*
+	 * Using malloc_strtoumax in this method isn't as handy as you might
+	 * expect (I tried). In the fractional part, significant leading zeros
+	 * mean that you still need to do your own parsing, now with trickier
+	 * math.  In the integer part, the casting (uintmax_t to uint32_t)
+	 * forces more reasoning about bounds than just checking for overflow as
+	 * we parse.
+	 */
+	uint32_t integer_part = 0;
+
+	const char *cur = str;
+
+	/* The string must start with a digit or a decimal point. */
+	if (*cur != '.' && !fxp_isdigit(*cur)) {
+		return true;
+	}
+
+	while ('0' <= *cur && *cur <= '9') {
+		integer_part *= 10;
+		integer_part += *cur - '0';
+		if (integer_part >= (1U << 16)) {
+			return true;
+		}
+		cur++;
+	}
+
+	/*
+	 * We've parsed all digits at the beginning of the string, without
+	 * overflow.  Either we're done, or there's a fractional part.
+	 */
+	if (*cur != '.') {
+		*result = (integer_part << 16);
+		if (end != NULL) {
+			*end = (char *)cur;
+		}
+		return false;
+	}
+
+	/* There's a fractional part. */
+	cur++;
+	if (!fxp_isdigit(*cur)) {
+		/* Shouldn't end on the decimal point. */
+		return true;
+	}
+
+	/*
+	 * We use a lot of precision for the fractional part, even though we'll
+	 * discard most of it; this lets us get exact values for the important
+	 * special case where the denominator is a small power of 2 (for
+	 * instance, 1/512 == 0.001953125 is exactly representable even with
+	 * only 16 bits of fractional precision).  We need to left-shift by 16
+	 * before dividing so we pick the number of digits to be
+	 * floor(log(2**48)) = 14.
+	 */
+	uint64_t fractional_part = 0;
+	uint64_t frac_div = 1;
+	for (int i = 0; i < FXP_FRACTIONAL_PART_DIGITS; i++) {
+		fractional_part *= 10;
+		frac_div *= 10;
+		if (fxp_isdigit(*cur)) {
+			fractional_part += *cur - '0';
+			cur++;
+		}
+	}
+	/*
+	 * We only parse the first maxdigits characters, but we can still ignore
+	 * any digits after that.
+	 */
+	while (fxp_isdigit(*cur)) {
+		cur++;
+	}
+
+	assert(fractional_part < frac_div);
+	uint32_t fractional_repr = (uint32_t)(
+	    (fractional_part << 16) / frac_div);
+
+	/* Success! */
+	*result = (integer_part << 16) + fractional_repr;
+	if (end != NULL) {
+		*end = (char *)cur;
+	}
+	return false;
+}
+
+void
+fxp_print(fxp_t a, char buf[FXP_BUF_SIZE]) {
+	uint32_t integer_part = fxp_round_down(a);
+	uint32_t fractional_part = (a & ((1U << 16) - 1));
+
+	int leading_fraction_zeros = 0;
+	uint64_t fraction_digits = fractional_part;
+	for (int i = 0; i < FXP_FRACTIONAL_PART_DIGITS; i++) {
+		if (fraction_digits < (1U << 16)
+		    && fraction_digits * 10 >= (1U << 16)) {
+			leading_fraction_zeros = i;
+		}
+		fraction_digits *= 10;
+	}
+	fraction_digits >>= 16;
+	while (fraction_digits > 0 && fraction_digits % 10 == 0) {
+		fraction_digits /= 10;
+	}
+
+	size_t printed = malloc_snprintf(buf, FXP_BUF_SIZE, "%"FMTu32".",
+	    integer_part);
+	for (int i = 0; i < leading_fraction_zeros; i++) {
+		buf[printed] = '0';
+		printed++;
+	}
+	malloc_snprintf(&buf[printed], FXP_BUF_SIZE - printed, "%"FMTu64,
+	    fraction_digits);
+}

test/unit/fxp.c

@@ -0,0 +1,344 @@
+#include "test/jemalloc_test.h"
+
+#include "jemalloc/internal/fxp.h"
+
+static double
+fxp2double(fxp_t a) {
+	double intpart = (double)(a >> 16);
+	double fracpart = (double)(a & ((1U << 16) - 1)) / (1U << 16);
+	return intpart + fracpart;
+}
+
+/* Is a close to b? */
+static bool
+double_close(double a, double b) {
+	/*
+	 * Our implementation doesn't try for precision.  Correspondingly, don't
+	 * enforce it too strenuously here; accept values that are close in
+	 * either relative or absolute terms.
+	 */
+	return fabs(a - b) < 0.01 || fabs(a - b) / a < 0.01;
+}
+
+static bool
+fxp_close(fxp_t a, fxp_t b) {
+	return double_close(fxp2double(a), fxp2double(b));
+}
+
+static fxp_t
+xparse_fxp(const char *str) {
+	fxp_t result;
+	bool err = fxp_parse(&result, str, NULL);
+	assert_false(err, "Invalid fxp string: %s", str);
+	return result;
+}
+
+static void
+expect_parse_accurate(const char *str, const char *parse_str) {
+	double true_val = strtod(str, NULL);
+	fxp_t fxp_val;
+	char *end;
+	bool err = fxp_parse(&fxp_val, parse_str, &end);
+	expect_false(err, "Unexpected parse failure");
+	expect_ptr_eq(parse_str + strlen(str), end,
+	    "Didn't parse whole string");
+	expect_true(double_close(fxp2double(fxp_val), true_val),
+	    "Misparsed %s", str);
+}
+
+static void
+parse_valid_trial(const char *str) {
+	/* The value it parses should be correct. */
+	expect_parse_accurate(str, str);
+	char buf[100];
+	snprintf(buf, sizeof(buf), "%swith_some_trailing_text", str);
+	expect_parse_accurate(str, buf);
+	snprintf(buf, sizeof(buf), "%s with a space", str);
+	expect_parse_accurate(str, buf);
+	snprintf(buf, sizeof(buf), "%s,in_a_malloc_conf_string:1", str);
+	expect_parse_accurate(str, buf);
+}
+
+TEST_BEGIN(test_parse_valid) {
+	parse_valid_trial("0");
+	parse_valid_trial("1");
+	parse_valid_trial("2");
+	parse_valid_trial("100");
+	parse_valid_trial("345");
+	parse_valid_trial("00000000123");
+	parse_valid_trial("00000000987");
+
+	parse_valid_trial("0.0");
+	parse_valid_trial("0.00000000000456456456");
+	parse_valid_trial("100.00000000000456456456");
+
+	parse_valid_trial("123.1");
+	parse_valid_trial("123.01");
+	parse_valid_trial("123.001");
+	parse_valid_trial("123.0001");
+	parse_valid_trial("123.00001");
+	parse_valid_trial("123.000001");
+	parse_valid_trial("123.0000001");
+
+	parse_valid_trial(".0");
+	parse_valid_trial(".1");
+	parse_valid_trial(".01");
+	parse_valid_trial(".001");
+	parse_valid_trial(".0001");
+	parse_valid_trial(".00001");
+	parse_valid_trial(".000001");
+
+	parse_valid_trial(".1");
+	parse_valid_trial(".10");
+	parse_valid_trial(".100");
+	parse_valid_trial(".1000");
+	parse_valid_trial(".100000");
+}
+TEST_END
+
+static void expect_parse_failure(const char *str) {
+	fxp_t result = FXP_INIT_INT(333);
+	char *end = (void *)0x123;
+	bool err = fxp_parse(&result, str, &end);
+	expect_true(err, "Expected a parse error on: %s", str);
+	expect_ptr_eq((void *)0x123, end,
+	    "Parse error shouldn't change results");
+	expect_u32_eq(result, FXP_INIT_INT(333),
+	    "Parse error shouldn't change results");
+}
+
+TEST_BEGIN(test_parse_invalid) {
+	expect_parse_failure("123.");
+	expect_parse_failure("3.a");
+	expect_parse_failure(".a");
+	expect_parse_failure("a.1");
+	expect_parse_failure("a");
+	/* A valid string, but one that overflows. */
+	expect_parse_failure("123456789");
+	expect_parse_failure("0000000123456789");
+	expect_parse_failure("1000000");
+}
+TEST_END
+
+static void
+expect_add(const char *astr, const char *bstr, const char* resultstr) {
+	fxp_t a = xparse_fxp(astr);
+	fxp_t b = xparse_fxp(bstr);
+	fxp_t result = xparse_fxp(resultstr);
+	expect_true(fxp_close(fxp_add(a, b), result),
+	    "Expected %s + %s == %s", astr, bstr, resultstr);
+}
+
+TEST_BEGIN(test_add_simple) {
+	expect_add("0", "0", "0");
+	expect_add("0", "1", "1");
+	expect_add("1", "1", "2");
+	expect_add("1.5", "1.5", "3");
+	expect_add("0.1", "0.1", "0.2");
+	expect_add("123", "456", "579");
+}
+TEST_END
+
+static void
+expect_sub(const char *astr, const char *bstr, const char* resultstr) {
+	fxp_t a = xparse_fxp(astr);
+	fxp_t b = xparse_fxp(bstr);
+	fxp_t result = xparse_fxp(resultstr);
+	expect_true(fxp_close(fxp_sub(a, b), result),
+	    "Expected %s - %s == %s", astr, bstr, resultstr);
+}
+
+TEST_BEGIN(test_sub_simple) {
+	expect_sub("0", "0", "0");
+	expect_sub("1", "0", "1");
+	expect_sub("1", "1", "0");
+	expect_sub("3.5", "1.5", "2");
+	expect_sub("0.3", "0.1", "0.2");
+	expect_sub("456", "123", "333");
+}
+TEST_END
+
+static void
+expect_mul(const char *astr, const char *bstr, const char* resultstr) {
+	fxp_t a = xparse_fxp(astr);
+	fxp_t b = xparse_fxp(bstr);
+	fxp_t result = xparse_fxp(resultstr);
+	expect_true(fxp_close(fxp_mul(a, b), result),
+	    "Expected %s * %s == %s", astr, bstr, resultstr);
+}
+
+TEST_BEGIN(test_mul_simple) {
+	expect_mul("0", "0", "0");
+	expect_mul("1", "0", "0");
+	expect_mul("1", "1", "1");
+	expect_mul("1.5", "1.5", "2.25");
+	expect_mul("100.0", "10", "1000");
+	expect_mul(".1", "10", "1");
+}
+TEST_END
+
+static void
+expect_div(const char *astr, const char *bstr, const char* resultstr) {
+	fxp_t a = xparse_fxp(astr);
+	fxp_t b = xparse_fxp(bstr);
+	fxp_t result = xparse_fxp(resultstr);
+	expect_true(fxp_close(fxp_div(a, b), result),
+	    "Expected %s / %s == %s", astr, bstr, resultstr);
+}
+
+TEST_BEGIN(test_div_simple) {
+	expect_div("1", "1", "1");
+	expect_div("0", "1", "0");
+	expect_div("2", "1", "2");
+	expect_div("3", "2", "1.5");
+	expect_div("3", "1.5", "2");
+	expect_div("10", ".1", "100");
+	expect_div("123", "456", ".2697368421");
+}
+TEST_END
+
+static void
+expect_round(const char *str, uint32_t rounded_down, uint32_t rounded_nearest) {
+	fxp_t fxp = xparse_fxp(str);
+	uint32_t fxp_rounded_down = fxp_round_down(fxp);
+	uint32_t fxp_rounded_nearest = fxp_round_nearest(fxp);
+	expect_u32_eq(rounded_down, fxp_rounded_down,
+	    "Mistake rounding %s down", str);
+	expect_u32_eq(rounded_nearest, fxp_rounded_nearest,
+	    "Mistake rounding %s to nearest", str);
+}
+
+TEST_BEGIN(test_round_simple) {
+	expect_round("1.5", 1, 2);
+	expect_round("0", 0, 0);
+	expect_round("0.1", 0, 0);
+	expect_round("0.4", 0, 0);
+	expect_round("0.40000", 0, 0);
+	expect_round("0.5", 0, 1);
+	expect_round("0.6", 0, 1);
+	expect_round("123", 123, 123);
+	expect_round("123.4", 123, 123);
+	expect_round("123.5", 123, 124);
+}
+TEST_END
+
+static void
+expect_print(const char *str) {
+	fxp_t fxp = xparse_fxp(str);
+	char buf[FXP_BUF_SIZE];
+	fxp_print(fxp, buf);
+	expect_d_eq(0, strcmp(str, buf), "Couldn't round-trip print %s", str);
+}
+
+TEST_BEGIN(test_print_simple) {
+	expect_print("0.0");
+	expect_print("1.0");
+	expect_print("2.0");
+	expect_print("123.0");
+	/*
+	 * We hit the possibility of roundoff errors whenever the fractional
+	 * component isn't a round binary number; only check these here (we
+	 * round-trip properly in the stress test).
+	 */
+	expect_print("1.5");
+	expect_print("3.375");
+	expect_print("0.25");
+	expect_print("0.125");
+	/* 1 / 2**14 */
+	expect_print("0.00006103515625");
+}
+TEST_END
+
+TEST_BEGIN(test_stress) {
+	const char *numbers[] = {
+		"0.0", "0.1", "0.2", "0.3", "0.4",
+		"0.5", "0.6", "0.7", "0.8", "0.9",
+
+		"1.0", "1.1", "1.2", "1.3", "1.4",
+		"1.5", "1.6", "1.7", "1.8", "1.9",
+
+		"2.0", "2.1", "2.2", "2.3", "2.4",
+		"2.5", "2.6", "2.7", "2.8", "2.9",
+
+		"17.0", "17.1", "17.2", "17.3", "17.4",
+		"17.5", "17.6", "17.7", "17.8", "17.9",
+
+		"18.0", "18.1", "18.2", "18.3", "18.4",
+		"18.5", "18.6", "18.7", "18.8", "18.9",
+
+		"123.0", "123.1", "123.2", "123.3", "123.4",
+		"123.5", "123.6", "123.7", "123.8", "123.9",
+
+		"124.0", "124.1", "124.2", "124.3", "124.4",
+		"124.5", "124.6", "124.7", "124.8", "124.9",
+
+		"125.0", "125.1", "125.2", "125.3", "125.4",
+		"125.5", "125.6", "125.7", "125.8", "125.9"};
+	size_t numbers_len = sizeof(numbers)/sizeof(numbers[0]);
+	for (size_t i = 0; i < numbers_len; i++) {
+		fxp_t fxp_a = xparse_fxp(numbers[i]);
+		double double_a = strtod(numbers[i], NULL);
+
+		uint32_t fxp_rounded_down = fxp_round_down(fxp_a);
+		uint32_t fxp_rounded_nearest = fxp_round_nearest(fxp_a);
+		uint32_t double_rounded_down = (uint32_t)double_a;
+		uint32_t double_rounded_nearest = (uint32_t)round(double_a);
+
+		expect_u32_eq(double_rounded_down, fxp_rounded_down,
+		    "Incorrectly rounded down %s", numbers[i]);
+		expect_u32_eq(double_rounded_nearest, fxp_rounded_nearest,
+		    "Incorrectly rounded-to-nearest %s", numbers[i]);
+
+		for (size_t j = 0; j < numbers_len; j++) {
+			fxp_t fxp_b = xparse_fxp(numbers[j]);
+			double double_b = strtod(numbers[j], NULL);
+
+			fxp_t fxp_sum = fxp_add(fxp_a, fxp_b);
+			double double_sum = double_a + double_b;
+			expect_true(
+			    double_close(fxp2double(fxp_sum), double_sum),
+			    "Miscomputed %s + %s", numbers[i], numbers[j]);
+
+			if (double_a > double_b) {
+				fxp_t fxp_diff = fxp_sub(fxp_a, fxp_b);
+				double double_diff = double_a - double_b;
+				expect_true(
+				    double_close(fxp2double(fxp_diff),
+				    double_diff),
+				    "Miscomputed %s - %s", numbers[i],
+				    numbers[j]);
+			}
+
+			fxp_t fxp_prod = fxp_mul(fxp_a, fxp_b);
+			double double_prod = double_a * double_b;
+			expect_true(
+			    double_close(fxp2double(fxp_prod), double_prod),
+			    "Miscomputed %s * %s", numbers[i], numbers[j]);
+
+			if (double_b != 0.0) {
+				fxp_t fxp_quot = fxp_div(fxp_a, fxp_b);
+				double double_quot = double_a / double_b;
+				expect_true(
+				    double_close(fxp2double(fxp_quot),
+				    double_quot),
+				    "Miscomputed %s / %s", numbers[i],
+				    numbers[j]);
+			}
+		}
+	}
+}
+TEST_END
+
+int
+main(void) {
+	return test_no_reentrancy(
+	    test_parse_valid,
+	    test_parse_invalid,
+	    test_add_simple,
+	    test_sub_simple,
+	    test_mul_simple,
+	    test_div_simple,
+	    test_round_simple,
+	    test_print_simple,
+	    test_stress);
+}