project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add compact json option for emitter

Browse Source 
JSON format is largely meant for machine-machine communication, so
adding the option to the emitter.  According to local testing, the
savings in terms of bytes outputted is around 50% for stats printing
and around 25% for prof log printing.
This commit is contained in:
Yinan Zhang 2019-07-30 11:07:24 -07:00
parent 7fc6b1b259
commit 8c8466fa6e
2 changed files with 160 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
include/jemalloc/internal/emitter.h
View File

@ -6,6 +6,7 @@
typedef enum emitter_output_e emitter_output_t; typedef enum emitter_output_e emitter_output_t;
enum emitter_output_e { enum emitter_output_e {
emitter_output_json, emitter_output_json,
emitter_output_json_compact,
emitter_output_table emitter_output_table
}; };
@ -75,6 +76,12 @@ struct emitter_s {
bool emitted_key; bool emitted_key;
}; };
static inline bool
emitter_outputs_json(emitter_t *emitter) {
return emitter->output == emitter_output_json ||
emitter->output == emitter_output_json_compact;
}
/* Internal convenience function. Write to the emitter the given string. */ /* Internal convenience function. Write to the emitter the given string. */
JEMALLOC_FORMAT_PRINTF(2, 3) JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void static inline void
@ -135,7 +142,7 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
switch (value_type) { switch (value_type) {
case emitter_type_bool: case emitter_type_bool:
emitter_printf(emitter, emitter_printf(emitter,
emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width),
*(const bool *)value ? "true" : "false"); *(const bool *)value ? "true" : "false");
break; break;
@ -159,7 +166,7 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
* anywhere near the fmt size. * anywhere near the fmt size.
*/ */
assert(str_written < BUF_SIZE); assert(str_written < BUF_SIZE);
emitter_printf(emitter, emitter_printf(emitter,
emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), buf); emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), buf);
break; break;
case emitter_type_uint32: case emitter_type_uint32:
@ -196,6 +203,7 @@ static inline void
emitter_indent(emitter_t *emitter) { emitter_indent(emitter_t *emitter) {
int amount = emitter->nesting_depth; int amount = emitter->nesting_depth;
const char *indent_str; const char *indent_str;
assert(emitter->output != emitter_output_json_compact);
if (emitter->output == emitter_output_json) { if (emitter->output == emitter_output_json) {
indent_str = "\t"; indent_str = "\t";
} else { } else {
@ -209,12 +217,18 @@ emitter_indent(emitter_t *emitter) {
static inline void static inline void
emitter_json_key_prefix(emitter_t *emitter) { emitter_json_key_prefix(emitter_t *emitter) {
assert(emitter_outputs_json(emitter));
if (emitter->emitted_key) { if (emitter->emitted_key) {
emitter->emitted_key = false; emitter->emitted_key = false;
return; return;
} }
emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : ""); if (emitter->item_at_depth) {
emitter_indent(emitter); emitter_printf(emitter, ",");
}
if (emitter->output != emitter_output_json_compact) {
emitter_printf(emitter, "\n");
emitter_indent(emitter);
}
} }
/******************************************************************************/ /******************************************************************************/
@ -227,22 +241,23 @@ emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
emitter->write_cb = write_cb; emitter->write_cb = write_cb;
emitter->cbopaque = cbopaque; emitter->cbopaque = cbopaque;
emitter->item_at_depth = false; emitter->item_at_depth = false;
emitter->emitted_key = false; emitter->emitted_key = false;
emitter->nesting_depth = 0; emitter->nesting_depth = 0;
} }
/******************************************************************************/ /******************************************************************************/
/* JSON public API. */ /* JSON public API. */
/* /*
* Emits a key (e.g. as appears in an object). The next json entity emitted will * Emits a key (e.g. as appears in an object). The next json entity emitted will
* be the corresponding value. * be the corresponding value.
*/ */
static inline void static inline void
emitter_json_key(emitter_t *emitter, const char *json_key) { emitter_json_key(emitter_t *emitter, const char *json_key) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
emitter_json_key_prefix(emitter); emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": ", json_key); emitter_printf(emitter, "\"%s\":%s", json_key,
emitter->output == emitter_output_json_compact ? "" : " ");
emitter->emitted_key = true; emitter->emitted_key = true;
} }
} }
@ -250,7 +265,7 @@ emitter_json_key(emitter_t *emitter, const char *json_key) {
static inline void static inline void
emitter_json_value(emitter_t *emitter, emitter_type_t value_type, emitter_json_value(emitter_t *emitter, emitter_type_t value_type,
const void *value) { const void *value) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
emitter_json_key_prefix(emitter); emitter_json_key_prefix(emitter);
emitter_print_value(emitter, emitter_justify_none, -1, emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value); value_type, value);
@ -268,7 +283,7 @@ emitter_json_kv(emitter_t *emitter, const char *json_key,
static inline void static inline void
emitter_json_array_begin(emitter_t *emitter) { emitter_json_array_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
emitter_json_key_prefix(emitter); emitter_json_key_prefix(emitter);
emitter_printf(emitter, "["); emitter_printf(emitter, "[");
emitter_nest_inc(emitter); emitter_nest_inc(emitter);
@ -284,18 +299,20 @@ emitter_json_array_kv_begin(emitter_t *emitter, const char *json_key) {
static inline void static inline void
emitter_json_array_end(emitter_t *emitter) { emitter_json_array_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
assert(emitter->nesting_depth > 0); assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter); emitter_nest_dec(emitter);
emitter_printf(emitter, "\n"); if (emitter->output != emitter_output_json_compact) {
emitter_indent(emitter); emitter_printf(emitter, "\n");
emitter_indent(emitter);
}
emitter_printf(emitter, "]"); emitter_printf(emitter, "]");
} }
} }
static inline void static inline void
emitter_json_object_begin(emitter_t *emitter) { emitter_json_object_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
emitter_json_key_prefix(emitter); emitter_json_key_prefix(emitter);
emitter_printf(emitter, "{"); emitter_printf(emitter, "{");
emitter_nest_inc(emitter); emitter_nest_inc(emitter);
@ -311,11 +328,13 @@ emitter_json_object_kv_begin(emitter_t *emitter, const char *json_key) {
static inline void static inline void
emitter_json_object_end(emitter_t *emitter) { emitter_json_object_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
assert(emitter->nesting_depth > 0); assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter); emitter_nest_dec(emitter);
emitter_printf(emitter, "\n"); if (emitter->output != emitter_output_json_compact) {
emitter_indent(emitter); emitter_printf(emitter, "\n");
emitter_indent(emitter);
}
emitter_printf(emitter, "}"); emitter_printf(emitter, "}");
} }
} }
@ -420,7 +439,7 @@ emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_type_t value_type, const void *value, emitter_type_t value_type, const void *value,
const char *table_note_key, emitter_type_t table_note_value_type, const char *table_note_key, emitter_type_t table_note_value_type,
const void *table_note_value) { const void *table_note_value) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
emitter_json_key(emitter, json_key); emitter_json_key(emitter, json_key);
emitter_json_value(emitter, value_type, value); emitter_json_value(emitter, value_type, value);
} else { } else {
@ -440,7 +459,7 @@ emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key,
static inline void static inline void
emitter_dict_begin(emitter_t *emitter, const char *json_key, emitter_dict_begin(emitter_t *emitter, const char *json_key,
const char *table_header) { const char *table_header) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
emitter_json_key(emitter, json_key); emitter_json_key(emitter, json_key);
emitter_json_object_begin(emitter); emitter_json_object_begin(emitter);
} else { } else {
@ -450,7 +469,7 @@ emitter_dict_begin(emitter_t *emitter, const char *json_key,
static inline void static inline void
emitter_dict_end(emitter_t *emitter) { emitter_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
emitter_json_object_end(emitter); emitter_json_object_end(emitter);
} else { } else {
emitter_table_dict_end(emitter); emitter_table_dict_end(emitter);
@ -459,7 +478,7 @@ emitter_dict_end(emitter_t *emitter) {
static inline void static inline void
emitter_begin(emitter_t *emitter) { emitter_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
assert(emitter->nesting_depth == 0); assert(emitter->nesting_depth == 0);
emitter_printf(emitter, "{"); emitter_printf(emitter, "{");
emitter_nest_inc(emitter); emitter_nest_inc(emitter);
@ -476,10 +495,11 @@ emitter_begin(emitter_t *emitter) {
static inline void static inline void
emitter_end(emitter_t *emitter) { emitter_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) { if (emitter_outputs_json(emitter)) {
assert(emitter->nesting_depth == 1); assert(emitter->nesting_depth == 1);
emitter_nest_dec(emitter); emitter_nest_dec(emitter);
emitter_printf(emitter, "\n}\n"); emitter_printf(emitter, "%s", emitter->output ==
emitter_output_json_compact ? "}" : "\n}\n");
} }
} }

170
test/unit/emitter.c
View File

@ -66,7 +66,9 @@ forwarding_cb(void *buf_descriptor_v, const char *str) {
static void static void
assert_emit_output(void (*emit_fn)(emitter_t *), assert_emit_output(void (*emit_fn)(emitter_t *),
const char *expected_json_output, const char *expected_table_output) { const char *expected_json_output,
const char *expected_json_compact_output,
const char *expected_table_output) {
emitter_t emitter; emitter_t emitter;
char buf[MALLOC_PRINTF_BUFSIZE]; char buf[MALLOC_PRINTF_BUFSIZE];
buf_descriptor_t buf_descriptor; buf_descriptor_t buf_descriptor;
@ -84,6 +86,16 @@ assert_emit_output(void (*emit_fn)(emitter_t *),
buf_descriptor.len = MALLOC_PRINTF_BUFSIZE; buf_descriptor.len = MALLOC_PRINTF_BUFSIZE;
buf_descriptor.mid_quote = false; buf_descriptor.mid_quote = false;
emitter_init(&emitter, emitter_output_json_compact, &forwarding_cb,
&buf_descriptor);
(*emit_fn)(&emitter);
assert_str_eq(expected_json_compact_output, buf,
"compact json output failure");
buf_descriptor.buf = buf;
buf_descriptor.len = MALLOC_PRINTF_BUFSIZE;
buf_descriptor.mid_quote = false;
emitter_init(&emitter, emitter_output_table, &forwarding_cb, emitter_init(&emitter, emitter_output_table, &forwarding_cb,
&buf_descriptor); &buf_descriptor);
(*emit_fn)(&emitter); (*emit_fn)(&emitter);
@ -108,6 +120,7 @@ emit_dict(emitter_t *emitter) {
emitter_dict_end(emitter); emitter_dict_end(emitter);
emitter_end(emitter); emitter_end(emitter);
} }
static const char *dict_json = static const char *dict_json =
"{\n" "{\n"
"\t\"foo\": {\n" "\t\"foo\": {\n"
@ -117,6 +130,15 @@ static const char *dict_json =
"\t\t\"jkl\": \"a string\"\n" "\t\t\"jkl\": \"a string\"\n"
"\t}\n" "\t}\n"
"}\n"; "}\n";
static const char *dict_json_compact =
"{"
"\"foo\":{"
"\"abc\":false,"
"\"def\":true,"
"\"ghi\":123,"
"\"jkl\":\"a string\""
"}"
"}";
static const char *dict_table = static const char *dict_table =
"This is the foo table:\n" "This is the foo table:\n"
" ABC: false\n" " ABC: false\n"
@ -124,11 +146,6 @@ static const char *dict_table =
" GHI: 123 (note_key1: \"a string\")\n" " GHI: 123 (note_key1: \"a string\")\n"
" JKL: \"a string\" (note_key2: false)\n"; " JKL: \"a string\" (note_key2: false)\n";
TEST_BEGIN(test_dict) {
assert_emit_output(&emit_dict, dict_json, dict_table);
}
TEST_END
static void static void
emit_table_printf(emitter_t *emitter) { emit_table_printf(emitter_t *emitter) {
emitter_begin(emitter); emitter_begin(emitter);
@ -141,17 +158,11 @@ emit_table_printf(emitter_t *emitter) {
static const char *table_printf_json = static const char *table_printf_json =
"{\n" "{\n"
"}\n"; "}\n";
static const char *table_printf_json_compact = "{}";
static const char *table_printf_table = static const char *table_printf_table =
"Table note 1\n" "Table note 1\n"
"Table note 2 with format string\n"; "Table note 2 with format string\n";
TEST_BEGIN(test_table_printf) {
assert_emit_output(&emit_table_printf, table_printf_json,
table_printf_table);
}
TEST_END
static void emit_nested_dict(emitter_t *emitter) { static void emit_nested_dict(emitter_t *emitter) {
int val = 123; int val = 123;
emitter_begin(emitter); emitter_begin(emitter);
@ -169,7 +180,7 @@ static void emit_nested_dict(emitter_t *emitter) {
emitter_end(emitter); emitter_end(emitter);
} }
static const char *nested_object_json = static const char *nested_dict_json =
"{\n" "{\n"
"\t\"json1\": {\n" "\t\"json1\": {\n"
"\t\t\"json2\": {\n" "\t\t\"json2\": {\n"
@ -182,8 +193,20 @@ static const char *nested_object_json =
"\t\t\"primitive\": 123\n" "\t\t\"primitive\": 123\n"
"\t}\n" "\t}\n"
"}\n"; "}\n";
static const char *nested_dict_json_compact =
static const char *nested_object_table = "{"
"\"json1\":{"
"\"json2\":{"
"\"primitive\":123"
"},"
"\"json3\":{"
"}"
"},"
"\"json4\":{"
"\"primitive\":123"
"}"
"}";
static const char *nested_dict_table =
"Dict 1\n" "Dict 1\n"
" Dict 2\n" " Dict 2\n"
" A primitive: 123\n" " A primitive: 123\n"
@ -191,12 +214,6 @@ static const char *nested_object_table =
"Dict 4\n" "Dict 4\n"
" Another primitive: 123\n"; " Another primitive: 123\n";
TEST_BEGIN(test_nested_dict) {
assert_emit_output(&emit_nested_dict, nested_object_json,
nested_object_table);
}
TEST_END
static void static void
emit_types(emitter_t *emitter) { emit_types(emitter_t *emitter) {
bool b = false; bool b = false;
@ -235,7 +252,17 @@ static const char *types_json =
"\t\"k7\": 789,\n" "\t\"k7\": 789,\n"
"\t\"k8\": 10000000000\n" "\t\"k8\": 10000000000\n"
"}\n"; "}\n";
static const char *types_json_compact =
"{"
"\"k1\":false,"
"\"k2\":-123,"
"\"k3\":123,"
"\"k4\":-456,"
"\"k5\":456,"
"\"k6\":\"string\","
"\"k7\":789,"
"\"k8\":10000000000"
"}";
static const char *types_table = static const char *types_table =
"K1: false\n" "K1: false\n"
"K2: -123\n" "K2: -123\n"
@ -246,11 +273,6 @@ static const char *types_table =
"K7: 789\n" "K7: 789\n"
"K8: 10000000000\n"; "K8: 10000000000\n";
TEST_BEGIN(test_types) {
assert_emit_output(&emit_types, types_json, types_table);
}
TEST_END
static void static void
emit_modal(emitter_t *emitter) { emit_modal(emitter_t *emitter) {
int val = 123; int val = 123;
@ -283,7 +305,18 @@ const char *modal_json =
"\t\t\"i6\": 123\n" "\t\t\"i6\": 123\n"
"\t}\n" "\t}\n"
"}\n"; "}\n";
const char *modal_json_compact =
"{"
"\"j0\":{"
"\"j1\":{"
"\"i1\":123,"
"\"i2\":123,"
"\"i4\":123"
"},"
"\"i5\":123,"
"\"i6\":123"
"}"
"}";
const char *modal_table = const char *modal_table =
"T0\n" "T0\n"
" I1: 123\n" " I1: 123\n"
@ -293,13 +326,8 @@ const char *modal_table =
" I5: 123\n" " I5: 123\n"
" I6: 123\n"; " I6: 123\n";
TEST_BEGIN(test_modal) {
assert_emit_output(&emit_modal, modal_json, modal_table);
}
TEST_END
static void static void
emit_json_arr(emitter_t *emitter) { emit_json_array(emitter_t *emitter) {
int ival = 123; int ival = 123;
emitter_begin(emitter); emitter_begin(emitter);
@ -338,14 +366,24 @@ static const char *json_array_json =
"\t\t]\n" "\t\t]\n"
"\t}\n" "\t}\n"
"}\n"; "}\n";
static const char *json_array_json_compact =
"{"
"\"dict\":{"
"\"arr\":["
"{"
"\"foo\":123"
"},"
"123,"
"123,"
"{"
"\"bar\":123,"
"\"baz\":123"
"}"
"]"
"}"
"}";
static const char *json_array_table = ""; static const char *json_array_table = "";
TEST_BEGIN(test_json_arr) {
assert_emit_output(&emit_json_arr, json_array_json, json_array_table);
}
TEST_END
static void static void
emit_json_nested_array(emitter_t *emitter) { emit_json_nested_array(emitter_t *emitter) {
int ival = 123; int ival = 123;
@ -391,12 +429,27 @@ static const char *json_nested_array_json =
"\t\t]\n" "\t\t]\n"
"\t]\n" "\t]\n"
"}\n"; "}\n";
static const char *json_nested_array_json_compact =
TEST_BEGIN(test_json_nested_arr) { "{"
assert_emit_output(&emit_json_nested_array, json_nested_array_json, "["
json_array_table); "["
} "123,"
TEST_END "\"foo\","
"123,"
"\"foo\""
"],"
"["
"123"
"],"
"["
"\"foo\","
"123"
"],"
"["
"]"
"]"
"}";
static const char *json_nested_array_table = "";
static void static void
emit_table_row(emitter_t *emitter) { emit_table_row(emitter_t *emitter) {
@ -443,18 +496,29 @@ emit_table_row(emitter_t *emitter) {
static const char *table_row_json = static const char *table_row_json =
"{\n" "{\n"
"}\n"; "}\n";
static const char *table_row_json_compact = "{}";
static const char *table_row_table = static const char *table_row_table =
"ABC title DEF title GHI\n" "ABC title DEF title GHI\n"
"123 true 456\n" "123 true 456\n"
"789 false 1011\n" "789 false 1011\n"
"\"a string\" false ghi\n"; "\"a string\" false ghi\n";
TEST_BEGIN(test_table_row) { #define GENERATE_TEST(feature) \
assert_emit_output(&emit_table_row, table_row_json, table_row_table); TEST_BEGIN(test_##feature) { \
} assert_emit_output(emit_##feature, feature##_json, \
feature##_json_compact, feature##_table); \
} \
TEST_END TEST_END
GENERATE_TEST(dict)
GENERATE_TEST(table_printf)
GENERATE_TEST(nested_dict)
GENERATE_TEST(types)
GENERATE_TEST(modal)
GENERATE_TEST(json_array)
GENERATE_TEST(json_nested_array)
GENERATE_TEST(table_row)
int int
main(void) { main(void) {
return test_no_reentrancy( return test_no_reentrancy(
@ -463,7 +527,7 @@ main(void) {
test_nested_dict, test_nested_dict,
test_types, test_types,
test_modal, test_modal,
test_json_arr, test_json_array,
test_json_nested_arr, test_json_nested_array,
test_table_row); test_table_row);
} }