From c1baa0a9b7b05ebf98221dc7deb12c28e170a399 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 17 Nov 2016 13:36:17 -0800 Subject: [PATCH] Add huge page configuration and pages_[no}huge(). Add the --with-lg-hugepage configure option, but automatically configure LG_HUGEPAGE even if it isn't specified. Add the pages_[no]huge() functions, which toggle huge page state via madvise(..., MADV_[NO]HUGEPAGE) calls. --- INSTALL | 5 +++ Makefile.in | 1 + configure.ac | 44 +++++++++++++++++-- .../jemalloc/internal/jemalloc_internal.h.in | 7 +++ .../internal/jemalloc_internal_defs.h.in | 13 ++++++ include/jemalloc/internal/pages.h | 14 +++++- include/jemalloc/internal/private_symbols.txt | 2 + src/pages.c | 31 ++++++++++++- test/unit/pages.c | 30 +++++++++++++ 9 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 test/unit/pages.c diff --git a/INSTALL b/INSTALL index 6e593219..d7496612 100644 --- a/INSTALL +++ b/INSTALL @@ -227,6 +227,11 @@ any of the following arguments (not a definitive list) to 'configure': --with-lg-page, but its primary use case is for integration with FreeBSD's libc, wherein jemalloc is embedded. +--with-lg-hugepage= + Specify the base 2 log of the system huge page size. This option is useful + when cross compiling, or when overriding the default for systems that do + not explicitly support huge pages. + --with-lg-size-class-group= Specify the base 2 log of how many size classes to use for each doubling in size. By default jemalloc uses =2, which results in diff --git a/Makefile.in b/Makefile.in index 22b11349..052688bd 100644 --- a/Makefile.in +++ b/Makefile.in @@ -170,6 +170,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/mq.c \ $(srcroot)test/unit/mtx.c \ $(srcroot)test/unit/pack.c \ + $(srcroot)test/unit/pages.c \ $(srcroot)test/unit/ph.c \ $(srcroot)test/unit/prng.c \ $(srcroot)test/unit/prof_accum.c \ diff --git a/configure.ac b/configure.ac index bf4ea8f7..f886aeb5 100644 --- a/configure.ac +++ b/configure.ac @@ -1317,6 +1317,36 @@ else AC_MSG_ERROR([cannot determine value for LG_PAGE]) fi +AC_ARG_WITH([lg_hugepage], + [AS_HELP_STRING([--with-lg-hugepage=], + [Base 2 log of sytem huge page size])], + [je_cv_lg_hugepage="${with_lg_hugepage}"], + [je_cv_lg_hugepage=""]) +if test "x${je_cv_lg_hugepage}" = "x" ; then + dnl Look in /proc/meminfo (Linux-specific) for information on the default huge + dnl page size, if any. The relevant line looks like: + dnl + dnl Hugepagesize: 2048 kB + if test -e "/proc/meminfo" ; then + hpsk=[`cat /proc/meminfo 2>/dev/null | \ + grep -e '^Hugepagesize:[[:space:]]\+[0-9]\+[[:space:]]kB$' | \ + awk '{print $2}'`] + if test "x${hpsk}" != "x" ; then + je_cv_lg_hugepage=10 + while test "${hpsk}" -gt 1 ; do + hpsk="$((hpsk / 2))" + je_cv_lg_hugepage="$((je_cv_lg_hugepage + 1))" + done + fi + fi + + dnl Set default if unable to automatically configure. + if test "x${je_cv_lg_hugepage}" = "x" ; then + je_cv_lg_hugepage=21 + fi +fi +AC_DEFINE_UNQUOTED([LG_HUGEPAGE], [${je_cv_lg_hugepage}]) + AC_ARG_WITH([lg_page_sizes], [AS_HELP_STRING([--with-lg-page-sizes=], [Base 2 logs of system page sizes to support])], @@ -1690,6 +1720,8 @@ JE_COMPILABLE([madvise(2)], [ madvise((void *)0, 0, 0); ], [je_cv_madvise]) if test "x${je_cv_madvise}" = "xyes" ; then + AC_DEFINE([JEMALLOC_HAVE_MADVISE], [ ]) + dnl Check for madvise(..., MADV_FREE). JE_COMPILABLE([madvise(..., MADV_FREE)], [ #include @@ -1710,9 +1742,15 @@ if test "x${je_cv_madvise}" = "xyes" ; then AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ]) fi - if test "x${je_cv_madv_free}" = "xyes" \ - -o "x${je_cv_madv_dontneed}" = "xyes" ; then - AC_DEFINE([JEMALLOC_HAVE_MADVISE], [ ]) + dnl Check for madvise(..., MADV_[NO]HUGEPAGE). + JE_COMPILABLE([madvise(..., MADV_[[NO]]HUGEPAGE)], [ +#include +], [ + madvise((void *)0, 0, MADV_HUGEPAGE); + madvise((void *)0, 0, MADV_NOHUGEPAGE); +], [je_cv_thp]) + if test "x${je_cv_thp}" = "xyes" ; then + AC_DEFINE([JEMALLOC_THP], [ ]) fi fi diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index ba5207d8..bfa84a22 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -138,6 +138,13 @@ static const bool config_cache_oblivious = false #endif ; +static const bool have_thp = +#ifdef JEMALLOC_THP + true +#else + false +#endif + ; #if defined(JEMALLOC_C11ATOMICS) && !defined(__cplusplus) #include diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index aa0c0474..722c41dd 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -187,6 +187,13 @@ /* One page is 2^LG_PAGE bytes. */ #undef LG_PAGE +/* + * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the + * system does not explicitly support huge pages; system calls that require + * explicit huge page support are separately configured. + */ +#undef LG_HUGEPAGE + /* * If defined, adjacent virtual memory mappings with identical attributes * automatically coalesce, and they fragment when changes are made to subranges. @@ -262,6 +269,12 @@ #undef JEMALLOC_PURGE_MADVISE_FREE #undef JEMALLOC_PURGE_MADVISE_DONTNEED +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +#undef JEMALLOC_THP + /* Define if operating system has alloca.h header. */ #undef JEMALLOC_HAS_ALLOCA_H diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h index 16c657a0..034a8aac 100644 --- a/include/jemalloc/internal/pages.h +++ b/include/jemalloc/internal/pages.h @@ -7,15 +7,23 @@ #endif #define PAGE ((size_t)(1U << LG_PAGE)) #define PAGE_MASK ((size_t)(PAGE - 1)) - /* Return the page base address for the page containing address a. */ #define PAGE_ADDR2BASE(a) \ ((void *)((uintptr_t)(a) & ~PAGE_MASK)) - /* Return the smallest pagesize multiple that is >= s. */ #define PAGE_CEILING(s) \ (((s) + PAGE_MASK) & ~PAGE_MASK) +/* Huge page size. LG_HUGEPAGE is determined by the configure script. */ +#define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE)) +#define HUGEPAGE_MASK ((size_t)(HUGEPAGE - 1)) +/* Return the huge page base address for the huge page containing address a. */ +#define HUGEPAGE_ADDR2BASE(a) \ + ((void *)((uintptr_t)(a) & ~HUGEPAGE_MASK)) +/* Return the smallest pagesize multiple that is >= s. */ +#define HUGEPAGE_CEILING(s) \ + (((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK) + #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS @@ -31,6 +39,8 @@ void *pages_trim(void *addr, size_t alloc_size, size_t leadsize, bool pages_commit(void *addr, size_t size); bool pages_decommit(void *addr, size_t size); bool pages_purge(void *addr, size_t size); +bool pages_huge(void *addr, size_t size); +bool pages_nohuge(void *addr, size_t size); void pages_boot(void); #endif /* JEMALLOC_H_EXTERNS */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 63974880..1facc928 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -324,7 +324,9 @@ opt_zero pages_boot pages_commit pages_decommit +pages_huge pages_map +pages_nohuge pages_purge pages_trim pages_unmap diff --git a/src/pages.c b/src/pages.c index 6af228ac..8bef6fac 100644 --- a/src/pages.c +++ b/src/pages.c @@ -170,7 +170,8 @@ pages_purge(void *addr, size_t size) #ifdef _WIN32 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); unzeroed = true; -#elif defined(JEMALLOC_HAVE_MADVISE) +#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \ + defined(JEMALLOC_PURGE_MADVISE_DONTNEED)) # if defined(JEMALLOC_PURGE_MADVISE_FREE) # define JEMALLOC_MADV_PURGE MADV_FREE # define JEMALLOC_MADV_ZEROS false @@ -191,6 +192,34 @@ pages_purge(void *addr, size_t size) return (unzeroed); } +bool +pages_huge(void *addr, size_t size) +{ + + assert(HUGEPAGE_ADDR2BASE(addr) == addr); + assert(HUGEPAGE_CEILING(size) == size); + +#ifdef JEMALLOC_THP + return (madvise(addr, size, MADV_HUGEPAGE) != 0); +#else + return (true); +#endif +} + +bool +pages_nohuge(void *addr, size_t size) +{ + + assert(HUGEPAGE_ADDR2BASE(addr) == addr); + assert(HUGEPAGE_CEILING(size) == size); + +#ifdef JEMALLOC_THP + return (madvise(addr, size, MADV_NOHUGEPAGE) != 0); +#else + return (false); +#endif +} + #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT static bool os_overcommits_sysctl(void) diff --git a/test/unit/pages.c b/test/unit/pages.c new file mode 100644 index 00000000..f297215a --- /dev/null +++ b/test/unit/pages.c @@ -0,0 +1,30 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_pages_huge) +{ + size_t alloc_size; + bool commit; + void *pages, *hugepage; + + alloc_size = HUGEPAGE * 2 - PAGE; + commit = true; + pages = pages_map(NULL, alloc_size, &commit); + assert_ptr_not_null(pages, "Unexpected pages_map() error"); + + hugepage = (void *)(ALIGNMENT_CEILING((uintptr_t)pages, HUGEPAGE)); + assert_b_ne(pages_huge(hugepage, HUGEPAGE), have_thp, + "Unexpected pages_huge() result"); + assert_false(pages_nohuge(hugepage, HUGEPAGE), + "Unexpected pages_nohuge() result"); + + pages_unmap(pages, alloc_size); +} +TEST_END + +int +main(void) +{ + + return (test( + test_pages_huge)); +}