Add huge page configuration and pages_[no}huge().

Add the --with-lg-hugepage configure option, but automatically configure
LG_HUGEPAGE even if it isn't specified.

Add the pages_[no]huge() functions, which toggle huge page state via
madvise(..., MADV_[NO]HUGEPAGE) calls.
This commit is contained in:
Jason Evans 2016-11-17 13:36:17 -08:00
parent eab3b180e5
commit c1baa0a9b7
9 changed files with 141 additions and 6 deletions

View File

@ -227,6 +227,11 @@ any of the following arguments (not a definitive list) to 'configure':
--with-lg-page, but its primary use case is for integration with FreeBSD's --with-lg-page, but its primary use case is for integration with FreeBSD's
libc, wherein jemalloc is embedded. libc, wherein jemalloc is embedded.
--with-lg-hugepage=<lg-hugepage>
Specify the base 2 log of the system huge page size. This option is useful
when cross compiling, or when overriding the default for systems that do
not explicitly support huge pages.
--with-lg-size-class-group=<lg-size-class-group> --with-lg-size-class-group=<lg-size-class-group>
Specify the base 2 log of how many size classes to use for each doubling in Specify the base 2 log of how many size classes to use for each doubling in
size. By default jemalloc uses <lg-size-class-group>=2, which results in size. By default jemalloc uses <lg-size-class-group>=2, which results in

View File

@ -170,6 +170,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/mq.c \ $(srcroot)test/unit/mq.c \
$(srcroot)test/unit/mtx.c \ $(srcroot)test/unit/mtx.c \
$(srcroot)test/unit/pack.c \ $(srcroot)test/unit/pack.c \
$(srcroot)test/unit/pages.c \
$(srcroot)test/unit/ph.c \ $(srcroot)test/unit/ph.c \
$(srcroot)test/unit/prng.c \ $(srcroot)test/unit/prng.c \
$(srcroot)test/unit/prof_accum.c \ $(srcroot)test/unit/prof_accum.c \

View File

@ -1317,6 +1317,36 @@ else
AC_MSG_ERROR([cannot determine value for LG_PAGE]) AC_MSG_ERROR([cannot determine value for LG_PAGE])
fi fi
AC_ARG_WITH([lg_hugepage],
[AS_HELP_STRING([--with-lg-hugepage=<lg-hugepage>],
[Base 2 log of sytem huge page size])],
[je_cv_lg_hugepage="${with_lg_hugepage}"],
[je_cv_lg_hugepage=""])
if test "x${je_cv_lg_hugepage}" = "x" ; then
dnl Look in /proc/meminfo (Linux-specific) for information on the default huge
dnl page size, if any. The relevant line looks like:
dnl
dnl Hugepagesize: 2048 kB
if test -e "/proc/meminfo" ; then
hpsk=[`cat /proc/meminfo 2>/dev/null | \
grep -e '^Hugepagesize:[[:space:]]\+[0-9]\+[[:space:]]kB$' | \
awk '{print $2}'`]
if test "x${hpsk}" != "x" ; then
je_cv_lg_hugepage=10
while test "${hpsk}" -gt 1 ; do
hpsk="$((hpsk / 2))"
je_cv_lg_hugepage="$((je_cv_lg_hugepage + 1))"
done
fi
fi
dnl Set default if unable to automatically configure.
if test "x${je_cv_lg_hugepage}" = "x" ; then
je_cv_lg_hugepage=21
fi
fi
AC_DEFINE_UNQUOTED([LG_HUGEPAGE], [${je_cv_lg_hugepage}])
AC_ARG_WITH([lg_page_sizes], AC_ARG_WITH([lg_page_sizes],
[AS_HELP_STRING([--with-lg-page-sizes=<lg-page-sizes>], [AS_HELP_STRING([--with-lg-page-sizes=<lg-page-sizes>],
[Base 2 logs of system page sizes to support])], [Base 2 logs of system page sizes to support])],
@ -1690,6 +1720,8 @@ JE_COMPILABLE([madvise(2)], [
madvise((void *)0, 0, 0); madvise((void *)0, 0, 0);
], [je_cv_madvise]) ], [je_cv_madvise])
if test "x${je_cv_madvise}" = "xyes" ; then if test "x${je_cv_madvise}" = "xyes" ; then
AC_DEFINE([JEMALLOC_HAVE_MADVISE], [ ])
dnl Check for madvise(..., MADV_FREE). dnl Check for madvise(..., MADV_FREE).
JE_COMPILABLE([madvise(..., MADV_FREE)], [ JE_COMPILABLE([madvise(..., MADV_FREE)], [
#include <sys/mman.h> #include <sys/mman.h>
@ -1710,9 +1742,15 @@ if test "x${je_cv_madvise}" = "xyes" ; then
AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ]) AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ])
fi fi
if test "x${je_cv_madv_free}" = "xyes" \ dnl Check for madvise(..., MADV_[NO]HUGEPAGE).
-o "x${je_cv_madv_dontneed}" = "xyes" ; then JE_COMPILABLE([madvise(..., MADV_[[NO]]HUGEPAGE)], [
AC_DEFINE([JEMALLOC_HAVE_MADVISE], [ ]) #include <sys/mman.h>
], [
madvise((void *)0, 0, MADV_HUGEPAGE);
madvise((void *)0, 0, MADV_NOHUGEPAGE);
], [je_cv_thp])
if test "x${je_cv_thp}" = "xyes" ; then
AC_DEFINE([JEMALLOC_THP], [ ])
fi fi
fi fi

View File

@ -138,6 +138,13 @@ static const bool config_cache_oblivious =
false false
#endif #endif
; ;
static const bool have_thp =
#ifdef JEMALLOC_THP
true
#else
false
#endif
;
#if defined(JEMALLOC_C11ATOMICS) && !defined(__cplusplus) #if defined(JEMALLOC_C11ATOMICS) && !defined(__cplusplus)
#include <stdatomic.h> #include <stdatomic.h>

View File

@ -187,6 +187,13 @@
/* One page is 2^LG_PAGE bytes. */ /* One page is 2^LG_PAGE bytes. */
#undef LG_PAGE #undef LG_PAGE
/*
* One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the
* system does not explicitly support huge pages; system calls that require
* explicit huge page support are separately configured.
*/
#undef LG_HUGEPAGE
/* /*
* If defined, adjacent virtual memory mappings with identical attributes * If defined, adjacent virtual memory mappings with identical attributes
* automatically coalesce, and they fragment when changes are made to subranges. * automatically coalesce, and they fragment when changes are made to subranges.
@ -262,6 +269,12 @@
#undef JEMALLOC_PURGE_MADVISE_FREE #undef JEMALLOC_PURGE_MADVISE_FREE
#undef JEMALLOC_PURGE_MADVISE_DONTNEED #undef JEMALLOC_PURGE_MADVISE_DONTNEED
/*
* Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE
* arguments to madvise(2).
*/
#undef JEMALLOC_THP
/* Define if operating system has alloca.h header. */ /* Define if operating system has alloca.h header. */
#undef JEMALLOC_HAS_ALLOCA_H #undef JEMALLOC_HAS_ALLOCA_H

View File

@ -7,15 +7,23 @@
#endif #endif
#define PAGE ((size_t)(1U << LG_PAGE)) #define PAGE ((size_t)(1U << LG_PAGE))
#define PAGE_MASK ((size_t)(PAGE - 1)) #define PAGE_MASK ((size_t)(PAGE - 1))
/* Return the page base address for the page containing address a. */ /* Return the page base address for the page containing address a. */
#define PAGE_ADDR2BASE(a) \ #define PAGE_ADDR2BASE(a) \
((void *)((uintptr_t)(a) & ~PAGE_MASK)) ((void *)((uintptr_t)(a) & ~PAGE_MASK))
/* Return the smallest pagesize multiple that is >= s. */ /* Return the smallest pagesize multiple that is >= s. */
#define PAGE_CEILING(s) \ #define PAGE_CEILING(s) \
(((s) + PAGE_MASK) & ~PAGE_MASK) (((s) + PAGE_MASK) & ~PAGE_MASK)
/* Huge page size. LG_HUGEPAGE is determined by the configure script. */
#define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE))
#define HUGEPAGE_MASK ((size_t)(HUGEPAGE - 1))
/* Return the huge page base address for the huge page containing address a. */
#define HUGEPAGE_ADDR2BASE(a) \
((void *)((uintptr_t)(a) & ~HUGEPAGE_MASK))
/* Return the smallest pagesize multiple that is >= s. */
#define HUGEPAGE_CEILING(s) \
(((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK)
#endif /* JEMALLOC_H_TYPES */ #endif /* JEMALLOC_H_TYPES */
/******************************************************************************/ /******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS #ifdef JEMALLOC_H_STRUCTS
@ -31,6 +39,8 @@ void *pages_trim(void *addr, size_t alloc_size, size_t leadsize,
bool pages_commit(void *addr, size_t size); bool pages_commit(void *addr, size_t size);
bool pages_decommit(void *addr, size_t size); bool pages_decommit(void *addr, size_t size);
bool pages_purge(void *addr, size_t size); bool pages_purge(void *addr, size_t size);
bool pages_huge(void *addr, size_t size);
bool pages_nohuge(void *addr, size_t size);
void pages_boot(void); void pages_boot(void);
#endif /* JEMALLOC_H_EXTERNS */ #endif /* JEMALLOC_H_EXTERNS */

View File

@ -324,7 +324,9 @@ opt_zero
pages_boot pages_boot
pages_commit pages_commit
pages_decommit pages_decommit
pages_huge
pages_map pages_map
pages_nohuge
pages_purge pages_purge
pages_trim pages_trim
pages_unmap pages_unmap

View File

@ -170,7 +170,8 @@ pages_purge(void *addr, size_t size)
#ifdef _WIN32 #ifdef _WIN32
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
unzeroed = true; unzeroed = true;
#elif defined(JEMALLOC_HAVE_MADVISE) #elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \
defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
# if defined(JEMALLOC_PURGE_MADVISE_FREE) # if defined(JEMALLOC_PURGE_MADVISE_FREE)
# define JEMALLOC_MADV_PURGE MADV_FREE # define JEMALLOC_MADV_PURGE MADV_FREE
# define JEMALLOC_MADV_ZEROS false # define JEMALLOC_MADV_ZEROS false
@ -191,6 +192,34 @@ pages_purge(void *addr, size_t size)
return (unzeroed); return (unzeroed);
} }
bool
pages_huge(void *addr, size_t size)
{
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
#ifdef JEMALLOC_THP
return (madvise(addr, size, MADV_HUGEPAGE) != 0);
#else
return (true);
#endif
}
bool
pages_nohuge(void *addr, size_t size)
{
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
assert(HUGEPAGE_CEILING(size) == size);
#ifdef JEMALLOC_THP
return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
#else
return (false);
#endif
}
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
static bool static bool
os_overcommits_sysctl(void) os_overcommits_sysctl(void)

30
test/unit/pages.c Normal file
View File

@ -0,0 +1,30 @@
#include "test/jemalloc_test.h"
TEST_BEGIN(test_pages_huge)
{
size_t alloc_size;
bool commit;
void *pages, *hugepage;
alloc_size = HUGEPAGE * 2 - PAGE;
commit = true;
pages = pages_map(NULL, alloc_size, &commit);
assert_ptr_not_null(pages, "Unexpected pages_map() error");
hugepage = (void *)(ALIGNMENT_CEILING((uintptr_t)pages, HUGEPAGE));
assert_b_ne(pages_huge(hugepage, HUGEPAGE), have_thp,
"Unexpected pages_huge() result");
assert_false(pages_nohuge(hugepage, HUGEPAGE),
"Unexpected pages_nohuge() result");
pages_unmap(pages, alloc_size);
}
TEST_END
int
main(void)
{
return (test(
test_pages_huge));
}