Add huge page configuration and pages_[no}huge().
Add the --with-lg-hugepage configure option, but automatically configure LG_HUGEPAGE even if it isn't specified. Add the pages_[no]huge() functions, which toggle huge page state via madvise(..., MADV_[NO]HUGEPAGE) calls.
This commit is contained in:
parent
eab3b180e5
commit
c1baa0a9b7
5
INSTALL
5
INSTALL
@ -227,6 +227,11 @@ any of the following arguments (not a definitive list) to 'configure':
|
|||||||
--with-lg-page, but its primary use case is for integration with FreeBSD's
|
--with-lg-page, but its primary use case is for integration with FreeBSD's
|
||||||
libc, wherein jemalloc is embedded.
|
libc, wherein jemalloc is embedded.
|
||||||
|
|
||||||
|
--with-lg-hugepage=<lg-hugepage>
|
||||||
|
Specify the base 2 log of the system huge page size. This option is useful
|
||||||
|
when cross compiling, or when overriding the default for systems that do
|
||||||
|
not explicitly support huge pages.
|
||||||
|
|
||||||
--with-lg-size-class-group=<lg-size-class-group>
|
--with-lg-size-class-group=<lg-size-class-group>
|
||||||
Specify the base 2 log of how many size classes to use for each doubling in
|
Specify the base 2 log of how many size classes to use for each doubling in
|
||||||
size. By default jemalloc uses <lg-size-class-group>=2, which results in
|
size. By default jemalloc uses <lg-size-class-group>=2, which results in
|
||||||
|
@ -170,6 +170,7 @@ TESTS_UNIT := \
|
|||||||
$(srcroot)test/unit/mq.c \
|
$(srcroot)test/unit/mq.c \
|
||||||
$(srcroot)test/unit/mtx.c \
|
$(srcroot)test/unit/mtx.c \
|
||||||
$(srcroot)test/unit/pack.c \
|
$(srcroot)test/unit/pack.c \
|
||||||
|
$(srcroot)test/unit/pages.c \
|
||||||
$(srcroot)test/unit/ph.c \
|
$(srcroot)test/unit/ph.c \
|
||||||
$(srcroot)test/unit/prng.c \
|
$(srcroot)test/unit/prng.c \
|
||||||
$(srcroot)test/unit/prof_accum.c \
|
$(srcroot)test/unit/prof_accum.c \
|
||||||
|
44
configure.ac
44
configure.ac
@ -1317,6 +1317,36 @@ else
|
|||||||
AC_MSG_ERROR([cannot determine value for LG_PAGE])
|
AC_MSG_ERROR([cannot determine value for LG_PAGE])
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
AC_ARG_WITH([lg_hugepage],
|
||||||
|
[AS_HELP_STRING([--with-lg-hugepage=<lg-hugepage>],
|
||||||
|
[Base 2 log of sytem huge page size])],
|
||||||
|
[je_cv_lg_hugepage="${with_lg_hugepage}"],
|
||||||
|
[je_cv_lg_hugepage=""])
|
||||||
|
if test "x${je_cv_lg_hugepage}" = "x" ; then
|
||||||
|
dnl Look in /proc/meminfo (Linux-specific) for information on the default huge
|
||||||
|
dnl page size, if any. The relevant line looks like:
|
||||||
|
dnl
|
||||||
|
dnl Hugepagesize: 2048 kB
|
||||||
|
if test -e "/proc/meminfo" ; then
|
||||||
|
hpsk=[`cat /proc/meminfo 2>/dev/null | \
|
||||||
|
grep -e '^Hugepagesize:[[:space:]]\+[0-9]\+[[:space:]]kB$' | \
|
||||||
|
awk '{print $2}'`]
|
||||||
|
if test "x${hpsk}" != "x" ; then
|
||||||
|
je_cv_lg_hugepage=10
|
||||||
|
while test "${hpsk}" -gt 1 ; do
|
||||||
|
hpsk="$((hpsk / 2))"
|
||||||
|
je_cv_lg_hugepage="$((je_cv_lg_hugepage + 1))"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
dnl Set default if unable to automatically configure.
|
||||||
|
if test "x${je_cv_lg_hugepage}" = "x" ; then
|
||||||
|
je_cv_lg_hugepage=21
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
AC_DEFINE_UNQUOTED([LG_HUGEPAGE], [${je_cv_lg_hugepage}])
|
||||||
|
|
||||||
AC_ARG_WITH([lg_page_sizes],
|
AC_ARG_WITH([lg_page_sizes],
|
||||||
[AS_HELP_STRING([--with-lg-page-sizes=<lg-page-sizes>],
|
[AS_HELP_STRING([--with-lg-page-sizes=<lg-page-sizes>],
|
||||||
[Base 2 logs of system page sizes to support])],
|
[Base 2 logs of system page sizes to support])],
|
||||||
@ -1690,6 +1720,8 @@ JE_COMPILABLE([madvise(2)], [
|
|||||||
madvise((void *)0, 0, 0);
|
madvise((void *)0, 0, 0);
|
||||||
], [je_cv_madvise])
|
], [je_cv_madvise])
|
||||||
if test "x${je_cv_madvise}" = "xyes" ; then
|
if test "x${je_cv_madvise}" = "xyes" ; then
|
||||||
|
AC_DEFINE([JEMALLOC_HAVE_MADVISE], [ ])
|
||||||
|
|
||||||
dnl Check for madvise(..., MADV_FREE).
|
dnl Check for madvise(..., MADV_FREE).
|
||||||
JE_COMPILABLE([madvise(..., MADV_FREE)], [
|
JE_COMPILABLE([madvise(..., MADV_FREE)], [
|
||||||
#include <sys/mman.h>
|
#include <sys/mman.h>
|
||||||
@ -1710,9 +1742,15 @@ if test "x${je_cv_madvise}" = "xyes" ; then
|
|||||||
AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ])
|
AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ])
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if test "x${je_cv_madv_free}" = "xyes" \
|
dnl Check for madvise(..., MADV_[NO]HUGEPAGE).
|
||||||
-o "x${je_cv_madv_dontneed}" = "xyes" ; then
|
JE_COMPILABLE([madvise(..., MADV_[[NO]]HUGEPAGE)], [
|
||||||
AC_DEFINE([JEMALLOC_HAVE_MADVISE], [ ])
|
#include <sys/mman.h>
|
||||||
|
], [
|
||||||
|
madvise((void *)0, 0, MADV_HUGEPAGE);
|
||||||
|
madvise((void *)0, 0, MADV_NOHUGEPAGE);
|
||||||
|
], [je_cv_thp])
|
||||||
|
if test "x${je_cv_thp}" = "xyes" ; then
|
||||||
|
AC_DEFINE([JEMALLOC_THP], [ ])
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
@ -138,6 +138,13 @@ static const bool config_cache_oblivious =
|
|||||||
false
|
false
|
||||||
#endif
|
#endif
|
||||||
;
|
;
|
||||||
|
static const bool have_thp =
|
||||||
|
#ifdef JEMALLOC_THP
|
||||||
|
true
|
||||||
|
#else
|
||||||
|
false
|
||||||
|
#endif
|
||||||
|
;
|
||||||
|
|
||||||
#if defined(JEMALLOC_C11ATOMICS) && !defined(__cplusplus)
|
#if defined(JEMALLOC_C11ATOMICS) && !defined(__cplusplus)
|
||||||
#include <stdatomic.h>
|
#include <stdatomic.h>
|
||||||
|
@ -187,6 +187,13 @@
|
|||||||
/* One page is 2^LG_PAGE bytes. */
|
/* One page is 2^LG_PAGE bytes. */
|
||||||
#undef LG_PAGE
|
#undef LG_PAGE
|
||||||
|
|
||||||
|
/*
|
||||||
|
* One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the
|
||||||
|
* system does not explicitly support huge pages; system calls that require
|
||||||
|
* explicit huge page support are separately configured.
|
||||||
|
*/
|
||||||
|
#undef LG_HUGEPAGE
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If defined, adjacent virtual memory mappings with identical attributes
|
* If defined, adjacent virtual memory mappings with identical attributes
|
||||||
* automatically coalesce, and they fragment when changes are made to subranges.
|
* automatically coalesce, and they fragment when changes are made to subranges.
|
||||||
@ -262,6 +269,12 @@
|
|||||||
#undef JEMALLOC_PURGE_MADVISE_FREE
|
#undef JEMALLOC_PURGE_MADVISE_FREE
|
||||||
#undef JEMALLOC_PURGE_MADVISE_DONTNEED
|
#undef JEMALLOC_PURGE_MADVISE_DONTNEED
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE
|
||||||
|
* arguments to madvise(2).
|
||||||
|
*/
|
||||||
|
#undef JEMALLOC_THP
|
||||||
|
|
||||||
/* Define if operating system has alloca.h header. */
|
/* Define if operating system has alloca.h header. */
|
||||||
#undef JEMALLOC_HAS_ALLOCA_H
|
#undef JEMALLOC_HAS_ALLOCA_H
|
||||||
|
|
||||||
|
@ -7,15 +7,23 @@
|
|||||||
#endif
|
#endif
|
||||||
#define PAGE ((size_t)(1U << LG_PAGE))
|
#define PAGE ((size_t)(1U << LG_PAGE))
|
||||||
#define PAGE_MASK ((size_t)(PAGE - 1))
|
#define PAGE_MASK ((size_t)(PAGE - 1))
|
||||||
|
|
||||||
/* Return the page base address for the page containing address a. */
|
/* Return the page base address for the page containing address a. */
|
||||||
#define PAGE_ADDR2BASE(a) \
|
#define PAGE_ADDR2BASE(a) \
|
||||||
((void *)((uintptr_t)(a) & ~PAGE_MASK))
|
((void *)((uintptr_t)(a) & ~PAGE_MASK))
|
||||||
|
|
||||||
/* Return the smallest pagesize multiple that is >= s. */
|
/* Return the smallest pagesize multiple that is >= s. */
|
||||||
#define PAGE_CEILING(s) \
|
#define PAGE_CEILING(s) \
|
||||||
(((s) + PAGE_MASK) & ~PAGE_MASK)
|
(((s) + PAGE_MASK) & ~PAGE_MASK)
|
||||||
|
|
||||||
|
/* Huge page size. LG_HUGEPAGE is determined by the configure script. */
|
||||||
|
#define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE))
|
||||||
|
#define HUGEPAGE_MASK ((size_t)(HUGEPAGE - 1))
|
||||||
|
/* Return the huge page base address for the huge page containing address a. */
|
||||||
|
#define HUGEPAGE_ADDR2BASE(a) \
|
||||||
|
((void *)((uintptr_t)(a) & ~HUGEPAGE_MASK))
|
||||||
|
/* Return the smallest pagesize multiple that is >= s. */
|
||||||
|
#define HUGEPAGE_CEILING(s) \
|
||||||
|
(((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK)
|
||||||
|
|
||||||
#endif /* JEMALLOC_H_TYPES */
|
#endif /* JEMALLOC_H_TYPES */
|
||||||
/******************************************************************************/
|
/******************************************************************************/
|
||||||
#ifdef JEMALLOC_H_STRUCTS
|
#ifdef JEMALLOC_H_STRUCTS
|
||||||
@ -31,6 +39,8 @@ void *pages_trim(void *addr, size_t alloc_size, size_t leadsize,
|
|||||||
bool pages_commit(void *addr, size_t size);
|
bool pages_commit(void *addr, size_t size);
|
||||||
bool pages_decommit(void *addr, size_t size);
|
bool pages_decommit(void *addr, size_t size);
|
||||||
bool pages_purge(void *addr, size_t size);
|
bool pages_purge(void *addr, size_t size);
|
||||||
|
bool pages_huge(void *addr, size_t size);
|
||||||
|
bool pages_nohuge(void *addr, size_t size);
|
||||||
void pages_boot(void);
|
void pages_boot(void);
|
||||||
|
|
||||||
#endif /* JEMALLOC_H_EXTERNS */
|
#endif /* JEMALLOC_H_EXTERNS */
|
||||||
|
@ -324,7 +324,9 @@ opt_zero
|
|||||||
pages_boot
|
pages_boot
|
||||||
pages_commit
|
pages_commit
|
||||||
pages_decommit
|
pages_decommit
|
||||||
|
pages_huge
|
||||||
pages_map
|
pages_map
|
||||||
|
pages_nohuge
|
||||||
pages_purge
|
pages_purge
|
||||||
pages_trim
|
pages_trim
|
||||||
pages_unmap
|
pages_unmap
|
||||||
|
31
src/pages.c
31
src/pages.c
@ -170,7 +170,8 @@ pages_purge(void *addr, size_t size)
|
|||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
|
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
|
||||||
unzeroed = true;
|
unzeroed = true;
|
||||||
#elif defined(JEMALLOC_HAVE_MADVISE)
|
#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \
|
||||||
|
defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
|
||||||
# if defined(JEMALLOC_PURGE_MADVISE_FREE)
|
# if defined(JEMALLOC_PURGE_MADVISE_FREE)
|
||||||
# define JEMALLOC_MADV_PURGE MADV_FREE
|
# define JEMALLOC_MADV_PURGE MADV_FREE
|
||||||
# define JEMALLOC_MADV_ZEROS false
|
# define JEMALLOC_MADV_ZEROS false
|
||||||
@ -191,6 +192,34 @@ pages_purge(void *addr, size_t size)
|
|||||||
return (unzeroed);
|
return (unzeroed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool
|
||||||
|
pages_huge(void *addr, size_t size)
|
||||||
|
{
|
||||||
|
|
||||||
|
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
|
||||||
|
assert(HUGEPAGE_CEILING(size) == size);
|
||||||
|
|
||||||
|
#ifdef JEMALLOC_THP
|
||||||
|
return (madvise(addr, size, MADV_HUGEPAGE) != 0);
|
||||||
|
#else
|
||||||
|
return (true);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
bool
|
||||||
|
pages_nohuge(void *addr, size_t size)
|
||||||
|
{
|
||||||
|
|
||||||
|
assert(HUGEPAGE_ADDR2BASE(addr) == addr);
|
||||||
|
assert(HUGEPAGE_CEILING(size) == size);
|
||||||
|
|
||||||
|
#ifdef JEMALLOC_THP
|
||||||
|
return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
|
||||||
|
#else
|
||||||
|
return (false);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||||
static bool
|
static bool
|
||||||
os_overcommits_sysctl(void)
|
os_overcommits_sysctl(void)
|
||||||
|
30
test/unit/pages.c
Normal file
30
test/unit/pages.c
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
#include "test/jemalloc_test.h"
|
||||||
|
|
||||||
|
TEST_BEGIN(test_pages_huge)
|
||||||
|
{
|
||||||
|
size_t alloc_size;
|
||||||
|
bool commit;
|
||||||
|
void *pages, *hugepage;
|
||||||
|
|
||||||
|
alloc_size = HUGEPAGE * 2 - PAGE;
|
||||||
|
commit = true;
|
||||||
|
pages = pages_map(NULL, alloc_size, &commit);
|
||||||
|
assert_ptr_not_null(pages, "Unexpected pages_map() error");
|
||||||
|
|
||||||
|
hugepage = (void *)(ALIGNMENT_CEILING((uintptr_t)pages, HUGEPAGE));
|
||||||
|
assert_b_ne(pages_huge(hugepage, HUGEPAGE), have_thp,
|
||||||
|
"Unexpected pages_huge() result");
|
||||||
|
assert_false(pages_nohuge(hugepage, HUGEPAGE),
|
||||||
|
"Unexpected pages_nohuge() result");
|
||||||
|
|
||||||
|
pages_unmap(pages, alloc_size);
|
||||||
|
}
|
||||||
|
TEST_END
|
||||||
|
|
||||||
|
int
|
||||||
|
main(void)
|
||||||
|
{
|
||||||
|
|
||||||
|
return (test(
|
||||||
|
test_pages_huge));
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user