2012-03-22 09:33:03 +08:00
|
|
|
#define JEMALLOC_TSD_C_
|
|
|
|
#include "jemalloc/internal/jemalloc_internal.h"
|
|
|
|
|
|
|
|
/******************************************************************************/
|
|
|
|
/* Data. */
|
|
|
|
|
|
|
|
static unsigned ncleanups;
|
|
|
|
static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
|
|
|
|
|
2014-09-23 12:09:23 +08:00
|
|
|
malloc_tsd_data(, , tsd_t, TSD_INITIALIZER)
|
|
|
|
|
2012-03-22 09:33:03 +08:00
|
|
|
/******************************************************************************/
|
|
|
|
|
|
|
|
void *
|
|
|
|
malloc_tsd_malloc(size_t size)
|
|
|
|
{
|
|
|
|
|
2014-11-28 03:22:36 +08:00
|
|
|
return (a0malloc(CACHELINE_CEILING(size)));
|
2012-03-22 09:33:03 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
malloc_tsd_dalloc(void *wrapper)
|
|
|
|
{
|
|
|
|
|
2015-01-21 07:37:51 +08:00
|
|
|
a0dalloc(wrapper);
|
2012-03-22 09:33:03 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
malloc_tsd_no_cleanup(void *arg)
|
|
|
|
{
|
|
|
|
|
|
|
|
not_reached();
|
|
|
|
}
|
|
|
|
|
2012-04-22 12:27:46 +08:00
|
|
|
#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32)
|
2012-04-30 18:38:29 +08:00
|
|
|
#ifndef _WIN32
|
|
|
|
JEMALLOC_EXPORT
|
|
|
|
#endif
|
2012-03-22 09:33:03 +08:00
|
|
|
void
|
|
|
|
_malloc_thread_cleanup(void)
|
|
|
|
{
|
2012-04-25 05:22:02 +08:00
|
|
|
bool pending[MALLOC_TSD_CLEANUPS_MAX], again;
|
2012-03-22 09:33:03 +08:00
|
|
|
unsigned i;
|
|
|
|
|
|
|
|
for (i = 0; i < ncleanups; i++)
|
|
|
|
pending[i] = true;
|
|
|
|
|
|
|
|
do {
|
|
|
|
again = false;
|
|
|
|
for (i = 0; i < ncleanups; i++) {
|
|
|
|
if (pending[i]) {
|
2012-04-19 00:29:49 +08:00
|
|
|
pending[i] = cleanups[i]();
|
2012-03-22 09:33:03 +08:00
|
|
|
if (pending[i])
|
|
|
|
again = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} while (again);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
void
|
2012-04-19 00:29:49 +08:00
|
|
|
malloc_tsd_cleanup_register(bool (*f)(void))
|
2012-03-22 09:33:03 +08:00
|
|
|
{
|
|
|
|
|
|
|
|
assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX);
|
2012-04-19 00:29:49 +08:00
|
|
|
cleanups[ncleanups] = f;
|
2012-03-22 09:33:03 +08:00
|
|
|
ncleanups++;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2014-09-23 12:09:23 +08:00
|
|
|
tsd_cleanup(void *arg)
|
|
|
|
{
|
|
|
|
tsd_t *tsd = (tsd_t *)arg;
|
|
|
|
|
|
|
|
switch (tsd->state) {
|
2015-09-25 07:53:18 +08:00
|
|
|
case tsd_state_uninitialized:
|
|
|
|
/* Do nothing. */
|
|
|
|
break;
|
2014-09-23 12:09:23 +08:00
|
|
|
case tsd_state_nominal:
|
2016-06-03 02:11:35 +08:00
|
|
|
#define MALLOC_TSD_cleanup_yes(n, t) \
|
2014-09-23 12:09:23 +08:00
|
|
|
n##_cleanup(tsd);
|
2016-06-03 02:11:35 +08:00
|
|
|
#define MALLOC_TSD_cleanup_no(n, t)
|
|
|
|
#define O(n, t, c) \
|
|
|
|
MALLOC_TSD_cleanup_##c(n, t)
|
2014-09-23 12:09:23 +08:00
|
|
|
MALLOC_TSD
|
2016-06-03 02:11:35 +08:00
|
|
|
#undef MALLOC_TSD_cleanup_yes
|
|
|
|
#undef MALLOC_TSD_cleanup_no
|
2014-09-23 12:09:23 +08:00
|
|
|
#undef O
|
|
|
|
tsd->state = tsd_state_purgatory;
|
|
|
|
tsd_set(tsd);
|
|
|
|
break;
|
|
|
|
case tsd_state_purgatory:
|
|
|
|
/*
|
|
|
|
* The previous time this destructor was called, we set the
|
|
|
|
* state to tsd_state_purgatory so that other destructors
|
|
|
|
* wouldn't cause re-creation of the tsd. This time, do
|
|
|
|
* nothing, and do not request another callback.
|
|
|
|
*/
|
|
|
|
break;
|
|
|
|
case tsd_state_reincarnated:
|
|
|
|
/*
|
|
|
|
* Another destructor deallocated memory after this destructor
|
|
|
|
* was called. Reset state to tsd_state_purgatory and request
|
|
|
|
* another callback.
|
|
|
|
*/
|
|
|
|
tsd->state = tsd_state_purgatory;
|
|
|
|
tsd_set(tsd);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
not_reached();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-14 14:36:15 +08:00
|
|
|
tsd_t *
|
Refactor/fix arenas manipulation.
Abstract arenas access to use arena_get() (or a0get() where appropriate)
rather than directly reading e.g. arenas[ind]. Prior to the addition of
the arenas.extend mallctl, the worst possible outcome of directly
accessing arenas was a stale read, but arenas.extend may allocate and
assign a new array to arenas.
Add a tsd-based arenas_cache, which amortizes arenas reads. This
introduces some subtle bootstrapping issues, with tsd_boot() now being
split into tsd_boot[01]() to support tsd wrapper allocation
bootstrapping, as well as an arenas_cache_bypass tsd variable which
dynamically terminates allocation of arenas_cache itself.
Promote a0malloc(), a0calloc(), and a0free() to be generally useful for
internal allocation, and use them in several places (more may be
appropriate).
Abstract arena->nthreads management and fix a missing decrement during
thread destruction (recent tsd refactoring left arenas_cleanup()
unused).
Change arena_choose() to propagate OOM, and handle OOM in all callers.
This is important for providing consistent allocation behavior when the
MALLOCX_ARENA() flag is being used. Prior to this fix, it was possible
for an OOM to result in allocation silently allocating from a different
arena than the one specified.
2014-10-08 14:14:57 +08:00
|
|
|
malloc_tsd_boot0(void)
|
2012-03-22 09:33:03 +08:00
|
|
|
{
|
2016-04-14 14:36:15 +08:00
|
|
|
tsd_t *tsd;
|
2012-03-22 09:33:03 +08:00
|
|
|
|
|
|
|
ncleanups = 0;
|
Refactor/fix arenas manipulation.
Abstract arenas access to use arena_get() (or a0get() where appropriate)
rather than directly reading e.g. arenas[ind]. Prior to the addition of
the arenas.extend mallctl, the worst possible outcome of directly
accessing arenas was a stale read, but arenas.extend may allocate and
assign a new array to arenas.
Add a tsd-based arenas_cache, which amortizes arenas reads. This
introduces some subtle bootstrapping issues, with tsd_boot() now being
split into tsd_boot[01]() to support tsd wrapper allocation
bootstrapping, as well as an arenas_cache_bypass tsd variable which
dynamically terminates allocation of arenas_cache itself.
Promote a0malloc(), a0calloc(), and a0free() to be generally useful for
internal allocation, and use them in several places (more may be
appropriate).
Abstract arena->nthreads management and fix a missing decrement during
thread destruction (recent tsd refactoring left arenas_cleanup()
unused).
Change arena_choose() to propagate OOM, and handle OOM in all callers.
This is important for providing consistent allocation behavior when the
MALLOCX_ARENA() flag is being used. Prior to this fix, it was possible
for an OOM to result in allocation silently allocating from a different
arena than the one specified.
2014-10-08 14:14:57 +08:00
|
|
|
if (tsd_boot0())
|
2016-04-14 14:36:15 +08:00
|
|
|
return (NULL);
|
|
|
|
tsd = tsd_fetch();
|
|
|
|
*tsd_arenas_tdata_bypassp_get(tsd) = true;
|
|
|
|
return (tsd);
|
2012-03-22 09:33:03 +08:00
|
|
|
}
|
2012-04-22 12:27:46 +08:00
|
|
|
|
Refactor/fix arenas manipulation.
Abstract arenas access to use arena_get() (or a0get() where appropriate)
rather than directly reading e.g. arenas[ind]. Prior to the addition of
the arenas.extend mallctl, the worst possible outcome of directly
accessing arenas was a stale read, but arenas.extend may allocate and
assign a new array to arenas.
Add a tsd-based arenas_cache, which amortizes arenas reads. This
introduces some subtle bootstrapping issues, with tsd_boot() now being
split into tsd_boot[01]() to support tsd wrapper allocation
bootstrapping, as well as an arenas_cache_bypass tsd variable which
dynamically terminates allocation of arenas_cache itself.
Promote a0malloc(), a0calloc(), and a0free() to be generally useful for
internal allocation, and use them in several places (more may be
appropriate).
Abstract arena->nthreads management and fix a missing decrement during
thread destruction (recent tsd refactoring left arenas_cleanup()
unused).
Change arena_choose() to propagate OOM, and handle OOM in all callers.
This is important for providing consistent allocation behavior when the
MALLOCX_ARENA() flag is being used. Prior to this fix, it was possible
for an OOM to result in allocation silently allocating from a different
arena than the one specified.
2014-10-08 14:14:57 +08:00
|
|
|
void
|
|
|
|
malloc_tsd_boot1(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
tsd_boot1();
|
2016-02-20 11:37:10 +08:00
|
|
|
*tsd_arenas_tdata_bypassp_get(tsd_fetch()) = false;
|
Refactor/fix arenas manipulation.
Abstract arenas access to use arena_get() (or a0get() where appropriate)
rather than directly reading e.g. arenas[ind]. Prior to the addition of
the arenas.extend mallctl, the worst possible outcome of directly
accessing arenas was a stale read, but arenas.extend may allocate and
assign a new array to arenas.
Add a tsd-based arenas_cache, which amortizes arenas reads. This
introduces some subtle bootstrapping issues, with tsd_boot() now being
split into tsd_boot[01]() to support tsd wrapper allocation
bootstrapping, as well as an arenas_cache_bypass tsd variable which
dynamically terminates allocation of arenas_cache itself.
Promote a0malloc(), a0calloc(), and a0free() to be generally useful for
internal allocation, and use them in several places (more may be
appropriate).
Abstract arena->nthreads management and fix a missing decrement during
thread destruction (recent tsd refactoring left arenas_cleanup()
unused).
Change arena_choose() to propagate OOM, and handle OOM in all callers.
This is important for providing consistent allocation behavior when the
MALLOCX_ARENA() flag is being used. Prior to this fix, it was possible
for an OOM to result in allocation silently allocating from a different
arena than the one specified.
2014-10-08 14:14:57 +08:00
|
|
|
}
|
|
|
|
|
2012-04-22 12:27:46 +08:00
|
|
|
#ifdef _WIN32
|
|
|
|
static BOOL WINAPI
|
|
|
|
_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
|
|
|
|
{
|
|
|
|
|
|
|
|
switch (fdwReason) {
|
|
|
|
#ifdef JEMALLOC_LAZY_LOCK
|
|
|
|
case DLL_THREAD_ATTACH:
|
|
|
|
isthreaded = true;
|
|
|
|
break;
|
|
|
|
#endif
|
|
|
|
case DLL_THREAD_DETACH:
|
|
|
|
_malloc_thread_cleanup();
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return (true);
|
|
|
|
}
|
|
|
|
|
2012-04-30 18:38:31 +08:00
|
|
|
#ifdef _MSC_VER
|
|
|
|
# ifdef _M_IX86
|
|
|
|
# pragma comment(linker, "/INCLUDE:__tls_used")
|
2016-02-02 18:27:18 +08:00
|
|
|
# pragma comment(linker, "/INCLUDE:_tls_callback")
|
2012-04-30 18:38:31 +08:00
|
|
|
# else
|
|
|
|
# pragma comment(linker, "/INCLUDE:_tls_used")
|
2016-02-02 18:27:18 +08:00
|
|
|
# pragma comment(linker, "/INCLUDE:tls_callback")
|
2012-04-30 18:38:31 +08:00
|
|
|
# endif
|
|
|
|
# pragma section(".CRT$XLY",long,read)
|
|
|
|
#endif
|
2012-04-30 18:38:29 +08:00
|
|
|
JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used)
|
2016-02-02 18:27:18 +08:00
|
|
|
BOOL (WINAPI *const tls_callback)(HINSTANCE hinstDLL,
|
2015-07-08 11:28:22 +08:00
|
|
|
DWORD fdwReason, LPVOID lpvReserved) = _tls_callback;
|
2012-04-22 12:27:46 +08:00
|
|
|
#endif
|
2013-10-22 05:12:16 +08:00
|
|
|
|
|
|
|
#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
|
|
|
|
!defined(_WIN32))
|
|
|
|
void *
|
|
|
|
tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block)
|
|
|
|
{
|
|
|
|
pthread_t self = pthread_self();
|
|
|
|
tsd_init_block_t *iter;
|
|
|
|
|
|
|
|
/* Check whether this thread has already inserted into the list. */
|
2016-05-13 12:07:08 +08:00
|
|
|
malloc_mutex_lock(TSDN_NULL, &head->lock);
|
2013-10-22 05:12:16 +08:00
|
|
|
ql_foreach(iter, &head->blocks, link) {
|
|
|
|
if (iter->thread == self) {
|
2016-05-13 12:07:08 +08:00
|
|
|
malloc_mutex_unlock(TSDN_NULL, &head->lock);
|
2013-10-22 05:12:16 +08:00
|
|
|
return (iter->data);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/* Insert block into list. */
|
|
|
|
ql_elm_new(block, link);
|
|
|
|
block->thread = self;
|
|
|
|
ql_tail_insert(&head->blocks, block, link);
|
2016-05-13 12:07:08 +08:00
|
|
|
malloc_mutex_unlock(TSDN_NULL, &head->lock);
|
2013-10-22 05:12:16 +08:00
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block)
|
|
|
|
{
|
|
|
|
|
2016-05-13 12:07:08 +08:00
|
|
|
malloc_mutex_lock(TSDN_NULL, &head->lock);
|
2013-10-22 05:12:16 +08:00
|
|
|
ql_remove(&head->blocks, block, link);
|
2016-05-13 12:07:08 +08:00
|
|
|
malloc_mutex_unlock(TSDN_NULL, &head->lock);
|
2013-10-22 05:12:16 +08:00
|
|
|
}
|
|
|
|
#endif
|