project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mark huge allocations as unlikely

Browse Source 
This cleans up the fast path a bit more by moving away more code.
This commit is contained in:
Daniel Micay 2014-10-23 10:30:52 -04:00 committed by Jason Evans
parent c93ed81cd0
commit 809b0ac391
4 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -655,7 +655,7 @@ sa2u(size_t size, size_t alignment)
} }
/* Try for a large size class. */ /* Try for a large size class. */
if (size <= arena_maxclass && alignment < chunksize) { if (likely(size <= arena_maxclass) && likely(alignment < chunksize)) {
/* /*
* We can't achieve subpage alignment, so round up alignment * We can't achieve subpage alignment, so round up alignment
* to the minimum that can actually be supported. * to the minimum that can actually be supported.
@ -805,7 +805,7 @@ imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
assert(size != 0); assert(size != 0);
if (size <= arena_maxclass) if (likely(size <= arena_maxclass))
return (arena_malloc(tsd, arena, size, false, try_tcache)); return (arena_malloc(tsd, arena, size, false, try_tcache));
else else
return (huge_malloc(tsd, arena, size, false, try_tcache)); return (huge_malloc(tsd, arena, size, false, try_tcache));
@ -822,7 +822,7 @@ JEMALLOC_ALWAYS_INLINE void *
icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena)
{ {
if (size <= arena_maxclass) if (likely(size <= arena_maxclass))
return (arena_malloc(tsd, arena, size, true, try_tcache)); return (arena_malloc(tsd, arena, size, true, try_tcache));
else else
return (huge_malloc(tsd, arena, size, true, try_tcache)); return (huge_malloc(tsd, arena, size, true, try_tcache));
@ -847,12 +847,12 @@ ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache,
if (usize <= SMALL_MAXCLASS && alignment < PAGE) if (usize <= SMALL_MAXCLASS && alignment < PAGE)
ret = arena_malloc(tsd, arena, usize, zero, try_tcache); ret = arena_malloc(tsd, arena, usize, zero, try_tcache);
else { else {
if (usize <= arena_maxclass) { if (likely(usize <= arena_maxclass)) {
arena = arena_choose(tsd, arena); arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL)) if (unlikely(arena == NULL))
return (NULL); return (NULL);
ret = arena_palloc(arena, usize, alignment, zero); ret = arena_palloc(arena, usize, alignment, zero);
} else if (alignment <= chunksize) } else if (likely(alignment <= chunksize))
ret = huge_malloc(tsd, arena, usize, zero, try_tcache); ret = huge_malloc(tsd, arena, usize, zero, try_tcache);
else { else {
ret = huge_palloc(tsd, arena, usize, alignment, zero, ret = huge_palloc(tsd, arena, usize, alignment, zero,
@ -887,7 +887,7 @@ isalloc(const void *ptr, bool demote)
assert(config_prof || !demote); assert(config_prof || !demote);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) if (likely(chunk != ptr))
ret = arena_salloc(ptr, demote); ret = arena_salloc(ptr, demote);
else else
ret = huge_salloc(ptr); ret = huge_salloc(ptr);
@ -936,7 +936,7 @@ idalloct(tsd_t *tsd, void *ptr, bool try_tcache)
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) if (likely(chunk != ptr))
arena_dalloc(tsd, chunk, ptr, try_tcache); arena_dalloc(tsd, chunk, ptr, try_tcache);
else else
huge_dalloc(tsd, ptr, try_tcache); huge_dalloc(tsd, ptr, try_tcache);
@ -950,7 +950,7 @@ isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache)
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) if (likely(chunk != ptr))
arena_sdalloc(tsd, chunk, ptr, size, try_tcache); arena_sdalloc(tsd, chunk, ptr, size, try_tcache);
else else
huge_dalloc(tsd, ptr, try_tcache); huge_dalloc(tsd, ptr, try_tcache);
@ -1038,7 +1038,7 @@ iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero,
zero, try_tcache_alloc, try_tcache_dalloc, arena)); zero, try_tcache_alloc, try_tcache_dalloc, arena));
} }
if (size <= arena_maxclass) { if (likely(size <= arena_maxclass)) {
return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0, return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0,
alignment, zero, try_tcache_alloc, try_tcache_dalloc)); alignment, zero, try_tcache_alloc, try_tcache_dalloc));
} else { } else {
@ -1069,7 +1069,7 @@ ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
return (true); return (true);
} }
if (size <= arena_maxclass) if (likely(size <= arena_maxclass))
return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero)); return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
else else
return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero)); return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero));

4
include/jemalloc/internal/prof.h
View File

@ -361,7 +361,7 @@ prof_tctx_get(const void *ptr)
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) { if (likely(chunk != ptr)) {
/* Region. */ /* Region. */
ret = arena_prof_tctx_get(ptr); ret = arena_prof_tctx_get(ptr);
} else } else
@ -379,7 +379,7 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) { if (likely(chunk != ptr)) {
/* Region. */ /* Region. */
arena_prof_tctx_set(ptr, tctx); arena_prof_tctx_set(ptr, tctx);
} else } else

4
src/arena.c
View File

@ -2095,7 +2095,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t usize; size_t usize;
/* Make sure extra can't cause size_t overflow. */ /* Make sure extra can't cause size_t overflow. */
if (extra >= arena_maxclass) if (unlikely(extra >= arena_maxclass))
return (true); return (true);
usize = s2u(size + extra); usize = s2u(size + extra);
@ -2142,7 +2142,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
/* /*
* Avoid moving the allocation if the size class can be left the same. * Avoid moving the allocation if the size class can be left the same.
*/ */
if (oldsize <= arena_maxclass) { if (likely(oldsize <= arena_maxclass)) {
if (oldsize <= SMALL_MAXCLASS) { if (oldsize <= SMALL_MAXCLASS) {
assert(arena_bin_info[size2index(oldsize)].reg_size assert(arena_bin_info[size2index(oldsize)].reg_size
== oldsize); == oldsize);

4
src/jemalloc.c
View File

@ -264,7 +264,7 @@ a0alloc(size_t size, bool zero)
if (size == 0) if (size == 0)
size = 1; size = 1;
if (size <= arena_maxclass) if (likely(size <= arena_maxclass))
ret = arena_malloc(NULL, a0get(), size, zero, false); ret = arena_malloc(NULL, a0get(), size, zero, false);
else else
ret = huge_malloc(NULL, a0get(), size, zero, false); ret = huge_malloc(NULL, a0get(), size, zero, false);
@ -295,7 +295,7 @@ a0free(void *ptr)
return; return;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) if (likely(chunk != ptr))
arena_dalloc(NULL, chunk, ptr, false); arena_dalloc(NULL, chunk, ptr, false);
else else
huge_dalloc(NULL, ptr, false); huge_dalloc(NULL, ptr, false);