diff options
Diffstat (limited to 'deps/jemalloc/src/arena.c')
| -rw-r--r-- | deps/jemalloc/src/arena.c | 3863 |
1 files changed, 3863 insertions, 0 deletions
diff --git a/deps/jemalloc/src/arena.c b/deps/jemalloc/src/arena.c new file mode 100644 index 0000000..648a8da --- /dev/null +++ b/deps/jemalloc/src/arena.c @@ -0,0 +1,3863 @@ +#define JEMALLOC_ARENA_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +purge_mode_t opt_purge = PURGE_DEFAULT; +const char *purge_mode_names[] = { + "ratio", + "decay", + "N/A" +}; +ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; +static ssize_t lg_dirty_mult_default; +ssize_t opt_decay_time = DECAY_TIME_DEFAULT; +static ssize_t decay_time_default; + +arena_bin_info_t arena_bin_info[NBINS]; + +size_t map_bias; +size_t map_misc_offset; +size_t arena_maxrun; /* Max run size for arenas. */ +size_t large_maxclass; /* Max large size class. */ +unsigned nlclasses; /* Number of large size classes. */ +unsigned nhclasses; /* Number of huge size classes. */ + +/******************************************************************************/ +/* + * Function prototypes for static functions that are referenced prior to + * definition. + */ + +static void arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, + arena_chunk_t *chunk); +static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, + size_t ndirty_limit); +static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, + bool dirty, bool cleaned, bool decommitted); +static void arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, + arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin); +static void arena_bin_lower_run(arena_t *arena, arena_run_t *run, + arena_bin_t *bin); + +/******************************************************************************/ + +JEMALLOC_INLINE_C size_t +arena_miscelm_size_get(const arena_chunk_map_misc_t *miscelm) +{ + arena_chunk_t *chunk; + size_t pageind, mapbits; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + pageind = arena_miscelm_to_pageind(miscelm); + mapbits = arena_mapbits_get(chunk, pageind); + return (arena_mapbits_size_decode(mapbits)); +} + +JEMALLOC_INLINE_C const extent_node_t * +arena_miscelm_extent_get(const arena_chunk_map_misc_t *miscelm) +{ + arena_chunk_t *chunk; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + return (&chunk->node); +} + +JEMALLOC_INLINE_C int +arena_sn_comp(const arena_chunk_map_misc_t *a, const arena_chunk_map_misc_t *b) +{ + size_t a_sn, b_sn; + + assert(a != NULL); + assert(b != NULL); + + a_sn = extent_node_sn_get(arena_miscelm_extent_get(a)); + b_sn = extent_node_sn_get(arena_miscelm_extent_get(b)); + + return ((a_sn > b_sn) - (a_sn < b_sn)); +} + +JEMALLOC_INLINE_C int +arena_ad_comp(const arena_chunk_map_misc_t *a, + const arena_chunk_map_misc_t *b) +{ + uintptr_t a_miscelm = (uintptr_t)a; + uintptr_t b_miscelm = (uintptr_t)b; + + assert(a != NULL); + assert(b != NULL); + + return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm)); +} + +JEMALLOC_INLINE_C int +arena_snad_comp(const arena_chunk_map_misc_t *a, + const arena_chunk_map_misc_t *b) +{ + int ret; + + assert(a != NULL); + assert(b != NULL); + + ret = arena_sn_comp(a, b); + if (ret != 0) + return (ret); + + ret = arena_ad_comp(a, b); + return (ret); +} + +/* Generate pairing heap functions. */ +ph_gen(static UNUSED, arena_run_heap_, arena_run_heap_t, arena_chunk_map_misc_t, + ph_link, arena_snad_comp) + +#ifdef JEMALLOC_JET +#undef run_quantize_floor +#define run_quantize_floor JEMALLOC_N(n_run_quantize_floor) +#endif +static size_t +run_quantize_floor(size_t size) +{ + size_t ret; + pszind_t pind; + + assert(size > 0); + assert(size <= HUGE_MAXCLASS); + assert((size & PAGE_MASK) == 0); + + assert(size != 0); + assert(size == PAGE_CEILING(size)); + + pind = psz2ind(size - large_pad + 1); + if (pind == 0) { + /* + * Avoid underflow. This short-circuit would also do the right + * thing for all sizes in the range for which there are + * PAGE-spaced size classes, but it's simplest to just handle + * the one case that would cause erroneous results. + */ + return (size); + } + ret = pind2sz(pind - 1) + large_pad; + assert(ret <= size); + return (ret); +} +#ifdef JEMALLOC_JET +#undef run_quantize_floor +#define run_quantize_floor JEMALLOC_N(run_quantize_floor) +run_quantize_t *run_quantize_floor = JEMALLOC_N(n_run_quantize_floor); +#endif + +#ifdef JEMALLOC_JET +#undef run_quantize_ceil +#define run_quantize_ceil JEMALLOC_N(n_run_quantize_ceil) +#endif +static size_t +run_quantize_ceil(size_t size) +{ + size_t ret; + + assert(size > 0); + assert(size <= HUGE_MAXCLASS); + assert((size & PAGE_MASK) == 0); + + ret = run_quantize_floor(size); + if (ret < size) { + /* + * Skip a quantization that may have an adequately large run, + * because under-sized runs may be mixed in. This only happens + * when an unusual size is requested, i.e. for aligned + * allocation, and is just one of several places where linear + * search would potentially find sufficiently aligned available + * memory somewhere lower. + */ + ret = pind2sz(psz2ind(ret - large_pad + 1)) + large_pad; + } + return (ret); +} +#ifdef JEMALLOC_JET +#undef run_quantize_ceil +#define run_quantize_ceil JEMALLOC_N(run_quantize_ceil) +run_quantize_t *run_quantize_ceil = JEMALLOC_N(n_run_quantize_ceil); +#endif + +static void +arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages) +{ + pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get( + arena_miscelm_get_const(chunk, pageind)))); + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + assert((npages << LG_PAGE) < chunksize); + assert(pind2sz(pind) <= chunksize); + arena_run_heap_insert(&arena->runs_avail[pind], + arena_miscelm_get_mutable(chunk, pageind)); +} + +static void +arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages) +{ + pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get( + arena_miscelm_get_const(chunk, pageind)))); + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + assert((npages << LG_PAGE) < chunksize); + assert(pind2sz(pind) <= chunksize); + arena_run_heap_remove(&arena->runs_avail[pind], + arena_miscelm_get_mutable(chunk, pageind)); +} + +static void +arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages) +{ + arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, + pageind); + + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); + assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == + CHUNK_MAP_DIRTY); + + qr_new(&miscelm->rd, rd_link); + qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link); + arena->ndirty += npages; +} + +static void +arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages) +{ + arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, + pageind); + + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); + assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == + CHUNK_MAP_DIRTY); + + qr_remove(&miscelm->rd, rd_link); + assert(arena->ndirty >= npages); + arena->ndirty -= npages; +} + +static size_t +arena_chunk_dirty_npages(const extent_node_t *node) +{ + + return (extent_node_size_get(node) >> LG_PAGE); +} + +void +arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, bool cache) +{ + + if (cache) { + extent_node_dirty_linkage_init(node); + extent_node_dirty_insert(node, &arena->runs_dirty, + &arena->chunks_cache); + arena->ndirty += arena_chunk_dirty_npages(node); + } +} + +void +arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty) +{ + + if (dirty) { + extent_node_dirty_remove(node); + assert(arena->ndirty >= arena_chunk_dirty_npages(node)); + arena->ndirty -= arena_chunk_dirty_npages(node); + } +} + +JEMALLOC_INLINE_C void * +arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) +{ + void *ret; + size_t regind; + arena_chunk_map_misc_t *miscelm; + void *rpages; + + assert(run->nfree > 0); + assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info)); + + regind = (unsigned)bitmap_sfu(run->bitmap, &bin_info->bitmap_info); + miscelm = arena_run_to_miscelm(run); + rpages = arena_miscelm_to_rpages(miscelm); + ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset + + (uintptr_t)(bin_info->reg_interval * regind)); + run->nfree--; + return (ret); +} + +JEMALLOC_INLINE_C void +arena_run_reg_dalloc(arena_run_t *run, void *ptr) +{ + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t mapbits = arena_mapbits_get(chunk, pageind); + szind_t binind = arena_ptr_small_binind_get(ptr, mapbits); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + size_t regind = arena_run_regind(run, bin_info, ptr); + + assert(run->nfree < bin_info->nregs); + /* Freeing an interior pointer can cause assertion failure. */ + assert(((uintptr_t)ptr - + ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + + (uintptr_t)bin_info->reg0_offset)) % + (uintptr_t)bin_info->reg_interval == 0); + assert((uintptr_t)ptr >= + (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + + (uintptr_t)bin_info->reg0_offset); + /* Freeing an unallocated pointer can cause assertion failure. */ + assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)); + + bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind); + run->nfree++; +} + +JEMALLOC_INLINE_C void +arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) +{ + + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (npages << LG_PAGE)); + memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, + (npages << LG_PAGE)); +} + +JEMALLOC_INLINE_C void +arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) +{ + + JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind + << LG_PAGE)), PAGE); +} + +JEMALLOC_INLINE_C void +arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) +{ + size_t i; + UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); + + arena_run_page_mark_zeroed(chunk, run_ind); + for (i = 0; i < PAGE / sizeof(size_t); i++) + assert(p[i] == 0); +} + +static void +arena_nactive_add(arena_t *arena, size_t add_pages) +{ + + if (config_stats) { + size_t cactive_add = CHUNK_CEILING((arena->nactive + + add_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << + LG_PAGE); + if (cactive_add != 0) + stats_cactive_add(cactive_add); + } + arena->nactive += add_pages; +} + +static void +arena_nactive_sub(arena_t *arena, size_t sub_pages) +{ + + if (config_stats) { + size_t cactive_sub = CHUNK_CEILING(arena->nactive << LG_PAGE) - + CHUNK_CEILING((arena->nactive - sub_pages) << LG_PAGE); + if (cactive_sub != 0) + stats_cactive_sub(cactive_sub); + } + arena->nactive -= sub_pages; +} + +static void +arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, + size_t flag_dirty, size_t flag_decommitted, size_t need_pages) +{ + size_t total_pages, rem_pages; + + assert(flag_dirty == 0 || flag_decommitted == 0); + + total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> + LG_PAGE; + assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == + flag_dirty); + assert(need_pages <= total_pages); + rem_pages = total_pages - need_pages; + + arena_avail_remove(arena, chunk, run_ind, total_pages); + if (flag_dirty != 0) + arena_run_dirty_remove(arena, chunk, run_ind, total_pages); + arena_nactive_add(arena, need_pages); + + /* Keep track of trailing unused pages for later use. */ + if (rem_pages > 0) { + size_t flags = flag_dirty | flag_decommitted; + size_t flag_unzeroed_mask = (flags == 0) ? CHUNK_MAP_UNZEROED : + 0; + + arena_mapbits_unallocated_set(chunk, run_ind+need_pages, + (rem_pages << LG_PAGE), flags | + (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & + flag_unzeroed_mask)); + arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, + (rem_pages << LG_PAGE), flags | + (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & + flag_unzeroed_mask)); + if (flag_dirty != 0) { + arena_run_dirty_insert(arena, chunk, run_ind+need_pages, + rem_pages); + } + arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); + } +} + +static bool +arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, + bool remove, bool zero) +{ + arena_chunk_t *chunk; + arena_chunk_map_misc_t *miscelm; + size_t flag_dirty, flag_decommitted, run_ind, need_pages; + size_t flag_unzeroed_mask; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + miscelm = arena_run_to_miscelm(run); + run_ind = arena_miscelm_to_pageind(miscelm); + flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); + flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); + need_pages = (size >> LG_PAGE); + assert(need_pages > 0); + + if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, + run_ind << LG_PAGE, size, arena->ind)) + return (true); + + if (remove) { + arena_run_split_remove(arena, chunk, run_ind, flag_dirty, + flag_decommitted, need_pages); + } + + if (zero) { + if (flag_decommitted != 0) { + /* The run is untouched, and therefore zeroed. */ + JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void + *)((uintptr_t)chunk + (run_ind << LG_PAGE)), + (need_pages << LG_PAGE)); + } else if (flag_dirty != 0) { + /* The run is dirty, so all pages must be zeroed. */ + arena_run_zero(chunk, run_ind, need_pages); + } else { + /* + * The run is clean, so some pages may be zeroed (i.e. + * never before touched). + */ + size_t i; + for (i = 0; i < need_pages; i++) { + if (arena_mapbits_unzeroed_get(chunk, run_ind+i) + != 0) + arena_run_zero(chunk, run_ind+i, 1); + else if (config_debug) { + arena_run_page_validate_zeroed(chunk, + run_ind+i); + } else { + arena_run_page_mark_zeroed(chunk, + run_ind+i); + } + } + } + } else { + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); + } + + /* + * Set the last element first, in case the run only contains one page + * (i.e. both statements set the same element). + */ + flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? + CHUNK_MAP_UNZEROED : 0; + arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | + (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, + run_ind+need_pages-1))); + arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | + (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); + return (false); +} + +static bool +arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) +{ + + return (arena_run_split_large_helper(arena, run, size, true, zero)); +} + +static bool +arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) +{ + + return (arena_run_split_large_helper(arena, run, size, false, zero)); +} + +static bool +arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, + szind_t binind) +{ + arena_chunk_t *chunk; + arena_chunk_map_misc_t *miscelm; + size_t flag_dirty, flag_decommitted, run_ind, need_pages, i; + + assert(binind != BININD_INVALID); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + miscelm = arena_run_to_miscelm(run); + run_ind = arena_miscelm_to_pageind(miscelm); + flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); + flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); + need_pages = (size >> LG_PAGE); + assert(need_pages > 0); + + if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, + run_ind << LG_PAGE, size, arena->ind)) + return (true); + + arena_run_split_remove(arena, chunk, run_ind, flag_dirty, + flag_decommitted, need_pages); + + for (i = 0; i < need_pages; i++) { + size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk, + run_ind+i); + arena_mapbits_small_set(chunk, run_ind+i, i, binind, + flag_unzeroed); + if (config_debug && flag_dirty == 0 && flag_unzeroed == 0) + arena_run_page_validate_zeroed(chunk, run_ind+i); + } + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); + return (false); +} + +static arena_chunk_t * +arena_chunk_init_spare(arena_t *arena) +{ + arena_chunk_t *chunk; + + assert(arena->spare != NULL); + + chunk = arena->spare; + arena->spare = NULL; + + assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); + assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); + assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == + arena_maxrun); + assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == + arena_maxrun); + assert(arena_mapbits_dirty_get(chunk, map_bias) == + arena_mapbits_dirty_get(chunk, chunk_npages-1)); + + return (chunk); +} + +static bool +arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + size_t sn, bool zero) +{ + + /* + * The extent node notion of "committed" doesn't directly apply to + * arena chunks. Arbitrarily mark them as committed. The commit state + * of runs is tracked individually, and upon chunk deallocation the + * entire chunk is in a consistent commit state. + */ + extent_node_init(&chunk->node, arena, chunk, chunksize, sn, zero, true); + extent_node_achunk_set(&chunk->node, true); + return (chunk_register(tsdn, chunk, &chunk->node)); +} + +static arena_chunk_t * +arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, + chunk_hooks_t *chunk_hooks, bool *zero, bool *commit) +{ + arena_chunk_t *chunk; + size_t sn; + + malloc_mutex_unlock(tsdn, &arena->lock); + + chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks, + NULL, chunksize, chunksize, &sn, zero, commit); + if (chunk != NULL && !*commit) { + /* Commit header. */ + if (chunk_hooks->commit(chunk, chunksize, 0, map_bias << + LG_PAGE, arena->ind)) { + chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, + (void *)chunk, chunksize, sn, *zero, *commit); + chunk = NULL; + } + } + if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, sn, + *zero)) { + if (!*commit) { + /* Undo commit of header. */ + chunk_hooks->decommit(chunk, chunksize, 0, map_bias << + LG_PAGE, arena->ind); + } + chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk, + chunksize, sn, *zero, *commit); + chunk = NULL; + } + + malloc_mutex_lock(tsdn, &arena->lock); + return (chunk); +} + +static arena_chunk_t * +arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, + bool *commit) +{ + arena_chunk_t *chunk; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; + size_t sn; + + chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize, + chunksize, &sn, zero, commit, true); + if (chunk != NULL) { + if (arena_chunk_register(tsdn, arena, chunk, sn, *zero)) { + chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, + chunksize, sn, true); + return (NULL); + } + } + if (chunk == NULL) { + chunk = arena_chunk_alloc_internal_hard(tsdn, arena, + &chunk_hooks, zero, commit); + } + + if (config_stats && chunk != NULL) { + arena->stats.mapped += chunksize; + arena->stats.metadata_mapped += (map_bias << LG_PAGE); + } + + return (chunk); +} + +static arena_chunk_t * +arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena) +{ + arena_chunk_t *chunk; + bool zero, commit; + size_t flag_unzeroed, flag_decommitted, i; + + assert(arena->spare == NULL); + + zero = false; + commit = false; + chunk = arena_chunk_alloc_internal(tsdn, arena, &zero, &commit); + if (chunk == NULL) + return (NULL); + + chunk->hugepage = true; + + /* + * Initialize the map to contain one maximal free untouched run. Mark + * the pages as zeroed if arena_chunk_alloc_internal() returned a zeroed + * or decommitted chunk. + */ + flag_unzeroed = (zero || !commit) ? 0 : CHUNK_MAP_UNZEROED; + flag_decommitted = commit ? 0 : CHUNK_MAP_DECOMMITTED; + arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, + flag_unzeroed | flag_decommitted); + /* + * There is no need to initialize the internal page map entries unless + * the chunk is not zeroed. + */ + if (!zero) { + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( + (void *)arena_bitselm_get_const(chunk, map_bias+1), + (size_t)((uintptr_t)arena_bitselm_get_const(chunk, + chunk_npages-1) - + (uintptr_t)arena_bitselm_get_const(chunk, map_bias+1))); + for (i = map_bias+1; i < chunk_npages-1; i++) + arena_mapbits_internal_set(chunk, i, flag_unzeroed); + } else { + JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void + *)arena_bitselm_get_const(chunk, map_bias+1), + (size_t)((uintptr_t)arena_bitselm_get_const(chunk, + chunk_npages-1) - + (uintptr_t)arena_bitselm_get_const(chunk, map_bias+1))); + if (config_debug) { + for (i = map_bias+1; i < chunk_npages-1; i++) { + assert(arena_mapbits_unzeroed_get(chunk, i) == + flag_unzeroed); + } + } + } + arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun, + flag_unzeroed); + + return (chunk); +} + +static arena_chunk_t * +arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena) +{ + arena_chunk_t *chunk; + + if (arena->spare != NULL) + chunk = arena_chunk_init_spare(arena); + else { + chunk = arena_chunk_init_hard(tsdn, arena); + if (chunk == NULL) + return (NULL); + } + + ql_elm_new(&chunk->node, ql_link); + ql_tail_insert(&arena->achunks, &chunk->node, ql_link); + arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); + + return (chunk); +} + +static void +arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) +{ + size_t sn, hugepage; + bool committed; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; + + chunk_deregister(chunk, &chunk->node); + + sn = extent_node_sn_get(&chunk->node); + hugepage = chunk->hugepage; + committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0); + if (!committed) { + /* + * Decommit the header. Mark the chunk as decommitted even if + * header decommit fails, since treating a partially committed + * chunk as committed has a high potential for causing later + * access of decommitted memory. + */ + chunk_hooks = chunk_hooks_get(tsdn, arena); + chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE, + arena->ind); + } + if (!hugepage) { + /* + * Convert chunk back to the default state, so that all + * subsequent chunk allocations start out with chunks that can + * be backed by transparent huge pages. + */ + pages_huge(chunk, chunksize); + } + + chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize, + sn, committed); + + if (config_stats) { + arena->stats.mapped -= chunksize; + arena->stats.metadata_mapped -= (map_bias << LG_PAGE); + } +} + +static void +arena_spare_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *spare) +{ + + assert(arena->spare != spare); + + if (arena_mapbits_dirty_get(spare, map_bias) != 0) { + arena_run_dirty_remove(arena, spare, map_bias, + chunk_npages-map_bias); + } + + arena_chunk_discard(tsdn, arena, spare); +} + +static void +arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) +{ + arena_chunk_t *spare; + + assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); + assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); + assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == + arena_maxrun); + assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == + arena_maxrun); + assert(arena_mapbits_dirty_get(chunk, map_bias) == + arena_mapbits_dirty_get(chunk, chunk_npages-1)); + assert(arena_mapbits_decommitted_get(chunk, map_bias) == + arena_mapbits_decommitted_get(chunk, chunk_npages-1)); + + /* Remove run from runs_avail, so that the arena does not use it. */ + arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); + + ql_remove(&arena->achunks, &chunk->node, ql_link); + spare = arena->spare; + arena->spare = chunk; + if (spare != NULL) + arena_spare_discard(tsdn, arena, spare); +} + +static void +arena_huge_malloc_stats_update(arena_t *arena, size_t usize) +{ + szind_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.nmalloc_huge++; + arena->stats.allocated_huge += usize; + arena->stats.hstats[index].nmalloc++; + arena->stats.hstats[index].curhchunks++; +} + +static void +arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) +{ + szind_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.nmalloc_huge--; + arena->stats.allocated_huge -= usize; + arena->stats.hstats[index].nmalloc--; + arena->stats.hstats[index].curhchunks--; +} + +static void +arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) +{ + szind_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.ndalloc_huge++; + arena->stats.allocated_huge -= usize; + arena->stats.hstats[index].ndalloc++; + arena->stats.hstats[index].curhchunks--; +} + +static void +arena_huge_reset_stats_cancel(arena_t *arena, size_t usize) +{ + szind_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.ndalloc_huge++; + arena->stats.hstats[index].ndalloc--; +} + +static void +arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize) +{ + szind_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.ndalloc_huge--; + arena->stats.allocated_huge += usize; + arena->stats.hstats[index].ndalloc--; + arena->stats.hstats[index].curhchunks++; +} + +static void +arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize) +{ + + arena_huge_dalloc_stats_update(arena, oldsize); + arena_huge_malloc_stats_update(arena, usize); +} + +static void +arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize, + size_t usize) +{ + + arena_huge_dalloc_stats_update_undo(arena, oldsize); + arena_huge_malloc_stats_update_undo(arena, usize); +} + +extent_node_t * +arena_node_alloc(tsdn_t *tsdn, arena_t *arena) +{ + extent_node_t *node; + + malloc_mutex_lock(tsdn, &arena->node_cache_mtx); + node = ql_last(&arena->node_cache, ql_link); + if (node == NULL) { + malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); + return (base_alloc(tsdn, sizeof(extent_node_t))); + } + ql_tail_remove(&arena->node_cache, extent_node_t, ql_link); + malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); + return (node); +} + +void +arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node) +{ + + malloc_mutex_lock(tsdn, &arena->node_cache_mtx); + ql_elm_new(node, ql_link); + ql_tail_insert(&arena->node_cache, node, ql_link); + malloc_mutex_unlock(tsdn, &arena->node_cache_mtx); +} + +static void * +arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena, + chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, size_t *sn, + bool *zero, size_t csize) +{ + void *ret; + bool commit = true; + + ret = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, csize, + alignment, sn, zero, &commit); + if (ret == NULL) { + /* Revert optimistic stats updates. */ + malloc_mutex_lock(tsdn, &arena->lock); + if (config_stats) { + arena_huge_malloc_stats_update_undo(arena, usize); + arena->stats.mapped -= usize; + } + arena_nactive_sub(arena, usize >> LG_PAGE); + malloc_mutex_unlock(tsdn, &arena->lock); + } + + return (ret); +} + +void * +arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, + size_t alignment, size_t *sn, bool *zero) +{ + void *ret; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; + size_t csize = CHUNK_CEILING(usize); + bool commit = true; + + malloc_mutex_lock(tsdn, &arena->lock); + + /* Optimistically update stats. */ + if (config_stats) { + arena_huge_malloc_stats_update(arena, usize); + arena->stats.mapped += usize; + } + arena_nactive_add(arena, usize >> LG_PAGE); + + ret = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, csize, + alignment, sn, zero, &commit, true); + malloc_mutex_unlock(tsdn, &arena->lock); + if (ret == NULL) { + ret = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks, + usize, alignment, sn, zero, csize); + } + + return (ret); +} + +void +arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize, + size_t sn) +{ + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; + size_t csize; + + csize = CHUNK_CEILING(usize); + malloc_mutex_lock(tsdn, &arena->lock); + if (config_stats) { + arena_huge_dalloc_stats_update(arena, usize); + arena->stats.mapped -= usize; + } + arena_nactive_sub(arena, usize >> LG_PAGE); + + chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, sn, true); + malloc_mutex_unlock(tsdn, &arena->lock); +} + +void +arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk, + size_t oldsize, size_t usize) +{ + + assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize)); + assert(oldsize != usize); + + malloc_mutex_lock(tsdn, &arena->lock); + if (config_stats) + arena_huge_ralloc_stats_update(arena, oldsize, usize); + if (oldsize < usize) + arena_nactive_add(arena, (usize - oldsize) >> LG_PAGE); + else + arena_nactive_sub(arena, (oldsize - usize) >> LG_PAGE); + malloc_mutex_unlock(tsdn, &arena->lock); +} + +void +arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk, + size_t oldsize, size_t usize, size_t sn) +{ + size_t udiff = oldsize - usize; + size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); + + malloc_mutex_lock(tsdn, &arena->lock); + if (config_stats) { + arena_huge_ralloc_stats_update(arena, oldsize, usize); + if (cdiff != 0) + arena->stats.mapped -= cdiff; + } + arena_nactive_sub(arena, udiff >> LG_PAGE); + + if (cdiff != 0) { + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; + void *nchunk = (void *)((uintptr_t)chunk + + CHUNK_CEILING(usize)); + + chunk_dalloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff, + sn, true); + } + malloc_mutex_unlock(tsdn, &arena->lock); +} + +static bool +arena_chunk_ralloc_huge_expand_hard(tsdn_t *tsdn, arena_t *arena, + chunk_hooks_t *chunk_hooks, void *chunk, size_t oldsize, size_t usize, + size_t *sn, bool *zero, void *nchunk, size_t udiff, size_t cdiff) +{ + bool err; + bool commit = true; + + err = (chunk_alloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff, + chunksize, sn, zero, &commit) == NULL); + if (err) { + /* Revert optimistic stats updates. */ + malloc_mutex_lock(tsdn, &arena->lock); + if (config_stats) { + arena_huge_ralloc_stats_update_undo(arena, oldsize, + usize); + arena->stats.mapped -= cdiff; + } + arena_nactive_sub(arena, udiff >> LG_PAGE); + malloc_mutex_unlock(tsdn, &arena->lock); + } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk, + cdiff, true, arena->ind)) { + chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff, + *sn, *zero, true); + err = true; + } + return (err); +} + +bool +arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk, + size_t oldsize, size_t usize, bool *zero) +{ + bool err; + chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); + void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); + size_t udiff = usize - oldsize; + size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); + size_t sn; + bool commit = true; + + malloc_mutex_lock(tsdn, &arena->lock); + + /* Optimistically update stats. */ + if (config_stats) { + arena_huge_ralloc_stats_update(arena, oldsize, usize); + arena->stats.mapped += cdiff; + } + arena_nactive_add(arena, udiff >> LG_PAGE); + + err = (chunk_alloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff, + chunksize, &sn, zero, &commit, true) == NULL); + malloc_mutex_unlock(tsdn, &arena->lock); + if (err) { + err = arena_chunk_ralloc_huge_expand_hard(tsdn, arena, + &chunk_hooks, chunk, oldsize, usize, &sn, zero, nchunk, + udiff, cdiff); + } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk, + cdiff, true, arena->ind)) { + chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, nchunk, cdiff, + sn, *zero, true); + err = true; + } + + return (err); +} + +/* + * Do first-best-fit run selection, i.e. select the lowest run that best fits. + * Run sizes are indexed, so not all candidate runs are necessarily exactly the + * same size. + */ +static arena_run_t * +arena_run_first_best_fit(arena_t *arena, size_t size) +{ + pszind_t pind, i; + + pind = psz2ind(run_quantize_ceil(size)); + + for (i = pind; pind2sz(i) <= chunksize; i++) { + arena_chunk_map_misc_t *miscelm = arena_run_heap_first( + &arena->runs_avail[i]); + if (miscelm != NULL) + return (&miscelm->run); + } + + return (NULL); +} + +static arena_run_t * +arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) +{ + arena_run_t *run = arena_run_first_best_fit(arena, size); + if (run != NULL) { + if (arena_run_split_large(arena, run, size, zero)) + run = NULL; + } + return (run); +} + +static arena_run_t * +arena_run_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t size, bool zero) +{ + arena_chunk_t *chunk; + arena_run_t *run; + + assert(size <= arena_maxrun); + assert(size == PAGE_CEILING(size)); + + /* Search the arena's chunks for the lowest best fit. */ + run = arena_run_alloc_large_helper(arena, size, zero); + if (run != NULL) + return (run); + + /* + * No usable runs. Create a new chunk from which to allocate the run. + */ + chunk = arena_chunk_alloc(tsdn, arena); + if (chunk != NULL) { + run = &arena_miscelm_get_mutable(chunk, map_bias)->run; + if (arena_run_split_large(arena, run, size, zero)) + run = NULL; + return (run); + } + + /* + * arena_chunk_alloc() failed, but another thread may have made + * sufficient memory available while this one dropped arena->lock in + * arena_chunk_alloc(), so search one more time. + */ + return (arena_run_alloc_large_helper(arena, size, zero)); +} + +static arena_run_t * +arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind) +{ + arena_run_t *run = arena_run_first_best_fit(arena, size); + if (run != NULL) { + if (arena_run_split_small(arena, run, size, binind)) + run = NULL; + } + return (run); +} + +static arena_run_t * +arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind) +{ + arena_chunk_t *chunk; + arena_run_t *run; + + assert(size <= arena_maxrun); + assert(size == PAGE_CEILING(size)); + assert(binind != BININD_INVALID); + + /* Search the arena's chunks for the lowest best fit. */ + run = arena_run_alloc_small_helper(arena, size, binind); + if (run != NULL) + return (run); + + /* + * No usable runs. Create a new chunk from which to allocate the run. + */ + chunk = arena_chunk_alloc(tsdn, arena); + if (chunk != NULL) { + run = &arena_miscelm_get_mutable(chunk, map_bias)->run; + if (arena_run_split_small(arena, run, size, binind)) + run = NULL; + return (run); + } + + /* + * arena_chunk_alloc() failed, but another thread may have made + * sufficient memory available while this one dropped arena->lock in + * arena_chunk_alloc(), so search one more time. + */ + return (arena_run_alloc_small_helper(arena, size, binind)); +} + +static bool +arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult) +{ + + return (lg_dirty_mult >= -1 && lg_dirty_mult < (ssize_t)(sizeof(size_t) + << 3)); +} + +ssize_t +arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena) +{ + ssize_t lg_dirty_mult; + + malloc_mutex_lock(tsdn, &arena->lock); + lg_dirty_mult = arena->lg_dirty_mult; + malloc_mutex_unlock(tsdn, &arena->lock); + + return (lg_dirty_mult); +} + +bool +arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena, ssize_t lg_dirty_mult) +{ + + if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) + return (true); + + malloc_mutex_lock(tsdn, &arena->lock); + arena->lg_dirty_mult = lg_dirty_mult; + arena_maybe_purge(tsdn, arena); + malloc_mutex_unlock(tsdn, &arena->lock); + + return (false); +} + +static void +arena_decay_deadline_init(arena_t *arena) +{ + + assert(opt_purge == purge_mode_decay); + + /* + * Generate a new deadline that is uniformly random within the next + * epoch after the current one. + */ + nstime_copy(&arena->decay.deadline, &arena->decay.epoch); + nstime_add(&arena->decay.deadline, &arena->decay.interval); + if (arena->decay.time > 0) { + nstime_t jitter; + + nstime_init(&jitter, prng_range_u64(&arena->decay.jitter_state, + nstime_ns(&arena->decay.interval))); + nstime_add(&arena->decay.deadline, &jitter); + } +} + +static bool +arena_decay_deadline_reached(const arena_t *arena, const nstime_t *time) +{ + + assert(opt_purge == purge_mode_decay); + + return (nstime_compare(&arena->decay.deadline, time) <= 0); +} + +static size_t +arena_decay_backlog_npages_limit(const arena_t *arena) +{ + static const uint64_t h_steps[] = { +#define STEP(step, h, x, y) \ + h, + SMOOTHSTEP +#undef STEP + }; + uint64_t sum; + size_t npages_limit_backlog; + unsigned i; + + assert(opt_purge == purge_mode_decay); + + /* + * For each element of decay_backlog, multiply by the corresponding + * fixed-point smoothstep decay factor. Sum the products, then divide + * to round down to the nearest whole number of pages. + */ + sum = 0; + for (i = 0; i < SMOOTHSTEP_NSTEPS; i++) + sum += arena->decay.backlog[i] * h_steps[i]; + npages_limit_backlog = (size_t)(sum >> SMOOTHSTEP_BFP); + + return (npages_limit_backlog); +} + +static void +arena_decay_backlog_update_last(arena_t *arena) +{ + size_t ndirty_delta = (arena->ndirty > arena->decay.ndirty) ? + arena->ndirty - arena->decay.ndirty : 0; + arena->decay.backlog[SMOOTHSTEP_NSTEPS-1] = ndirty_delta; +} + +static void +arena_decay_backlog_update(arena_t *arena, uint64_t nadvance_u64) +{ + + if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) { + memset(arena->decay.backlog, 0, (SMOOTHSTEP_NSTEPS-1) * + sizeof(size_t)); + } else { + size_t nadvance_z = (size_t)nadvance_u64; + + assert((uint64_t)nadvance_z == nadvance_u64); + + memmove(arena->decay.backlog, &arena->decay.backlog[nadvance_z], + (SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t)); + if (nadvance_z > 1) { + memset(&arena->decay.backlog[SMOOTHSTEP_NSTEPS - + nadvance_z], 0, (nadvance_z-1) * sizeof(size_t)); + } + } + + arena_decay_backlog_update_last(arena); +} + +static void +arena_decay_epoch_advance_helper(arena_t *arena, const nstime_t *time) +{ + uint64_t nadvance_u64; + nstime_t delta; + + assert(opt_purge == purge_mode_decay); + assert(arena_decay_deadline_reached(arena, time)); + + nstime_copy(&delta, time); + nstime_subtract(&delta, &arena->decay.epoch); + nadvance_u64 = nstime_divide(&delta, &arena->decay.interval); + assert(nadvance_u64 > 0); + + /* Add nadvance_u64 decay intervals to epoch. */ + nstime_copy(&delta, &arena->decay.interval); + nstime_imultiply(&delta, nadvance_u64); + nstime_add(&arena->decay.epoch, &delta); + + /* Set a new deadline. */ + arena_decay_deadline_init(arena); + + /* Update the backlog. */ + arena_decay_backlog_update(arena, nadvance_u64); +} + +static void +arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena) +{ + size_t ndirty_limit = arena_decay_backlog_npages_limit(arena); + + if (arena->ndirty > ndirty_limit) + arena_purge_to_limit(tsdn, arena, ndirty_limit); + arena->decay.ndirty = arena->ndirty; +} + +static void +arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, const nstime_t *time) +{ + + arena_decay_epoch_advance_helper(arena, time); + arena_decay_epoch_advance_purge(tsdn, arena); +} + +static void +arena_decay_init(arena_t *arena, ssize_t decay_time) +{ + + arena->decay.time = decay_time; + if (decay_time > 0) { + nstime_init2(&arena->decay.interval, decay_time, 0); + nstime_idivide(&arena->decay.interval, SMOOTHSTEP_NSTEPS); + } + + nstime_init(&arena->decay.epoch, 0); + nstime_update(&arena->decay.epoch); + arena->decay.jitter_state = (uint64_t)(uintptr_t)arena; + arena_decay_deadline_init(arena); + arena->decay.ndirty = arena->ndirty; + memset(arena->decay.backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t)); +} + +static bool +arena_decay_time_valid(ssize_t decay_time) +{ + + if (decay_time < -1) + return (false); + if (decay_time == -1 || (uint64_t)decay_time <= NSTIME_SEC_MAX) + return (true); + return (false); +} + +ssize_t +arena_decay_time_get(tsdn_t *tsdn, arena_t *arena) +{ + ssize_t decay_time; + + malloc_mutex_lock(tsdn, &arena->lock); + decay_time = arena->decay.time; + malloc_mutex_unlock(tsdn, &arena->lock); + + return (decay_time); +} + +bool +arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) +{ + + if (!arena_decay_time_valid(decay_time)) + return (true); + + malloc_mutex_lock(tsdn, &arena->lock); + /* + * Restart decay backlog from scratch, which may cause many dirty pages + * to be immediately purged. It would conceptually be possible to map + * the old backlog onto the new backlog, but there is no justification + * for such complexity since decay_time changes are intended to be + * infrequent, either between the {-1, 0, >0} states, or a one-time + * arbitrary change during initial arena configuration. + */ + arena_decay_init(arena, decay_time); + arena_maybe_purge(tsdn, arena); + malloc_mutex_unlock(tsdn, &arena->lock); + + return (false); +} + +static void +arena_maybe_purge_ratio(tsdn_t *tsdn, arena_t *arena) +{ + + assert(opt_purge == purge_mode_ratio); + + /* Don't purge if the option is disabled. */ + if (arena->lg_dirty_mult < 0) + return; + + /* + * Iterate, since preventing recursive purging could otherwise leave too + * many dirty pages. + */ + while (true) { + size_t threshold = (arena->nactive >> arena->lg_dirty_mult); + if (threshold < chunk_npages) + threshold = chunk_npages; + /* + * Don't purge unless the number of purgeable pages exceeds the + * threshold. + */ + if (arena->ndirty <= threshold) + return; + arena_purge_to_limit(tsdn, arena, threshold); + } +} + +static void +arena_maybe_purge_decay(tsdn_t *tsdn, arena_t *arena) +{ + nstime_t time; + + assert(opt_purge == purge_mode_decay); + + /* Purge all or nothing if the option is disabled. */ + if (arena->decay.time <= 0) { + if (arena->decay.time == 0) + arena_purge_to_limit(tsdn, arena, 0); + return; + } + + nstime_init(&time, 0); + nstime_update(&time); + if (unlikely(!nstime_monotonic() && nstime_compare(&arena->decay.epoch, + &time) > 0)) { + /* + * Time went backwards. Move the epoch back in time and + * generate a new deadline, with the expectation that time + * typically flows forward for long enough periods of time that + * epochs complete. Unfortunately, this strategy is susceptible + * to clock jitter triggering premature epoch advances, but + * clock jitter estimation and compensation isn't feasible here + * because calls into this code are event-driven. + */ + nstime_copy(&arena->decay.epoch, &time); + arena_decay_deadline_init(arena); + } else { + /* Verify that time does not go backwards. */ + assert(nstime_compare(&arena->decay.epoch, &time) <= 0); + } + + /* + * If the deadline has been reached, advance to the current epoch and + * purge to the new limit if necessary. Note that dirty pages created + * during the current epoch are not subject to purge until a future + * epoch, so as a result purging only happens during epoch advances. + */ + if (arena_decay_deadline_reached(arena, &time)) + arena_decay_epoch_advance(tsdn, arena, &time); +} + +void +arena_maybe_purge(tsdn_t *tsdn, arena_t *arena) +{ + + /* Don't recursively purge. */ + if (arena->purging) + return; + + if (opt_purge == purge_mode_ratio) + arena_maybe_purge_ratio(tsdn, arena); + else + arena_maybe_purge_decay(tsdn, arena); +} + +static size_t +arena_dirty_count(arena_t *arena) +{ + size_t ndirty = 0; + arena_runs_dirty_link_t *rdelm; + extent_node_t *chunkselm; + + for (rdelm = qr_next(&arena->runs_dirty, rd_link), + chunkselm = qr_next(&arena->chunks_cache, cc_link); + rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) { + size_t npages; + + if (rdelm == &chunkselm->rd) { + npages = extent_node_size_get(chunkselm) >> LG_PAGE; + chunkselm = qr_next(chunkselm, cc_link); + } else { + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( + rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); + assert(arena_mapbits_allocated_get(chunk, pageind) == + 0); + assert(arena_mapbits_large_get(chunk, pageind) == 0); + assert(arena_mapbits_dirty_get(chunk, pageind) != 0); + npages = arena_mapbits_unallocated_size_get(chunk, + pageind) >> LG_PAGE; + } + ndirty += npages; + } + + return (ndirty); +} + +static size_t +arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, + size_t ndirty_limit, arena_runs_dirty_link_t *purge_runs_sentinel, + extent_node_t *purge_chunks_sentinel) +{ + arena_runs_dirty_link_t *rdelm, *rdelm_next; + extent_node_t *chunkselm; + size_t nstashed = 0; + + /* Stash runs/chunks according to ndirty_limit. */ + for (rdelm = qr_next(&arena->runs_dirty, rd_link), + chunkselm = qr_next(&arena->chunks_cache, cc_link); + rdelm != &arena->runs_dirty; rdelm = rdelm_next) { + size_t npages; + rdelm_next = qr_next(rdelm, rd_link); + + if (rdelm == &chunkselm->rd) { + extent_node_t *chunkselm_next; + size_t sn; + bool zero, commit; + UNUSED void *chunk; + + npages = extent_node_size_get(chunkselm) >> LG_PAGE; + if (opt_purge == purge_mode_decay && arena->ndirty - + (nstashed + npages) < ndirty_limit) + break; + + chunkselm_next = qr_next(chunkselm, cc_link); + /* + * Allocate. chunkselm remains valid due to the + * dalloc_node=false argument to chunk_alloc_cache(). + */ + zero = false; + commit = false; + chunk = chunk_alloc_cache(tsdn, arena, chunk_hooks, + extent_node_addr_get(chunkselm), + extent_node_size_get(chunkselm), chunksize, &sn, + &zero, &commit, false); + assert(chunk == extent_node_addr_get(chunkselm)); + assert(zero == extent_node_zeroed_get(chunkselm)); + extent_node_dirty_insert(chunkselm, purge_runs_sentinel, + purge_chunks_sentinel); + assert(npages == (extent_node_size_get(chunkselm) >> + LG_PAGE)); + chunkselm = chunkselm_next; + } else { + arena_chunk_t *chunk = + (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); + arena_run_t *run = &miscelm->run; + size_t run_size = + arena_mapbits_unallocated_size_get(chunk, pageind); + + npages = run_size >> LG_PAGE; + if (opt_purge == purge_mode_decay && arena->ndirty - + (nstashed + npages) < ndirty_limit) + break; + + assert(pageind + npages <= chunk_npages); + assert(arena_mapbits_dirty_get(chunk, pageind) == + arena_mapbits_dirty_get(chunk, pageind+npages-1)); + + /* + * If purging the spare chunk's run, make it available + * prior to allocation. + */ + if (chunk == arena->spare) + arena_chunk_alloc(tsdn, arena); + + /* Temporarily allocate the free dirty run. */ + arena_run_split_large(arena, run, run_size, false); + /* Stash. */ + if (false) + qr_new(rdelm, rd_link); /* Redundant. */ + else { + assert(qr_next(rdelm, rd_link) == rdelm); + assert(qr_prev(rdelm, rd_link) == rdelm); + } + qr_meld(purge_runs_sentinel, rdelm, rd_link); + } + + nstashed += npages; + if (opt_purge == purge_mode_ratio && arena->ndirty - nstashed <= + ndirty_limit) + break; + } + + return (nstashed); +} + +static size_t +arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, + arena_runs_dirty_link_t *purge_runs_sentinel, + extent_node_t *purge_chunks_sentinel) +{ + size_t npurged, nmadvise; + arena_runs_dirty_link_t *rdelm; + extent_node_t *chunkselm; + + if (config_stats) + nmadvise = 0; + npurged = 0; + + malloc_mutex_unlock(tsdn, &arena->lock); + for (rdelm = qr_next(purge_runs_sentinel, rd_link), + chunkselm = qr_next(purge_chunks_sentinel, cc_link); + rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) { + size_t npages; + + if (rdelm == &chunkselm->rd) { + /* + * Don't actually purge the chunk here because 1) + * chunkselm is embedded in the chunk and must remain + * valid, and 2) we deallocate the chunk in + * arena_unstash_purged(), where it is destroyed, + * decommitted, or purged, depending on chunk + * deallocation policy. + */ + size_t size = extent_node_size_get(chunkselm); + npages = size >> LG_PAGE; + chunkselm = qr_next(chunkselm, cc_link); + } else { + size_t pageind, run_size, flag_unzeroed, flags, i; + bool decommitted; + arena_chunk_t *chunk = + (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + pageind = arena_miscelm_to_pageind(miscelm); + run_size = arena_mapbits_large_size_get(chunk, pageind); + npages = run_size >> LG_PAGE; + + /* + * If this is the first run purged within chunk, mark + * the chunk as non-huge. This will prevent all use of + * transparent huge pages for this chunk until the chunk + * as a whole is deallocated. + */ + if (chunk->hugepage) { + pages_nohuge(chunk, chunksize); + chunk->hugepage = false; + } + + assert(pageind + npages <= chunk_npages); + assert(!arena_mapbits_decommitted_get(chunk, pageind)); + assert(!arena_mapbits_decommitted_get(chunk, + pageind+npages-1)); + decommitted = !chunk_hooks->decommit(chunk, chunksize, + pageind << LG_PAGE, npages << LG_PAGE, arena->ind); + if (decommitted) { + flag_unzeroed = 0; + flags = CHUNK_MAP_DECOMMITTED; + } else { + flag_unzeroed = chunk_purge_wrapper(tsdn, arena, + chunk_hooks, chunk, chunksize, pageind << + LG_PAGE, run_size) ? CHUNK_MAP_UNZEROED : 0; + flags = flag_unzeroed; + } + arena_mapbits_large_set(chunk, pageind+npages-1, 0, + flags); + arena_mapbits_large_set(chunk, pageind, run_size, + flags); + + /* + * Set the unzeroed flag for internal pages, now that + * chunk_purge_wrapper() has returned whether the pages + * were zeroed as a side effect of purging. This chunk + * map modification is safe even though the arena mutex + * isn't currently owned by this thread, because the run + * is marked as allocated, thus protecting it from being + * modified by any other thread. As long as these + * writes don't perturb the first and last elements' + * CHUNK_MAP_ALLOCATED bits, behavior is well defined. + */ + for (i = 1; i < npages-1; i++) { + arena_mapbits_internal_set(chunk, pageind+i, + flag_unzeroed); + } + } + + npurged += npages; + if (config_stats) + nmadvise++; + } + malloc_mutex_lock(tsdn, &arena->lock); + + if (config_stats) { + arena->stats.nmadvise += nmadvise; + arena->stats.purged += npurged; + } + + return (npurged); +} + +static void +arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, + arena_runs_dirty_link_t *purge_runs_sentinel, + extent_node_t *purge_chunks_sentinel) +{ + arena_runs_dirty_link_t *rdelm, *rdelm_next; + extent_node_t *chunkselm; + + /* Deallocate chunks/runs. */ + for (rdelm = qr_next(purge_runs_sentinel, rd_link), + chunkselm = qr_next(purge_chunks_sentinel, cc_link); + rdelm != purge_runs_sentinel; rdelm = rdelm_next) { + rdelm_next = qr_next(rdelm, rd_link); + if (rdelm == &chunkselm->rd) { + extent_node_t *chunkselm_next = qr_next(chunkselm, + cc_link); + void *addr = extent_node_addr_get(chunkselm); + size_t size = extent_node_size_get(chunkselm); + size_t sn = extent_node_sn_get(chunkselm); + bool zeroed = extent_node_zeroed_get(chunkselm); + bool committed = extent_node_committed_get(chunkselm); + extent_node_dirty_remove(chunkselm); + arena_node_dalloc(tsdn, arena, chunkselm); + chunkselm = chunkselm_next; + chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr, + size, sn, zeroed, committed); + } else { + arena_chunk_t *chunk = + (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); + bool decommitted = (arena_mapbits_decommitted_get(chunk, + pageind) != 0); + arena_run_t *run = &miscelm->run; + qr_remove(rdelm, rd_link); + arena_run_dalloc(tsdn, arena, run, false, true, + decommitted); + } + } +} + +/* + * NB: ndirty_limit is interpreted differently depending on opt_purge: + * - purge_mode_ratio: Purge as few dirty run/chunks as possible to reach the + * desired state: + * (arena->ndirty <= ndirty_limit) + * - purge_mode_decay: Purge as many dirty runs/chunks as possible without + * violating the invariant: + * (arena->ndirty >= ndirty_limit) + */ +static void +arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) +{ + chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); + size_t npurge, npurged; + arena_runs_dirty_link_t purge_runs_sentinel; + extent_node_t purge_chunks_sentinel; + + arena->purging = true; + + /* + * Calls to arena_dirty_count() are disabled even for debug builds + * because overhead grows nonlinearly as memory usage increases. + */ + if (false && config_debug) { + size_t ndirty = arena_dirty_count(arena); + assert(ndirty == arena->ndirty); + } + assert(opt_purge != purge_mode_ratio || (arena->nactive >> + arena->lg_dirty_mult) < arena->ndirty || ndirty_limit == 0); + + qr_new(&purge_runs_sentinel, rd_link); + extent_node_dirty_linkage_init(&purge_chunks_sentinel); + + npurge = arena_stash_dirty(tsdn, arena, &chunk_hooks, ndirty_limit, + &purge_runs_sentinel, &purge_chunks_sentinel); + if (npurge == 0) + goto label_return; + npurged = arena_purge_stashed(tsdn, arena, &chunk_hooks, + &purge_runs_sentinel, &purge_chunks_sentinel); + assert(npurged == npurge); + arena_unstash_purged(tsdn, arena, &chunk_hooks, &purge_runs_sentinel, + &purge_chunks_sentinel); + + if (config_stats) + arena->stats.npurge++; + +label_return: + arena->purging = false; +} + +void +arena_purge(tsdn_t *tsdn, arena_t *arena, bool all) +{ + + malloc_mutex_lock(tsdn, &arena->lock); + if (all) + arena_purge_to_limit(tsdn, arena, 0); + else + arena_maybe_purge(tsdn, arena); + malloc_mutex_unlock(tsdn, &arena->lock); +} + +static void +arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk) +{ + size_t pageind, npages; + + cassert(config_prof); + assert(opt_prof); + + /* + * Iterate over the allocated runs and remove profiled allocations from + * the sample set. + */ + for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { + if (arena_mapbits_allocated_get(chunk, pageind) != 0) { + if (arena_mapbits_large_get(chunk, pageind) != 0) { + void *ptr = (void *)((uintptr_t)chunk + (pageind + << LG_PAGE)); + size_t usize = isalloc(tsd_tsdn(tsd), ptr, + config_prof); + + prof_free(tsd, ptr, usize); + npages = arena_mapbits_large_size_get(chunk, + pageind) >> LG_PAGE; + } else { + /* Skip small run. */ + size_t binind = arena_mapbits_binind_get(chunk, + pageind); + arena_bin_info_t *bin_info = + &arena_bin_info[binind]; + npages = bin_info->run_size >> LG_PAGE; + } + } else { + /* Skip unallocated run. */ + npages = arena_mapbits_unallocated_size_get(chunk, + pageind) >> LG_PAGE; + } + assert(pageind + npages <= chunk_npages); + } +} + +void +arena_reset(tsd_t *tsd, arena_t *arena) +{ + unsigned i; + extent_node_t *node; + + /* + * Locking in this function is unintuitive. The caller guarantees that + * no concurrent operations are happening in this arena, but there are + * still reasons that some locking is necessary: + * + * - Some of the functions in the transitive closure of calls assume + * appropriate locks are held, and in some cases these locks are + * temporarily dropped to avoid lock order reversal or deadlock due to + * reentry. + * - mallctl("epoch", ...) may concurrently refresh stats. While + * strictly speaking this is a "concurrent operation", disallowing + * stats refreshes would impose an inconvenient burden. + */ + + /* Remove large allocations from prof sample set. */ + if (config_prof && opt_prof) { + ql_foreach(node, &arena->achunks, ql_link) { + arena_achunk_prof_reset(tsd, arena, + extent_node_addr_get(node)); + } + } + + /* Reset curruns for large size classes. */ + if (config_stats) { + for (i = 0; i < nlclasses; i++) + arena->stats.lstats[i].curruns = 0; + } + + /* Huge allocations. */ + malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); + for (node = ql_last(&arena->huge, ql_link); node != NULL; node = + ql_last(&arena->huge, ql_link)) { + void *ptr = extent_node_addr_get(node); + size_t usize; + + malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); + if (config_stats || (config_prof && opt_prof)) + usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); + /* Remove huge allocation from prof sample set. */ + if (config_prof && opt_prof) + prof_free(tsd, ptr, usize); + huge_dalloc(tsd_tsdn(tsd), ptr); + malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); + /* Cancel out unwanted effects on stats. */ + if (config_stats) + arena_huge_reset_stats_cancel(arena, usize); + } + malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); + + malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock); + + /* Bins. */ + for (i = 0; i < NBINS; i++) { + arena_bin_t *bin = &arena->bins[i]; + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + bin->runcur = NULL; + arena_run_heap_new(&bin->runs); + if (config_stats) { + bin->stats.curregs = 0; + bin->stats.curruns = 0; + } + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + } + + /* + * Re-initialize runs_dirty such that the chunks_cache and runs_dirty + * chains directly correspond. + */ + qr_new(&arena->runs_dirty, rd_link); + for (node = qr_next(&arena->chunks_cache, cc_link); + node != &arena->chunks_cache; node = qr_next(node, cc_link)) { + qr_new(&node->rd, rd_link); + qr_meld(&arena->runs_dirty, &node->rd, rd_link); + } + + /* Arena chunks. */ + for (node = ql_last(&arena->achunks, ql_link); node != NULL; node = + ql_last(&arena->achunks, ql_link)) { + ql_remove(&arena->achunks, node, ql_link); + arena_chunk_discard(tsd_tsdn(tsd), arena, + extent_node_addr_get(node)); + } + + /* Spare. */ + if (arena->spare != NULL) { + arena_chunk_discard(tsd_tsdn(tsd), arena, arena->spare); + arena->spare = NULL; + } + + assert(!arena->purging); + arena->nactive = 0; + + for (i = 0; i < NPSIZES; i++) + arena_run_heap_new(&arena->runs_avail[i]); + + malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock); +} + +static void +arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, + size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty, + size_t flag_decommitted) +{ + size_t size = *p_size; + size_t run_ind = *p_run_ind; + size_t run_pages = *p_run_pages; + + /* Try to coalesce forward. */ + if (run_ind + run_pages < chunk_npages && + arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && + arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && + arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == + flag_decommitted) { + size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, + run_ind+run_pages); + size_t nrun_pages = nrun_size >> LG_PAGE; + + /* + * Remove successor from runs_avail; the coalesced run is + * inserted later. + */ + assert(arena_mapbits_unallocated_size_get(chunk, + run_ind+run_pages+nrun_pages-1) == nrun_size); + assert(arena_mapbits_dirty_get(chunk, + run_ind+run_pages+nrun_pages-1) == flag_dirty); + assert(arena_mapbits_decommitted_get(chunk, + run_ind+run_pages+nrun_pages-1) == flag_decommitted); + arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); + + /* + * If the successor is dirty, remove it from the set of dirty + * pages. + */ + if (flag_dirty != 0) { + arena_run_dirty_remove(arena, chunk, run_ind+run_pages, + nrun_pages); + } + + size += nrun_size; + run_pages += nrun_pages; + + arena_mapbits_unallocated_size_set(chunk, run_ind, size); + arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, + size); + } + + /* Try to coalesce backward. */ + if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, + run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == + flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == + flag_decommitted) { + size_t prun_size = arena_mapbits_unallocated_size_get(chunk, + run_ind-1); + size_t prun_pages = prun_size >> LG_PAGE; + + run_ind -= prun_pages; + + /* + * Remove predecessor from runs_avail; the coalesced run is + * inserted later. + */ + assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == + prun_size); + assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); + assert(arena_mapbits_decommitted_get(chunk, run_ind) == + flag_decommitted); + arena_avail_remove(arena, chunk, run_ind, prun_pages); + + /* + * If the predecessor is dirty, remove it from the set of dirty + * pages. + */ + if (flag_dirty != 0) { + arena_run_dirty_remove(arena, chunk, run_ind, + prun_pages); + } + + size += prun_size; + run_pages += prun_pages; + + arena_mapbits_unallocated_size_set(chunk, run_ind, size); + arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, + size); + } + + *p_size = size; + *p_run_ind = run_ind; + *p_run_pages = run_pages; +} + +static size_t +arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, + size_t run_ind) +{ + size_t size; + + assert(run_ind >= map_bias); + assert(run_ind < chunk_npages); + + if (arena_mapbits_large_get(chunk, run_ind) != 0) { + size = arena_mapbits_large_size_get(chunk, run_ind); + assert(size == PAGE || arena_mapbits_large_size_get(chunk, + run_ind+(size>>LG_PAGE)-1) == 0); + } else { + arena_bin_info_t *bin_info = &arena_bin_info[run->binind]; + size = bin_info->run_size; + } + + return (size); +} + +static void +arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty, + bool cleaned, bool decommitted) +{ + arena_chunk_t *chunk; + arena_chunk_map_misc_t *miscelm; + size_t size, run_ind, run_pages, flag_dirty, flag_decommitted; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + miscelm = arena_run_to_miscelm(run); + run_ind = arena_miscelm_to_pageind(miscelm); + assert(run_ind >= map_bias); + assert(run_ind < chunk_npages); + size = arena_run_size_get(arena, chunk, run, run_ind); + run_pages = (size >> LG_PAGE); + arena_nactive_sub(arena, run_pages); + + /* + * The run is dirty if the caller claims to have dirtied it, as well as + * if it was already dirty before being allocated and the caller + * doesn't claim to have cleaned it. + */ + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); + if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) + != 0) + dirty = true; + flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; + flag_decommitted = decommitted ? CHUNK_MAP_DECOMMITTED : 0; + + /* Mark pages as unallocated in the chunk map. */ + if (dirty || decommitted) { + size_t flags = flag_dirty | flag_decommitted; + arena_mapbits_unallocated_set(chunk, run_ind, size, flags); + arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, + flags); + } else { + arena_mapbits_unallocated_set(chunk, run_ind, size, + arena_mapbits_unzeroed_get(chunk, run_ind)); + arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, + arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); + } + + arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, + flag_dirty, flag_decommitted); + + /* Insert into runs_avail, now that coalescing is complete. */ + assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == + arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); + assert(arena_mapbits_decommitted_get(chunk, run_ind) == + arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); + arena_avail_insert(arena, chunk, run_ind, run_pages); + + if (dirty) + arena_run_dirty_insert(arena, chunk, run_ind, run_pages); + + /* Deallocate chunk if it is now completely unused. */ + if (size == arena_maxrun) { + assert(run_ind == map_bias); + assert(run_pages == (arena_maxrun >> LG_PAGE)); + arena_chunk_dalloc(tsdn, arena, chunk); + } + + /* + * It is okay to do dirty page processing here even if the chunk was + * deallocated above, since in that case it is the spare. Waiting + * until after possible chunk deallocation to do dirty processing + * allows for an old spare to be fully deallocated, thus decreasing the + * chances of spuriously crossing the dirty page purging threshold. + */ + if (dirty) + arena_maybe_purge(tsdn, arena); +} + +static void +arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + arena_run_t *run, size_t oldsize, size_t newsize) +{ + arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); + size_t pageind = arena_miscelm_to_pageind(miscelm); + size_t head_npages = (oldsize - newsize) >> LG_PAGE; + size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); + size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); + size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? + CHUNK_MAP_UNZEROED : 0; + + assert(oldsize > newsize); + + /* + * Update the chunk map so that arena_run_dalloc() can treat the + * leading run as separately allocated. Set the last element of each + * run first, in case of single-page runs. + */ + assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); + arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | + (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, + pageind+head_npages-1))); + arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty | + (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); + + if (config_debug) { + UNUSED size_t tail_npages = newsize >> LG_PAGE; + assert(arena_mapbits_large_size_get(chunk, + pageind+head_npages+tail_npages-1) == 0); + assert(arena_mapbits_dirty_get(chunk, + pageind+head_npages+tail_npages-1) == flag_dirty); + } + arena_mapbits_large_set(chunk, pageind+head_npages, newsize, + flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, + pageind+head_npages))); + + arena_run_dalloc(tsdn, arena, run, false, false, (flag_decommitted != + 0)); +} + +static void +arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + arena_run_t *run, size_t oldsize, size_t newsize, bool dirty) +{ + arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); + size_t pageind = arena_miscelm_to_pageind(miscelm); + size_t head_npages = newsize >> LG_PAGE; + size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); + size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); + size_t flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? + CHUNK_MAP_UNZEROED : 0; + arena_chunk_map_misc_t *tail_miscelm; + arena_run_t *tail_run; + + assert(oldsize > newsize); + + /* + * Update the chunk map so that arena_run_dalloc() can treat the + * trailing run as separately allocated. Set the last element of each + * run first, in case of single-page runs. + */ + assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); + arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | + (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, + pageind+head_npages-1))); + arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty | + (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); + + if (config_debug) { + UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; + assert(arena_mapbits_large_size_get(chunk, + pageind+head_npages+tail_npages-1) == 0); + assert(arena_mapbits_dirty_get(chunk, + pageind+head_npages+tail_npages-1) == flag_dirty); + } + arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, + flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, + pageind+head_npages))); + + tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages); + tail_run = &tail_miscelm->run; + arena_run_dalloc(tsdn, arena, tail_run, dirty, false, (flag_decommitted + != 0)); +} + +static void +arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) +{ + arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); + + arena_run_heap_insert(&bin->runs, miscelm); +} + +static arena_run_t * +arena_bin_nonfull_run_tryget(arena_bin_t *bin) +{ + arena_chunk_map_misc_t *miscelm; + + miscelm = arena_run_heap_remove_first(&bin->runs); + if (miscelm == NULL) + return (NULL); + if (config_stats) + bin->stats.reruns++; + + return (&miscelm->run); +} + +static arena_run_t * +arena_bin_nonfull_run_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) +{ + arena_run_t *run; + szind_t binind; + arena_bin_info_t *bin_info; + + /* Look for a usable run. */ + run = arena_bin_nonfull_run_tryget(bin); + if (run != NULL) + return (run); + /* No existing runs have any space available. */ + + binind = arena_bin_index(arena, bin); + bin_info = &arena_bin_info[binind]; + + /* Allocate a new run. */ + malloc_mutex_unlock(tsdn, &bin->lock); + /******************************/ + malloc_mutex_lock(tsdn, &arena->lock); + run = arena_run_alloc_small(tsdn, arena, bin_info->run_size, binind); + if (run != NULL) { + /* Initialize run internals. */ + run->binind = binind; + run->nfree = bin_info->nregs; + bitmap_init(run->bitmap, &bin_info->bitmap_info); + } + malloc_mutex_unlock(tsdn, &arena->lock); + /********************************/ + malloc_mutex_lock(tsdn, &bin->lock); + if (run != NULL) { + if (config_stats) { + bin->stats.nruns++; + bin->stats.curruns++; + } + return (run); + } + + /* + * arena_run_alloc_small() failed, but another thread may have made + * sufficient memory available while this one dropped bin->lock above, + * so search one more time. + */ + run = arena_bin_nonfull_run_tryget(bin); + if (run != NULL) + return (run); + + return (NULL); +} + +/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ +static void * +arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) +{ + szind_t binind; + arena_bin_info_t *bin_info; + arena_run_t *run; + + binind = arena_bin_index(arena, bin); + bin_info = &arena_bin_info[binind]; + bin->runcur = NULL; + run = arena_bin_nonfull_run_get(tsdn, arena, bin); + if (bin->runcur != NULL && bin->runcur->nfree > 0) { + /* + * Another thread updated runcur while this one ran without the + * bin lock in arena_bin_nonfull_run_get(). + */ + void *ret; + assert(bin->runcur->nfree > 0); + ret = arena_run_reg_alloc(bin->runcur, bin_info); + if (run != NULL) { + arena_chunk_t *chunk; + + /* + * arena_run_alloc_small() may have allocated run, or + * it may have pulled run from the bin's run tree. + * Therefore it is unsafe to make any assumptions about + * how run has previously been used, and + * arena_bin_lower_run() must be called, as if a region + * were just deallocated from the run. + */ + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + if (run->nfree == bin_info->nregs) { + arena_dalloc_bin_run(tsdn, arena, chunk, run, + bin); + } else + arena_bin_lower_run(arena, run, bin); + } + return (ret); + } + + if (run == NULL) + return (NULL); + + bin->runcur = run; + + assert(bin->runcur->nfree > 0); + + return (arena_run_reg_alloc(bin->runcur, bin_info)); +} + +void +arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_bin_t *tbin, + szind_t binind, uint64_t prof_accumbytes) +{ + unsigned i, nfill; + arena_bin_t *bin; + + assert(tbin->ncached == 0); + + if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) + prof_idump(tsdn); + bin = &arena->bins[binind]; + malloc_mutex_lock(tsdn, &bin->lock); + for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> + tbin->lg_fill_div); i < nfill; i++) { + arena_run_t *run; + void *ptr; + if ((run = bin->runcur) != NULL && run->nfree > 0) + ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); + else + ptr = arena_bin_malloc_hard(tsdn, arena, bin); + if (ptr == NULL) { + /* + * OOM. tbin->avail isn't yet filled down to its first + * element, so the successful allocations (if any) must + * be moved just before tbin->avail before bailing out. + */ + if (i > 0) { + memmove(tbin->avail - i, tbin->avail - nfill, + i * sizeof(void *)); + } + break; + } + if (config_fill && unlikely(opt_junk_alloc)) { + arena_alloc_junk_small(ptr, &arena_bin_info[binind], + true); + } + /* Insert such that low regions get used first. */ + *(tbin->avail - nfill + i) = ptr; + } + if (config_stats) { + bin->stats.nmalloc += i; + bin->stats.nrequests += tbin->tstats.nrequests; + bin->stats.curregs += i; + bin->stats.nfills++; + tbin->tstats.nrequests = 0; + } + malloc_mutex_unlock(tsdn, &bin->lock); + tbin->ncached = i; + arena_decay_tick(tsdn, arena); +} + +void +arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) +{ + + size_t redzone_size = bin_info->redzone_size; + + if (zero) { + memset((void *)((uintptr_t)ptr - redzone_size), + JEMALLOC_ALLOC_JUNK, redzone_size); + memset((void *)((uintptr_t)ptr + bin_info->reg_size), + JEMALLOC_ALLOC_JUNK, redzone_size); + } else { + memset((void *)((uintptr_t)ptr - redzone_size), + JEMALLOC_ALLOC_JUNK, bin_info->reg_interval); + } +} + +#ifdef JEMALLOC_JET +#undef arena_redzone_corruption +#define arena_redzone_corruption JEMALLOC_N(n_arena_redzone_corruption) +#endif +static void +arena_redzone_corruption(void *ptr, size_t usize, bool after, + size_t offset, uint8_t byte) +{ + + malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p " + "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", + after ? "after" : "before", ptr, usize, byte); +} +#ifdef JEMALLOC_JET +#undef arena_redzone_corruption +#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) +arena_redzone_corruption_t *arena_redzone_corruption = + JEMALLOC_N(n_arena_redzone_corruption); +#endif + +static void +arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) +{ + bool error = false; + + if (opt_junk_alloc) { + size_t size = bin_info->reg_size; + size_t redzone_size = bin_info->redzone_size; + size_t i; + + for (i = 1; i <= redzone_size; i++) { + uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); + if (*byte != JEMALLOC_ALLOC_JUNK) { + error = true; + arena_redzone_corruption(ptr, size, false, i, + *byte); + if (reset) + *byte = JEMALLOC_ALLOC_JUNK; + } + } + for (i = 0; i < redzone_size; i++) { + uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); + if (*byte != JEMALLOC_ALLOC_JUNK) { + error = true; + arena_redzone_corruption(ptr, size, true, i, + *byte); + if (reset) + *byte = JEMALLOC_ALLOC_JUNK; + } + } + } + + if (opt_abort && error) + abort(); +} + +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_small +#define arena_dalloc_junk_small JEMALLOC_N(n_arena_dalloc_junk_small) +#endif +void +arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) +{ + size_t redzone_size = bin_info->redzone_size; + + arena_redzones_validate(ptr, bin_info, false); + memset((void *)((uintptr_t)ptr - redzone_size), JEMALLOC_FREE_JUNK, + bin_info->reg_interval); +} +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_small +#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) +arena_dalloc_junk_small_t *arena_dalloc_junk_small = + JEMALLOC_N(n_arena_dalloc_junk_small); +#endif + +void +arena_quarantine_junk_small(void *ptr, size_t usize) +{ + szind_t binind; + arena_bin_info_t *bin_info; + cassert(config_fill); + assert(opt_junk_free); + assert(opt_quarantine); + assert(usize <= SMALL_MAXCLASS); + + binind = size2index(usize); + bin_info = &arena_bin_info[binind]; + arena_redzones_validate(ptr, bin_info, true); +} + +static void * +arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) +{ + void *ret; + arena_bin_t *bin; + size_t usize; + arena_run_t *run; + + assert(binind < NBINS); + bin = &arena->bins[binind]; + usize = index2size(binind); + + malloc_mutex_lock(tsdn, &bin->lock); + if ((run = bin->runcur) != NULL && run->nfree > 0) + ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); + else + ret = arena_bin_malloc_hard(tsdn, arena, bin); + + if (ret == NULL) { + malloc_mutex_unlock(tsdn, &bin->lock); + return (NULL); + } + + if (config_stats) { + bin->stats.nmalloc++; + bin->stats.nrequests++; + bin->stats.curregs++; + } + malloc_mutex_unlock(tsdn, &bin->lock); + if (config_prof && !isthreaded && arena_prof_accum(tsdn, arena, usize)) + prof_idump(tsdn); + + if (!zero) { + if (config_fill) { + if (unlikely(opt_junk_alloc)) { + arena_alloc_junk_small(ret, + &arena_bin_info[binind], false); + } else if (unlikely(opt_zero)) + memset(ret, 0, usize); + } + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, usize); + } else { + if (config_fill && unlikely(opt_junk_alloc)) { + arena_alloc_junk_small(ret, &arena_bin_info[binind], + true); + } + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, usize); + memset(ret, 0, usize); + } + + arena_decay_tick(tsdn, arena); + return (ret); +} + +void * +arena_malloc_large(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) +{ + void *ret; + size_t usize; + uintptr_t random_offset; + arena_run_t *run; + arena_chunk_map_misc_t *miscelm; + UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false); + + /* Large allocation. */ + usize = index2size(binind); + malloc_mutex_lock(tsdn, &arena->lock); + if (config_cache_oblivious) { + uint64_t r; + + /* + * Compute a uniformly distributed offset within the first page + * that is a multiple of the cacheline size, e.g. [0 .. 63) * 64 + * for 4 KiB pages and 64-byte cachelines. + */ + r = prng_lg_range_zu(&arena->offset_state, LG_PAGE - + LG_CACHELINE, false); + random_offset = ((uintptr_t)r) << LG_CACHELINE; + } else + random_offset = 0; + run = arena_run_alloc_large(tsdn, arena, usize + large_pad, zero); + if (run == NULL) { + malloc_mutex_unlock(tsdn, &arena->lock); + return (NULL); + } + miscelm = arena_run_to_miscelm(run); + ret = (void *)((uintptr_t)arena_miscelm_to_rpages(miscelm) + + random_offset); + if (config_stats) { + szind_t index = binind - NBINS; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += usize; + arena->stats.lstats[index].nmalloc++; + arena->stats.lstats[index].nrequests++; + arena->stats.lstats[index].curruns++; + } + if (config_prof) + idump = arena_prof_accum_locked(arena, usize); + malloc_mutex_unlock(tsdn, &arena->lock); + if (config_prof && idump) + prof_idump(tsdn); + + if (!zero) { + if (config_fill) { + if (unlikely(opt_junk_alloc)) + memset(ret, JEMALLOC_ALLOC_JUNK, usize); + else if (unlikely(opt_zero)) + memset(ret, 0, usize); + } + } + + arena_decay_tick(tsdn, arena); + return (ret); +} + +void * +arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, + bool zero) +{ + + assert(!tsdn_null(tsdn) || arena != NULL); + + if (likely(!tsdn_null(tsdn))) + arena = arena_choose(tsdn_tsd(tsdn), arena); + if (unlikely(arena == NULL)) + return (NULL); + + if (likely(size <= SMALL_MAXCLASS)) + return (arena_malloc_small(tsdn, arena, ind, zero)); + if (likely(size <= large_maxclass)) + return (arena_malloc_large(tsdn, arena, ind, zero)); + return (huge_malloc(tsdn, arena, index2size(ind), zero)); +} + +/* Only handles large allocations that require more than page alignment. */ +static void * +arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, + bool zero) +{ + void *ret; + size_t alloc_size, leadsize, trailsize; + arena_run_t *run; + arena_chunk_t *chunk; + arena_chunk_map_misc_t *miscelm; + void *rpages; + + assert(!tsdn_null(tsdn) || arena != NULL); + assert(usize == PAGE_CEILING(usize)); + + if (likely(!tsdn_null(tsdn))) + arena = arena_choose(tsdn_tsd(tsdn), arena); + if (unlikely(arena == NULL)) + return (NULL); + + alignment = PAGE_CEILING(alignment); + alloc_size = usize + large_pad + alignment - PAGE; + + malloc_mutex_lock(tsdn, &arena->lock); + run = arena_run_alloc_large(tsdn, arena, alloc_size, false); + if (run == NULL) { + malloc_mutex_unlock(tsdn, &arena->lock); + return (NULL); + } + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + miscelm = arena_run_to_miscelm(run); + rpages = arena_miscelm_to_rpages(miscelm); + + leadsize = ALIGNMENT_CEILING((uintptr_t)rpages, alignment) - + (uintptr_t)rpages; + assert(alloc_size >= leadsize + usize); + trailsize = alloc_size - leadsize - usize - large_pad; + if (leadsize != 0) { + arena_chunk_map_misc_t *head_miscelm = miscelm; + arena_run_t *head_run = run; + + miscelm = arena_miscelm_get_mutable(chunk, + arena_miscelm_to_pageind(head_miscelm) + (leadsize >> + LG_PAGE)); + run = &miscelm->run; + + arena_run_trim_head(tsdn, arena, chunk, head_run, alloc_size, + alloc_size - leadsize); + } + if (trailsize != 0) { + arena_run_trim_tail(tsdn, arena, chunk, run, usize + large_pad + + trailsize, usize + large_pad, false); + } + if (arena_run_init_large(arena, run, usize + large_pad, zero)) { + size_t run_ind = + arena_miscelm_to_pageind(arena_run_to_miscelm(run)); + bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); + bool decommitted = (arena_mapbits_decommitted_get(chunk, + run_ind) != 0); + + assert(decommitted); /* Cause of OOM. */ + arena_run_dalloc(tsdn, arena, run, dirty, false, decommitted); + malloc_mutex_unlock(tsdn, &arena->lock); + return (NULL); + } + ret = arena_miscelm_to_rpages(miscelm); + + if (config_stats) { + szind_t index = size2index(usize) - NBINS; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += usize; + arena->stats.lstats[index].nmalloc++; + arena->stats.lstats[index].nrequests++; + arena->stats.lstats[index].curruns++; + } + malloc_mutex_unlock(tsdn, &arena->lock); + + if (config_fill && !zero) { + if (unlikely(opt_junk_alloc)) + memset(ret, JEMALLOC_ALLOC_JUNK, usize); + else if (unlikely(opt_zero)) + memset(ret, 0, usize); + } + arena_decay_tick(tsdn, arena); + return (ret); +} + +void * +arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, + bool zero, tcache_t *tcache) +{ + void *ret; + + if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE + && (usize & PAGE_MASK) == 0))) { + /* Small; alignment doesn't require special run placement. */ + ret = arena_malloc(tsdn, arena, usize, size2index(usize), zero, + tcache, true); + } else if (usize <= large_maxclass && alignment <= PAGE) { + /* + * Large; alignment doesn't require special run placement. + * However, the cached pointer may be at a random offset from + * the base of the run, so do some bit manipulation to retrieve + * the base. + */ + ret = arena_malloc(tsdn, arena, usize, size2index(usize), zero, + tcache, true); + if (config_cache_oblivious) + ret = (void *)((uintptr_t)ret & ~PAGE_MASK); + } else { + if (likely(usize <= large_maxclass)) { + ret = arena_palloc_large(tsdn, arena, usize, alignment, + zero); + } else if (likely(alignment <= chunksize)) + ret = huge_malloc(tsdn, arena, usize, zero); + else { + ret = huge_palloc(tsdn, arena, usize, alignment, zero); + } + } + return (ret); +} + +void +arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size) +{ + arena_chunk_t *chunk; + size_t pageind; + szind_t binind; + + cassert(config_prof); + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS); + assert(isalloc(tsdn, ptr, true) == LARGE_MINCLASS); + assert(size <= SMALL_MAXCLASS); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + binind = size2index(size); + assert(binind < NBINS); + arena_mapbits_large_binind_set(chunk, pageind, binind); + + assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS); + assert(isalloc(tsdn, ptr, true) == size); +} + +static void +arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, + arena_bin_t *bin) +{ + + /* Dissociate run from bin. */ + if (run == bin->runcur) + bin->runcur = NULL; + else { + szind_t binind = arena_bin_index(extent_node_arena_get( + &chunk->node), bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + + /* + * The following block's conditional is necessary because if the + * run only contains one region, then it never gets inserted + * into the non-full runs tree. + */ + if (bin_info->nregs != 1) { + arena_chunk_map_misc_t *miscelm = + arena_run_to_miscelm(run); + + arena_run_heap_remove(&bin->runs, miscelm); + } + } +} + +static void +arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + arena_run_t *run, arena_bin_t *bin) +{ + + assert(run != bin->runcur); + + malloc_mutex_unlock(tsdn, &bin->lock); + /******************************/ + malloc_mutex_lock(tsdn, &arena->lock); + arena_run_dalloc(tsdn, arena, run, true, false, false); + malloc_mutex_unlock(tsdn, &arena->lock); + /****************************/ + malloc_mutex_lock(tsdn, &bin->lock); + if (config_stats) + bin->stats.curruns--; +} + +static void +arena_bin_lower_run(arena_t *arena, arena_run_t *run, arena_bin_t *bin) +{ + + /* + * Make sure that if bin->runcur is non-NULL, it refers to the + * oldest/lowest non-full run. It is okay to NULL runcur out rather + * than proactively keeping it pointing at the oldest/lowest non-full + * run. + */ + if (bin->runcur != NULL && + arena_snad_comp(arena_run_to_miscelm(bin->runcur), + arena_run_to_miscelm(run)) > 0) { + /* Switch runcur. */ + if (bin->runcur->nfree > 0) + arena_bin_runs_insert(bin, bin->runcur); + bin->runcur = run; + if (config_stats) + bin->stats.reruns++; + } else + arena_bin_runs_insert(bin, run); +} + +static void +arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + void *ptr, arena_chunk_map_bits_t *bitselm, bool junked) +{ + size_t pageind, rpages_ind; + arena_run_t *run; + arena_bin_t *bin; + arena_bin_info_t *bin_info; + szind_t binind; + + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); + run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; + binind = run->binind; + bin = &arena->bins[binind]; + bin_info = &arena_bin_info[binind]; + + if (!junked && config_fill && unlikely(opt_junk_free)) + arena_dalloc_junk_small(ptr, bin_info); + + arena_run_reg_dalloc(run, ptr); + if (run->nfree == bin_info->nregs) { + arena_dissociate_bin_run(chunk, run, bin); + arena_dalloc_bin_run(tsdn, arena, chunk, run, bin); + } else if (run->nfree == 1 && run != bin->runcur) + arena_bin_lower_run(arena, run, bin); + + if (config_stats) { + bin->stats.ndalloc++; + bin->stats.curregs--; + } +} + +void +arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, + arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm) +{ + + arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, true); +} + +void +arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind, arena_chunk_map_bits_t *bitselm) +{ + arena_run_t *run; + arena_bin_t *bin; + size_t rpages_ind; + + rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); + run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; + bin = &arena->bins[run->binind]; + malloc_mutex_lock(tsdn, &bin->lock); + arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, false); + malloc_mutex_unlock(tsdn, &bin->lock); +} + +void +arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + void *ptr, size_t pageind) +{ + arena_chunk_map_bits_t *bitselm; + + if (config_debug) { + /* arena_ptr_small_binind_get() does extra sanity checking. */ + assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, + pageind)) != BININD_INVALID); + } + bitselm = arena_bitselm_get_mutable(chunk, pageind); + arena_dalloc_bin(tsdn, arena, chunk, ptr, pageind, bitselm); + arena_decay_tick(tsdn, arena); +} + +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_large +#define arena_dalloc_junk_large JEMALLOC_N(n_arena_dalloc_junk_large) +#endif +void +arena_dalloc_junk_large(void *ptr, size_t usize) +{ + + if (config_fill && unlikely(opt_junk_free)) + memset(ptr, JEMALLOC_FREE_JUNK, usize); +} +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_large +#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) +arena_dalloc_junk_large_t *arena_dalloc_junk_large = + JEMALLOC_N(n_arena_dalloc_junk_large); +#endif + +static void +arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena, + arena_chunk_t *chunk, void *ptr, bool junked) +{ + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, + pageind); + arena_run_t *run = &miscelm->run; + + if (config_fill || config_stats) { + size_t usize = arena_mapbits_large_size_get(chunk, pageind) - + large_pad; + + if (!junked) + arena_dalloc_junk_large(ptr, usize); + if (config_stats) { + szind_t index = size2index(usize) - NBINS; + + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= usize; + arena->stats.lstats[index].ndalloc++; + arena->stats.lstats[index].curruns--; + } + } + + arena_run_dalloc(tsdn, arena, run, true, false, false); +} + +void +arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena, + arena_chunk_t *chunk, void *ptr) +{ + + arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, true); +} + +void +arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + void *ptr) +{ + + malloc_mutex_lock(tsdn, &arena->lock); + arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, false); + malloc_mutex_unlock(tsdn, &arena->lock); + arena_decay_tick(tsdn, arena); +} + +static void +arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + void *ptr, size_t oldsize, size_t size) +{ + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, + pageind); + arena_run_t *run = &miscelm->run; + + assert(size < oldsize); + + /* + * Shrink the run, and make trailing pages available for other + * allocations. + */ + malloc_mutex_lock(tsdn, &arena->lock); + arena_run_trim_tail(tsdn, arena, chunk, run, oldsize + large_pad, size + + large_pad, true); + if (config_stats) { + szind_t oldindex = size2index(oldsize) - NBINS; + szind_t index = size2index(size) - NBINS; + + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= oldsize; + arena->stats.lstats[oldindex].ndalloc++; + arena->stats.lstats[oldindex].curruns--; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[index].nmalloc++; + arena->stats.lstats[index].nrequests++; + arena->stats.lstats[index].curruns++; + } + malloc_mutex_unlock(tsdn, &arena->lock); +} + +static bool +arena_ralloc_large_grow(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, + void *ptr, size_t oldsize, size_t usize_min, size_t usize_max, bool zero) +{ + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t npages = (oldsize + large_pad) >> LG_PAGE; + size_t followsize; + + assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) - + large_pad); + + /* Try to extend the run. */ + malloc_mutex_lock(tsdn, &arena->lock); + if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk, + pageind+npages) != 0) + goto label_fail; + followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages); + if (oldsize + followsize >= usize_min) { + /* + * The next run is available and sufficiently large. Split the + * following run, then merge the first part with the existing + * allocation. + */ + arena_run_t *run; + size_t usize, splitsize, size, flag_dirty, flag_unzeroed_mask; + + usize = usize_max; + while (oldsize + followsize < usize) + usize = index2size(size2index(usize)-1); + assert(usize >= usize_min); + assert(usize >= oldsize); + splitsize = usize - oldsize; + if (splitsize == 0) + goto label_fail; + + run = &arena_miscelm_get_mutable(chunk, pageind+npages)->run; + if (arena_run_split_large(arena, run, splitsize, zero)) + goto label_fail; + + if (config_cache_oblivious && zero) { + /* + * Zero the trailing bytes of the original allocation's + * last page, since they are in an indeterminate state. + * There will always be trailing bytes, because ptr's + * offset from the beginning of the run is a multiple of + * CACHELINE in [0 .. PAGE). + */ + void *zbase = (void *)((uintptr_t)ptr + oldsize); + void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase + + PAGE)); + size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase; + assert(nzero > 0); + memset(zbase, 0, nzero); + } + + size = oldsize + splitsize; + npages = (size + large_pad) >> LG_PAGE; + + /* + * Mark the extended run as dirty if either portion of the run + * was dirty before allocation. This is rather pedantic, + * because there's not actually any sequence of events that + * could cause the resulting run to be passed to + * arena_run_dalloc() with the dirty argument set to false + * (which is when dirty flag consistency would really matter). + */ + flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | + arena_mapbits_dirty_get(chunk, pageind+npages-1); + flag_unzeroed_mask = flag_dirty == 0 ? CHUNK_MAP_UNZEROED : 0; + arena_mapbits_large_set(chunk, pageind, size + large_pad, + flag_dirty | (flag_unzeroed_mask & + arena_mapbits_unzeroed_get(chunk, pageind))); + arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty | + (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, + pageind+npages-1))); + + if (config_stats) { + szind_t oldindex = size2index(oldsize) - NBINS; + szind_t index = size2index(size) - NBINS; + + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= oldsize; + arena->stats.lstats[oldindex].ndalloc++; + arena->stats.lstats[oldindex].curruns--; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[index].nmalloc++; + arena->stats.lstats[index].nrequests++; + arena->stats.lstats[index].curruns++; + } + malloc_mutex_unlock(tsdn, &arena->lock); + return (false); + } +label_fail: + malloc_mutex_unlock(tsdn, &arena->lock); + return (true); +} + +#ifdef JEMALLOC_JET +#undef arena_ralloc_junk_large +#define arena_ralloc_junk_large JEMALLOC_N(n_arena_ralloc_junk_large) +#endif +static void +arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) +{ + + if (config_fill && unlikely(opt_junk_free)) { + memset((void *)((uintptr_t)ptr + usize), JEMALLOC_FREE_JUNK, + old_usize - usize); + } +} +#ifdef JEMALLOC_JET +#undef arena_ralloc_junk_large +#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) +arena_ralloc_junk_large_t *arena_ralloc_junk_large = + JEMALLOC_N(n_arena_ralloc_junk_large); +#endif + +/* + * Try to resize a large allocation, in order to avoid copying. This will + * always fail if growing an object, and the following run is already in use. + */ +static bool +arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min, + size_t usize_max, bool zero) +{ + arena_chunk_t *chunk; + arena_t *arena; + + if (oldsize == usize_max) { + /* Current size class is compatible and maximal. */ + return (false); + } + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + arena = extent_node_arena_get(&chunk->node); + + if (oldsize < usize_max) { + bool ret = arena_ralloc_large_grow(tsdn, arena, chunk, ptr, + oldsize, usize_min, usize_max, zero); + if (config_fill && !ret && !zero) { + if (unlikely(opt_junk_alloc)) { + memset((void *)((uintptr_t)ptr + oldsize), + JEMALLOC_ALLOC_JUNK, + isalloc(tsdn, ptr, config_prof) - oldsize); + } else if (unlikely(opt_zero)) { + memset((void *)((uintptr_t)ptr + oldsize), 0, + isalloc(tsdn, ptr, config_prof) - oldsize); + } + } + return (ret); + } + + assert(oldsize > usize_max); + /* Fill before shrinking in order avoid a race. */ + arena_ralloc_junk_large(ptr, oldsize, usize_max); + arena_ralloc_large_shrink(tsdn, arena, chunk, ptr, oldsize, usize_max); + return (false); +} + +bool +arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, + size_t extra, bool zero) +{ + size_t usize_min, usize_max; + + /* Calls with non-zero extra had to clamp extra. */ + assert(extra == 0 || size + extra <= HUGE_MAXCLASS); + + if (unlikely(size > HUGE_MAXCLASS)) + return (true); + + usize_min = s2u(size); + usize_max = s2u(size + extra); + if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) { + arena_chunk_t *chunk; + + /* + * Avoid moving the allocation if the size class can be left the + * same. + */ + if (oldsize <= SMALL_MAXCLASS) { + assert(arena_bin_info[size2index(oldsize)].reg_size == + oldsize); + if ((usize_max > SMALL_MAXCLASS || + size2index(usize_max) != size2index(oldsize)) && + (size > oldsize || usize_max < oldsize)) + return (true); + } else { + if (usize_max <= SMALL_MAXCLASS) + return (true); + if (arena_ralloc_large(tsdn, ptr, oldsize, usize_min, + usize_max, zero)) + return (true); + } + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + arena_decay_tick(tsdn, extent_node_arena_get(&chunk->node)); + return (false); + } else { + return (huge_ralloc_no_move(tsdn, ptr, oldsize, usize_min, + usize_max, zero)); + } +} + +static void * +arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, + size_t alignment, bool zero, tcache_t *tcache) +{ + + if (alignment == 0) + return (arena_malloc(tsdn, arena, usize, size2index(usize), + zero, tcache, true)); + usize = sa2u(usize, alignment); + if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) + return (NULL); + return (ipalloct(tsdn, usize, alignment, zero, tcache, arena)); +} + +void * +arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, + size_t alignment, bool zero, tcache_t *tcache) +{ + void *ret; + size_t usize; + + usize = s2u(size); + if (unlikely(usize == 0 || size > HUGE_MAXCLASS)) + return (NULL); + + if (likely(usize <= large_maxclass)) { + size_t copysize; + + /* Try to avoid moving the allocation. */ + if (!arena_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, 0, + zero)) + return (ptr); + + /* + * size and oldsize are different enough that we need to move + * the object. In that case, fall back to allocating new space + * and copying. + */ + ret = arena_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, + alignment, zero, tcache); + if (ret == NULL) + return (NULL); + + /* + * Junk/zero-filling were already done by + * ipalloc()/arena_malloc(). + */ + + copysize = (usize < oldsize) ? usize : oldsize; + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); + memcpy(ret, ptr, copysize); + isqalloc(tsd, ptr, oldsize, tcache, true); + } else { + ret = huge_ralloc(tsd, arena, ptr, oldsize, usize, alignment, + zero, tcache); + } + return (ret); +} + +dss_prec_t +arena_dss_prec_get(tsdn_t *tsdn, arena_t *arena) +{ + dss_prec_t ret; + + malloc_mutex_lock(tsdn, &arena->lock); + ret = arena->dss_prec; + malloc_mutex_unlock(tsdn, &arena->lock); + return (ret); +} + +bool +arena_dss_prec_set(tsdn_t *tsdn, arena_t *arena, dss_prec_t dss_prec) +{ + + if (!have_dss) + return (dss_prec != dss_prec_disabled); + malloc_mutex_lock(tsdn, &arena->lock); + arena->dss_prec = dss_prec; + malloc_mutex_unlock(tsdn, &arena->lock); + return (false); +} + +ssize_t +arena_lg_dirty_mult_default_get(void) +{ + + return ((ssize_t)atomic_read_z((size_t *)&lg_dirty_mult_default)); +} + +bool +arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult) +{ + + if (opt_purge != purge_mode_ratio) + return (true); + if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) + return (true); + atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult); + return (false); +} + +ssize_t +arena_decay_time_default_get(void) +{ + + return ((ssize_t)atomic_read_z((size_t *)&decay_time_default)); +} + +bool +arena_decay_time_default_set(ssize_t decay_time) +{ + + if (opt_purge != purge_mode_decay) + return (true); + if (!arena_decay_time_valid(decay_time)) + return (true); + atomic_write_z((size_t *)&decay_time_default, (size_t)decay_time); + return (false); +} + +static void +arena_basic_stats_merge_locked(arena_t *arena, unsigned *nthreads, + const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, + size_t *nactive, size_t *ndirty) +{ + + *nthreads += arena_nthreads_get(arena, false); + *dss = dss_prec_names[arena->dss_prec]; + *lg_dirty_mult = arena->lg_dirty_mult; + *decay_time = arena->decay.time; + *nactive += arena->nactive; + *ndirty += arena->ndirty; +} + +void +arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, + const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, + size_t *nactive, size_t *ndirty) +{ + + malloc_mutex_lock(tsdn, &arena->lock); + arena_basic_stats_merge_locked(arena, nthreads, dss, lg_dirty_mult, + decay_time, nactive, ndirty); + malloc_mutex_unlock(tsdn, &arena->lock); +} + +void +arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, + const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time, + size_t *nactive, size_t *ndirty, arena_stats_t *astats, + malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, + malloc_huge_stats_t *hstats) +{ + unsigned i; + + cassert(config_stats); + + malloc_mutex_lock(tsdn, &arena->lock); + arena_basic_stats_merge_locked(arena, nthreads, dss, lg_dirty_mult, + decay_time, nactive, ndirty); + + astats->mapped += arena->stats.mapped; + astats->retained += arena->stats.retained; + astats->npurge += arena->stats.npurge; + astats->nmadvise += arena->stats.nmadvise; + astats->purged += arena->stats.purged; + astats->metadata_mapped += arena->stats.metadata_mapped; + astats->metadata_allocated += arena_metadata_allocated_get(arena); + astats->allocated_large += arena->stats.allocated_large; + astats->nmalloc_large += arena->stats.nmalloc_large; + astats->ndalloc_large += arena->stats.ndalloc_large; + astats->nrequests_large += arena->stats.nrequests_large; + astats->allocated_huge += arena->stats.allocated_huge; + astats->nmalloc_huge += arena->stats.nmalloc_huge; + astats->ndalloc_huge += arena->stats.ndalloc_huge; + + for (i = 0; i < nlclasses; i++) { + lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; + lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; + lstats[i].nrequests += arena->stats.lstats[i].nrequests; + lstats[i].curruns += arena->stats.lstats[i].curruns; + } + + for (i = 0; i < nhclasses; i++) { + hstats[i].nmalloc += arena->stats.hstats[i].nmalloc; + hstats[i].ndalloc += arena->stats.hstats[i].ndalloc; + hstats[i].curhchunks += arena->stats.hstats[i].curhchunks; + } + malloc_mutex_unlock(tsdn, &arena->lock); + + for (i = 0; i < NBINS; i++) { + arena_bin_t *bin = &arena->bins[i]; + + malloc_mutex_lock(tsdn, &bin->lock); + bstats[i].nmalloc += bin->stats.nmalloc; + bstats[i].ndalloc += bin->stats.ndalloc; + bstats[i].nrequests += bin->stats.nrequests; + bstats[i].curregs += bin->stats.curregs; + if (config_tcache) { + bstats[i].nfills += bin->stats.nfills; + bstats[i].nflushes += bin->stats.nflushes; + } + bstats[i].nruns += bin->stats.nruns; + bstats[i].reruns += bin->stats.reruns; + bstats[i].curruns += bin->stats.curruns; + malloc_mutex_unlock(tsdn, &bin->lock); + } +} + +unsigned +arena_nthreads_get(arena_t *arena, bool internal) +{ + + return (atomic_read_u(&arena->nthreads[internal])); +} + +void +arena_nthreads_inc(arena_t *arena, bool internal) +{ + + atomic_add_u(&arena->nthreads[internal], 1); +} + +void +arena_nthreads_dec(arena_t *arena, bool internal) +{ + + atomic_sub_u(&arena->nthreads[internal], 1); +} + +size_t +arena_extent_sn_next(arena_t *arena) +{ + + return (atomic_add_z(&arena->extent_sn_next, 1) - 1); +} + +arena_t * +arena_new(tsdn_t *tsdn, unsigned ind) +{ + arena_t *arena; + unsigned i; + + /* + * Allocate arena, arena->lstats, and arena->hstats contiguously, mainly + * because there is no way to clean up if base_alloc() OOMs. + */ + if (config_stats) { + arena = (arena_t *)base_alloc(tsdn, + CACHELINE_CEILING(sizeof(arena_t)) + + QUANTUM_CEILING((nlclasses * sizeof(malloc_large_stats_t))) + + (nhclasses * sizeof(malloc_huge_stats_t))); + } else + arena = (arena_t *)base_alloc(tsdn, sizeof(arena_t)); + if (arena == NULL) + return (NULL); + + arena->ind = ind; + arena->nthreads[0] = arena->nthreads[1] = 0; + if (malloc_mutex_init(&arena->lock, "arena", WITNESS_RANK_ARENA)) + return (NULL); + + if (config_stats) { + memset(&arena->stats, 0, sizeof(arena_stats_t)); + arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena + + CACHELINE_CEILING(sizeof(arena_t))); + memset(arena->stats.lstats, 0, nlclasses * + sizeof(malloc_large_stats_t)); + arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena + + CACHELINE_CEILING(sizeof(arena_t)) + + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t))); + memset(arena->stats.hstats, 0, nhclasses * + sizeof(malloc_huge_stats_t)); + if (config_tcache) + ql_new(&arena->tcache_ql); + } + + if (config_prof) + arena->prof_accumbytes = 0; + + if (config_cache_oblivious) { + /* + * A nondeterministic seed based on the address of arena reduces + * the likelihood of lockstep non-uniform cache index + * utilization among identical concurrent processes, but at the + * cost of test repeatability. For debug builds, instead use a + * deterministic seed. + */ + arena->offset_state = config_debug ? ind : + (size_t)(uintptr_t)arena; + } + + arena->dss_prec = chunk_dss_prec_get(); + + ql_new(&arena->achunks); + + arena->extent_sn_next = 0; + + arena->spare = NULL; + + arena->lg_dirty_mult = arena_lg_dirty_mult_default_get(); + arena->purging = false; + arena->nactive = 0; + arena->ndirty = 0; + + for (i = 0; i < NPSIZES; i++) + arena_run_heap_new(&arena->runs_avail[i]); + + qr_new(&arena->runs_dirty, rd_link); + qr_new(&arena->chunks_cache, cc_link); + + if (opt_purge == purge_mode_decay) + arena_decay_init(arena, arena_decay_time_default_get()); + + ql_new(&arena->huge); + if (malloc_mutex_init(&arena->huge_mtx, "arena_huge", + WITNESS_RANK_ARENA_HUGE)) + return (NULL); + + extent_tree_szsnad_new(&arena->chunks_szsnad_cached); + extent_tree_ad_new(&arena->chunks_ad_cached); + extent_tree_szsnad_new(&arena->chunks_szsnad_retained); + extent_tree_ad_new(&arena->chunks_ad_retained); + if (malloc_mutex_init(&arena->chunks_mtx, "arena_chunks", + WITNESS_RANK_ARENA_CHUNKS)) + return (NULL); + ql_new(&arena->node_cache); + if (malloc_mutex_init(&arena->node_cache_mtx, "arena_node_cache", + WITNESS_RANK_ARENA_NODE_CACHE)) + return (NULL); + + arena->chunk_hooks = chunk_hooks_default; + + /* Initialize bins. */ + for (i = 0; i < NBINS; i++) { + arena_bin_t *bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock, "arena_bin", + WITNESS_RANK_ARENA_BIN)) + return (NULL); + bin->runcur = NULL; + arena_run_heap_new(&bin->runs); + if (config_stats) + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); + } + + return (arena); +} + +/* + * Calculate bin_info->run_size such that it meets the following constraints: + * + * *) bin_info->run_size <= arena_maxrun + * *) bin_info->nregs <= RUN_MAXREGS + * + * bin_info->nregs and bin_info->reg0_offset are also calculated here, since + * these settings are all interdependent. + */ +static void +bin_info_run_size_calc(arena_bin_info_t *bin_info) +{ + size_t pad_size; + size_t try_run_size, perfect_run_size, actual_run_size; + uint32_t try_nregs, perfect_nregs, actual_nregs; + + /* + * Determine redzone size based on minimum alignment and minimum + * redzone size. Add padding to the end of the run if it is needed to + * align the regions. The padding allows each redzone to be half the + * minimum alignment; without the padding, each redzone would have to + * be twice as large in order to maintain alignment. + */ + if (config_fill && unlikely(opt_redzone)) { + size_t align_min = ZU(1) << (ffs_zu(bin_info->reg_size) - 1); + if (align_min <= REDZONE_MINSIZE) { + bin_info->redzone_size = REDZONE_MINSIZE; + pad_size = 0; + } else { + bin_info->redzone_size = align_min >> 1; + pad_size = bin_info->redzone_size; + } + } else { + bin_info->redzone_size = 0; + pad_size = 0; + } + bin_info->reg_interval = bin_info->reg_size + + (bin_info->redzone_size << 1); + + /* + * Compute run size under ideal conditions (no redzones, no limit on run + * size). + */ + try_run_size = PAGE; + try_nregs = (uint32_t)(try_run_size / bin_info->reg_size); + do { + perfect_run_size = try_run_size; + perfect_nregs = try_nregs; + + try_run_size += PAGE; + try_nregs = (uint32_t)(try_run_size / bin_info->reg_size); + } while (perfect_run_size != perfect_nregs * bin_info->reg_size); + assert(perfect_nregs <= RUN_MAXREGS); + + actual_run_size = perfect_run_size; + actual_nregs = (uint32_t)((actual_run_size - pad_size) / + bin_info->reg_interval); + + /* + * Redzones can require enough padding that not even a single region can + * fit within the number of pages that would normally be dedicated to a + * run for this size class. Increase the run size until at least one + * region fits. + */ + while (actual_nregs == 0) { + assert(config_fill && unlikely(opt_redzone)); + + actual_run_size += PAGE; + actual_nregs = (uint32_t)((actual_run_size - pad_size) / + bin_info->reg_interval); + } + + /* + * Make sure that the run will fit within an arena chunk. + */ + while (actual_run_size > arena_maxrun) { + actual_run_size -= PAGE; + actual_nregs = (uint32_t)((actual_run_size - pad_size) / + bin_info->reg_interval); + } + assert(actual_nregs > 0); + assert(actual_run_size == s2u(actual_run_size)); + + /* Copy final settings. */ + bin_info->run_size = actual_run_size; + bin_info->nregs = actual_nregs; + bin_info->reg0_offset = (uint32_t)(actual_run_size - (actual_nregs * + bin_info->reg_interval) - pad_size + bin_info->redzone_size); + + assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs + * bin_info->reg_interval) + pad_size == bin_info->run_size); +} + +static void +bin_info_init(void) +{ + arena_bin_info_t *bin_info; + +#define BIN_INFO_INIT_bin_yes(index, size) \ + bin_info = &arena_bin_info[index]; \ + bin_info->reg_size = size; \ + bin_info_run_size_calc(bin_info); \ + bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); +#define BIN_INFO_INIT_bin_no(index, size) +#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \ + BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)) + SIZE_CLASSES +#undef BIN_INFO_INIT_bin_yes +#undef BIN_INFO_INIT_bin_no +#undef SC +} + +void +arena_boot(void) +{ + unsigned i; + + arena_lg_dirty_mult_default_set(opt_lg_dirty_mult); + arena_decay_time_default_set(opt_decay_time); + + /* + * Compute the header size such that it is large enough to contain the + * page map. The page map is biased to omit entries for the header + * itself, so some iteration is necessary to compute the map bias. + * + * 1) Compute safe header_size and map_bias values that include enough + * space for an unbiased page map. + * 2) Refine map_bias based on (1) to omit the header pages in the page + * map. The resulting map_bias may be one too small. + * 3) Refine map_bias based on (2). The result will be >= the result + * from (2), and will always be correct. + */ + map_bias = 0; + for (i = 0; i < 3; i++) { + size_t header_size = offsetof(arena_chunk_t, map_bits) + + ((sizeof(arena_chunk_map_bits_t) + + sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); + map_bias = (header_size + PAGE_MASK) >> LG_PAGE; + } + assert(map_bias > 0); + + map_misc_offset = offsetof(arena_chunk_t, map_bits) + + sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); + + arena_maxrun = chunksize - (map_bias << LG_PAGE); + assert(arena_maxrun > 0); + large_maxclass = index2size(size2index(chunksize)-1); + if (large_maxclass > arena_maxrun) { + /* + * For small chunk sizes it's possible for there to be fewer + * non-header pages available than are necessary to serve the + * size classes just below chunksize. + */ + large_maxclass = arena_maxrun; + } + assert(large_maxclass > 0); + nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS); + nhclasses = NSIZES - nlclasses - NBINS; + + bin_info_init(); +} + +void +arena_prefork0(tsdn_t *tsdn, arena_t *arena) +{ + + malloc_mutex_prefork(tsdn, &arena->lock); +} + +void +arena_prefork1(tsdn_t *tsdn, arena_t *arena) +{ + + malloc_mutex_prefork(tsdn, &arena->chunks_mtx); +} + +void +arena_prefork2(tsdn_t *tsdn, arena_t *arena) +{ + + malloc_mutex_prefork(tsdn, &arena->node_cache_mtx); +} + +void +arena_prefork3(tsdn_t *tsdn, arena_t *arena) +{ + unsigned i; + + for (i = 0; i < NBINS; i++) + malloc_mutex_prefork(tsdn, &arena->bins[i].lock); + malloc_mutex_prefork(tsdn, &arena->huge_mtx); +} + +void +arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) +{ + unsigned i; + + malloc_mutex_postfork_parent(tsdn, &arena->huge_mtx); + for (i = 0; i < NBINS; i++) + malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock); + malloc_mutex_postfork_parent(tsdn, &arena->node_cache_mtx); + malloc_mutex_postfork_parent(tsdn, &arena->chunks_mtx); + malloc_mutex_postfork_parent(tsdn, &arena->lock); +} + +void +arena_postfork_child(tsdn_t *tsdn, arena_t *arena) +{ + unsigned i; + + malloc_mutex_postfork_child(tsdn, &arena->huge_mtx); + for (i = 0; i < NBINS; i++) + malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock); + malloc_mutex_postfork_child(tsdn, &arena->node_cache_mtx); + malloc_mutex_postfork_child(tsdn, &arena->chunks_mtx); + malloc_mutex_postfork_child(tsdn, &arena->lock); +} |