#include "mimalloc.h"
#include "mimalloc/prim/prim-tls.h"
#include "mimalloc/util/stats.h"
#include "mimalloc/util/memid.h"
#include "mimalloc/alloc/page.h"
#include "mimalloc/arena/arena-alloc.h"
#include "mimalloc/profile/sample-guarded.h"
#include "mimalloc/arena/arena-page.h"
#include "arena-page.h"
static mi_page_guards_t mi_theap_page_guards(const mi_theap_t* theap, size_t size, bool singleton, bool os_align) {
const size_t gsize = _mi_os_secure_guard_page_size();
const bool guard_end = (theap->guard_end || (singleton && MI_SECURE >= 1)); mi_page_guards_t guards = { 0, 0 };
const mi_arena_t* const req_arena = mi_theap_heap(theap)->exclusive_arena;
if (req_arena != NULL && req_arena->memid.is_pinned) return guards;
if (theap->guard_front && !os_align && gsize <= size/4) { guards.front = gsize; }
if (guard_end && guards.front + gsize <= size/4) { guards.end = gsize; }
return guards;
}
static size_t mi_page_guard_front_block_start(size_t guard_front, size_t block_size) {
return mi_align_up(guard_front, mi_bounds_align(block_size)); }
static bool mi_page_block_can_use_large_cache_offset(size_t block_size) {
if (!mi_block_size_has_large_cache_offset(block_size)) return false;
#if MI_GUARDED
if (block_size % _mi_os_page_size() == 0) return false; #endif
return true;
}
static size_t mi_page_meta_in_front_block_start(size_t block_size, bool os_align) {
size_t offset;
#if MI_GUARDED
const size_t os_page_size = _mi_os_page_size();
mi_assert_internal(MI_PAGE_ALIGN >= os_page_size);
if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size ) {
offset = mi_align_up(mi_page_info_size(), os_page_size);
}
else
#endif
if (os_align) {
offset = MI_PAGE_ALIGN;
}
else if (block_size != 0 && mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) {
offset = mi_align_up(mi_page_info_size(), block_size);
}
else if (block_size != 0 && block_size <= MI_SMALL_SIZE_MAX) {
offset = mi_align_up(mi_page_info_size(), block_size);
}
else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) {
const size_t alignment = (mi_page_block_can_use_large_cache_offset(block_size) ? MI_PAGE_LARGE_CACHE_OFFSET_UNIT : MI_PAGE_OSPAGE_BLOCK_ALIGN2);
offset = mi_align_up(mi_page_info_size(), alignment);
}
else {
offset = mi_page_info_size();
}
return mi_align_up(offset, (os_align ? MI_MAX_ALIGN_SIZE : mi_max(MI_MAX_ALIGN_SIZE, mi_bounds_align(block_size))));
}
static size_t mi_page_cache_offset_index(uint8_t* slice_start, size_t slice_count, size_t noffsets) {
const size_t page_index = (size_t)(mi_addr(slice_start) / MI_ARENA_SLICE_SIZE) / slice_count;
#if MI_SIZE_BITS >= 64
const size_t hash = page_index * MI_ZU(0x9E3779B97F4A7C15);
#else
const size_t hash = page_index * MI_ZU(0x9E3779B9);
#endif
return (((hash >> (MI_SIZE_BITS - 16)) * noffsets) >> 16);
}
static size_t mi_page_small_block_cache_offset(uint8_t* slice_start, size_t slice_count, size_t block_size) {
const size_t unit = mi_max(MI_PAGE_SMALL_CACHE_OFFSET_UNIT, mi_natural_alignment_of_size(block_size));
if (unit > MI_PAGE_SMALL_CACHE_OFFSET_MAX) return 0;
return mi_page_cache_offset_index(slice_start, slice_count, 1 + MI_PAGE_SMALL_CACHE_OFFSET_MAX/unit) * unit;
}
static size_t mi_page_block_cache_offset(uint8_t* slice_start, size_t slice_count, size_t block_size, size_t block_start, bool os_align, size_t page_noguard_size) {
if (MI_PAGE_LARGE_CACHE_OFFSET == 0 || os_align) return 0;
if (!mi_block_size_has_large_cache_offset(block_size)) return mi_page_small_block_cache_offset(slice_start, slice_count, block_size);
if (!mi_page_block_can_use_large_cache_offset(block_size)) return 0;
const size_t unit = mi_max(MI_PAGE_LARGE_CACHE_OFFSET_UNIT, mi_bounds_align(block_size)); if (unit > MI_PAGE_LARGE_CACHE_OFFSET_MAX) return 0;
mi_assert_internal(block_start + block_size <= page_noguard_size);
size_t slack = (page_noguard_size - block_start) % block_size;
if (slack < unit && block_size <= page_noguard_size/8) { slack += block_size; } const size_t noffsets = 1 + (slack < MI_PAGE_LARGE_CACHE_OFFSET_MAX ? slack : MI_PAGE_LARGE_CACHE_OFFSET_MAX) / unit;
return mi_page_cache_offset_index(slice_start, slice_count, noffsets) * unit;
}
static mi_page_t* mi_page_meta_select(uint8_t* slice_start, size_t slice_count, mi_page_t* page_meta, size_t block_size, bool os_align,
mi_page_guards_t* guards, size_t* block_start) {
mi_assert_internal(page_meta == NULL || page_meta->block_size == 0);
mi_page_t* page;
size_t offset;
if (page_meta == NULL
#if MI_PAGE_META_SMALL_IS_ALIGNED
|| (!os_align && block_size <= MI_SMALL_MAX_OBJ_SIZE)
#endif
) {
offset = mi_page_meta_in_front_block_start(block_size, os_align);
page = (mi_page_t*)slice_start;
guards->front = 0;
}
else {
mi_assert_internal(guards->front == 0 || guards->front >= MI_PAGE_MAX_START_BLOCK_ALIGN2);
mi_memzero_aligned(page_meta, sizeof(*page_meta));
offset = mi_page_guard_front_block_start(guards->front, block_size);
page = page_meta;
}
offset += mi_page_block_cache_offset(slice_start, slice_count, block_size, offset, os_align, _mi_size_of_slices(slice_count) - guards->end);
mi_assert_internal(os_align || offset < MI_ARENA_SLICE_SIZE); *block_start = offset;
return mi_page_meta_bounded(page);
}
#if MI_PAGE_META_IS_ALIGNED
static bool mi_arena_page_meta_ensure_committed(mi_arena_t* arena, uint8_t* slice_start) {
uint8_t* const meta_slices = (uint8_t*)mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT) + MI_PAGE_META_ALIGNED_OFFSET;
mi_assert_internal(meta_slices >= mi_arena_start(arena));
const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE;
if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) {
const size_t meta_slice_count = mi_arena_page_meta_aligned_array_slices();
const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE;
if (!_mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) {
return false;
}
mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL);
}
return true;
}
#endif
static uint8_t* mi_arenas_page_alloc_from_slices(mi_theap_t* theap, size_t slice_count, bool commit, bool allow_large,
mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage_meta) {
mi_heap_t* const heap = mi_theap_heap(theap);
mi_tld_t* const tld = theap->tld;
const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node);
uint8_t* const start = (uint8_t*)_mi_arenas_try_alloc(heap, slice_count, MI_ARENA_SLICE_ALIGN, commit, allow_large, heap->exclusive_arena, tld->thread_seq, numa_node, memid);
if (start == NULL) return NULL;
mi_arena_t* const arena = mi_memid_arena(*memid);
mi_arena_pages_t* const arena_pages = _mi_heap_ensure_arena_pages(heap, arena);
if (arena_pages==NULL
#if MI_PAGE_META_IS_ALIGNED
|| !mi_arena_page_meta_ensure_committed(arena, start)
#endif
) {
_mi_arenas_free(heap->subproc, start, _mi_size_of_slices(slice_count), *memid); return NULL;
}
mi_assert_internal(mi_bitmap_is_clear(arena_pages->pages, memid->mem.arena.slice_index));
*parena_pages = arena_pages;
#if MI_PAGE_META_IS_ALIGNED
*ppage_meta = mi_aligned_ptr_page0(start);
#else
mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0 || arena->pages_meta == NULL);
*ppage_meta = (arena->pages_meta == NULL ? NULL : mi_page_meta_load(mi_arena_page_meta_at_slice(arena, memid->mem.arena.slice_index)));
#endif
return start;
}
#if MI_PAGE_META_OS_IS_SEPARATED
#if MI_PAGE_META_IS_ALIGNED
static size_t mi_page_meta_os_aligned_size(size_t page_offset, size_t max_page_meta_count) {
size_t count = 0;
const size_t slice_end = mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE, MI_ARENA_SLICE_SIZE) + max_page_meta_count;
for (size_t slice_idx = page_offset / MI_ARENA_SLICE_SIZE; slice_idx < slice_end; slice_idx++) {
count = mi_max(count, mi_page_meta_aligned_index(slice_idx) + 1);
}
return count * sizeof(mi_page_t);
}
#endif
static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, bool allow_large,
mi_memid_t* memid, mi_page_t** ppage_meta) {
MI_UNUSED(allow_large); MI_UNUSED(max_page_meta_count); MI_UNUSED(os_align);
mi_arena_t* const req_arena = heap->exclusive_arena;
const size_t alloc_size = _mi_size_of_slices(slice_count);
const size_t page_alignment = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment);
#if MI_PAGE_META_IS_ALIGNED
if (block_alignment >= MI_PAGE_META_ALIGNMENT) {
_mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB);
errno = EINVAL;
return NULL;
}
const size_t os_alignment = MI_PAGE_META_ALIGNMENT;
#else
const size_t os_alignment = page_alignment;
#endif
size_t page_offset = mi_align_up(_mi_size_of_slices(2*MI_ARENA_GUARD_SLICES + 1), page_alignment);
#if MI_PAGE_META_IS_ALIGNED
size_t page_meta_size = mi_page_meta_os_aligned_size(page_offset, max_page_meta_count);
while (_mi_size_of_slices(MI_ARENA_GUARD_SLICES) + mi_align_up(page_meta_size,_mi_os_page_size()) + _mi_os_secure_guard_page_size() > page_offset) {
page_offset += page_alignment;
page_meta_size = mi_page_meta_os_aligned_size(page_offset, max_page_meta_count);
}
#else
const size_t page_meta_size = sizeof(mi_page_t);
#endif
uint8_t* const os_start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, os_alignment, 0 , false , false , req_arena, memid);
if (os_start==NULL) return NULL;
uint8_t* const start = os_start + page_offset;
uint8_t* const page_meta_start = os_start + _mi_size_of_slices(MI_ARENA_GUARD_SLICES);
mi_assert_internal(page_meta_start + mi_align_up(page_meta_size,_mi_os_page_size()) + _mi_os_secure_guard_page_size() <= start);
bool is_zero;
bool ok = _mi_arena_commit(heap->subproc,req_arena,page_meta_start,page_meta_size,&is_zero,page_meta_size );
if (ok && commit) {
ok = _mi_arena_commit(heap->subproc,req_arena,start,alloc_size,NULL,0);
}
if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; }
if (!is_zero && !memid->initially_zero) {
mi_memzero_aligned(page_meta_start,page_meta_size);
}
#if MI_ARENA_GUARD_SLICES > 0
if (memid->initially_committed && !memid->is_pinned) {
_mi_os_secure_guard_pages_set_around(heap->subproc, page_meta_start, page_meta_size, *memid);
}
#endif
if (commit) {
memid->initially_committed = true; mi_memid_os_set_committed(memid, alloc_size);
}
mi_assert_internal(mi_is_aligned(start,page_alignment));
#if MI_PAGE_META_IS_ALIGNED
mi_assert_internal(mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT));
mi_assert_internal(mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start);
mi_assert_internal((uint8_t*)mi_aligned_ptr_page0(start) < page_meta_start + page_meta_size);
*ppage_meta = mi_aligned_ptr_page0(start);
#else
*ppage_meta = (mi_page_t*)mi_ptr_bounded_data(page_meta_start, sizeof(mi_page_t));
#endif
return start;
}
#else
static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, bool allow_large,
mi_memid_t* memid, mi_page_t** ppage_meta) {
MI_UNUSED(max_page_meta_count);
const size_t page_alignment = MI_ARENA_SLICE_ALIGN;
const size_t alloc_size = _mi_size_of_slices(slice_count);
uint8_t* start;
if (os_align) {
start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment , commit, allow_large, heap->exclusive_arena, memid);
mi_assert_internal(start==NULL || mi_is_aligned(start + page_alignment, block_alignment));
}
else {
start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 , commit, allow_large, heap->exclusive_arena, memid);
}
if (start == NULL) return NULL;
*ppage_meta = NULL;
return start;
}
#endif
static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, bool allow_large,
mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage_meta ) {
mi_assert_internal(parena_pages!=NULL && ppage_meta!=NULL);
*parena_pages = NULL;
*ppage_meta = NULL;
*memid = _mi_memid_none();
mi_heap_t* const heap = mi_theap_heap(theap);
uint8_t* slice_start = NULL;
if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && !os_align && slice_count <= mi_arena_max_object_size()/MI_ARENA_SLICE_SIZE) {
slice_start = mi_arenas_page_alloc_from_slices(theap, slice_count, commit, allow_large, memid, parena_pages, ppage_meta);
}
if (slice_start == NULL) {
slice_start = mi_arenas_page_alloc_from_os(heap, slice_count, max_page_meta_count, block_alignment, os_align, commit, allow_large, memid, ppage_meta);
if (slice_start == NULL) return NULL;
mi_heap_stat_increase(heap,pages_os_allocated,1);
mi_atomic_increment_relaxed(&heap->os_page_count);
}
mi_assert_internal(mi_is_aligned(slice_start, MI_PAGE_ALIGN));
return slice_start;
}
static bool mi_arenas_page_commit_first_block(mi_theap_t* theap, mi_memid_t memid, uint8_t* slice_start, size_t block_start, size_t block_size, size_t page_end, size_t* commit_size) {
*commit_size = 0;
if (memid.initially_committed) return true;
mi_assert_internal(!memid.guard_front); size_t csize = mi_align_up(block_start + block_size, mi_page_min_commit_size());
if (csize > page_end) { csize = page_end; }
bool is_zero = false;
if mi_unlikely(!_mi_arena_commit(mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, csize, &is_zero, 0)) {
return false;
}
*commit_size = csize;
return true;
}
static void mi_arenas_page_init_memory(mi_theap_t* theap, mi_page_t* page, uint8_t* slice_start, size_t slice_count, size_t block_start, size_t page_end, mi_memid_t* memid)
{
MI_UNUSED(slice_count); MI_UNUSED(block_start);
if (!memid->initially_zero && (uint8_t*)page == slice_start) {
mi_memzero_aligned(page, sizeof(*page));
}
if (!memid->initially_zero && memid->initially_committed) {
mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);
}
else if (memid->initially_committed) {
mi_track_mem_defined(slice_start, slice_count * MI_ARENA_SLICE_SIZE);
}
#if MI_DEBUG > 1
if (memid->initially_zero && memid->initially_committed) {
if (!mi_mem_is_zero(slice_start, page_end)) {
_mi_error_message(EFAULT, "internal error: page memory was not zero initialized.\n");
memid->initially_zero = false;
if (block_start > 0) { mi_memzero_aligned(page, sizeof(*page)); }
}
}
#endif
mi_subproc_t* const subproc = mi_theap_subproc(theap);
if (memid->guard_front) { _mi_os_secure_guard_page_set_at(subproc, slice_start, *memid); }
if (memid->guard_end) { _mi_os_secure_guard_page_set_at(subproc, slice_start + page_end, *memid); }
}
static mi_page_t* mi_page_meta_memtag(mi_page_t* page, uint8_t* slice_start, mi_memid_t memid) {
if (!mi_memtag_use_for_meta(memid.is_memtag)) return page;
if (mi_addr(page) == mi_addr(slice_start)) {
return mi_page_meta_bounded(mi_memtag_meta_init_class(page, sizeof(mi_page_t), 1));
}
mi_arena_t* const arena = mi_memid_arena(memid);
if (arena != NULL && arena->pages_meta != NULL) {
mi_assert_internal(mi_addr(page) == mi_addr(mi_arena_page_meta_at_slice(arena, memid.mem.arena.slice_index)));
return mi_page_meta_bounded(mi_arena_page_meta_memtag(arena, page));
}
#if MI_PAGE_META_IS_ALIGNED
return mi_page_meta_bounded(mi_memtag_meta_init_class(page, sizeof(mi_page_t), mi_page_meta_memtag_idx(page))); #else
return mi_page_meta_bounded(mi_memtag_meta_init_separate(page, sizeof(mi_page_t), mi_page_meta_memtag_idx(page), NULL));
#endif
}
static bool mi_page_meta_is_arena_entry(mi_page_t* page) {
#if MI_PAGE_META_IS_ALIGNED
MI_UNUSED(page); return false;
#else
return (page->memid.memkind == MI_MEM_ARENA && mi_page_meta_is_separated(page));
#endif
}
static void mi_page_meta_memtag_done(mi_page_t* page, mi_memid_t memid) {
if (!mi_memtag_use_for_meta(memid.is_memtag)) return;
if (mi_page_meta_is_arena_entry(page)) { mi_arena_page_meta_memtag(mi_memid_arena(memid), page); }
else { mi_memtag_clear(page, sizeof(mi_page_t)); }
}
static void mi_arenas_page_init_meta(mi_theap_t* theap, mi_page_t* page, uint8_t* slice_start, size_t block_start, size_t block_size,
size_t reserved, size_t commit_size, mi_memid_t memid)
{
mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX);
uint8_t* const start = slice_start + block_start;
mi_assert_internal(mi_addr(start) > mi_addr(page));
page->page_start = (uint8_t*)mi_ptr_bounded_data(start, reserved * block_size);
page->reserved = (uint16_t)reserved;
page->block_size = block_size;
#if MI_FREE_IS_CHECKED
mi_assert_internal(reserved==1 || (uint64_t)reserved * block_size <= UINT32_MAX);
page->block_recip = (reserved > 1 ? (UINT64_MAX / block_size) + 1 : 1);
#endif
page->memid = memid;
page->free_is_zero = memid.initially_zero;
page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size());
page->heap = mi_theap_heap(theap);
#if MI_DEBUG>0
mi_atomic_store_relaxed(&page->debug_fill_preserve, mi_atomic_load_relaxed(&page->heap->debug_fill_preserve));
#endif
if (block_size <= MI_LARGE_MAX_OBJ_SIZE && mi_memtag_use_for(memid.is_memtag, block_size)) { mi_page_flags_set(page, true, MI_PAGE_IS_MEMTAGGED);
}
mi_page_set_theap(page,theap);
}
#if MI_PAGE_META_IS_ALIGNED
static void mi_arenas_page_link_aligned_meta(mi_page_t* page, uint8_t* slice_start, size_t slice_count, size_t max_page_meta_count)
{
mi_page_t* const page_meta = mi_aligned_ptr_page0(slice_start);
mi_atomic_store_ptr_release(mi_page_t,&mi_page_meta_load(page_meta)->self,page);
if (slice_count > 1) {
for(size_t i = 1; i < max_page_meta_count; i++) {
mi_page_t* const meta = mi_page_meta_load(mi_aligned_ptr_page0(slice_start + _mi_size_of_slices(i)));
mi_assert_internal(meta->block_size == 0);
mi_atomic_store_ptr_release(mi_page_t,&meta->self,page);
}
}
#if MI_DEBUG>1
mi_page_t* pstart = mi_page_meta_load(mi_aligned_ptr_page0(slice_start));
mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page);
if (page->reserved>1) {
mi_page_t* pend = mi_page_meta_load(mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1));
mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page);
}
#endif
}
#endif
static bool mi_arenas_page_register(mi_theap_t* theap, mi_page_t* page, mi_arena_pages_t* arena_pages)
{
if (arena_pages!=NULL) {
mi_assert_internal(page->memid.memkind == MI_MEM_ARENA);
mi_bitmap_set(arena_pages->pages, page->memid.mem.arena.slice_index);
}
mi_page_claim_ownership(page);
if mi_unlikely(!_mi_page_map_register(page)) {
_mi_arenas_page_free_prim(page);
return false;
}
mi_theap_stat_increase(theap, pages, 1);
mi_theap_stat_increase(theap, page_bins[_mi_page_stats_bin(page)], 1);
return true;
}
static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit, mi_page_guards_t guards)
{
const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE);
const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count);
const bool allow_large = (guards.front == 0 && guards.end == 0);
mi_memid_t memid = _mi_memid_none();
mi_arena_pages_t* arena_pages = NULL;
mi_page_t* page_meta = NULL;
uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit || guards.front > 0,allow_large,&memid,&arena_pages,&page_meta);
if (!slice_start) return NULL;
#if MI_SECURE >= 1 || MI_GUARDED
if (memid.memkind == MI_MEM_ARENA && !memid.is_pinned && _mi_fork_is_torn()) {
_mi_os_unguard(slice_start, _mi_size_of_slices(slice_count), true );
}
#endif
size_t block_start = 0;
mi_page_t* page = mi_page_meta_select(slice_start, slice_count, page_meta, block_size, os_align, &guards, &block_start);
const size_t alloc_size = _mi_size_of_slices(slice_count);
const size_t reserved = (os_align ? 1 : (alloc_size - guards.end - block_start) / block_size);
const size_t page_end = (guards.end > 0 ? mi_align_up(block_start + reserved*block_size, _mi_os_page_size()) : alloc_size); mi_assert_internal(page_end + guards.end <= alloc_size);
memid.guard_front = (guards.front > 0);
memid.guard_end = (guards.end > 0);
mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0);
mi_assert_internal(block_size % mi_natural_alignment_of_size(block_size) == 0);
mi_assert_internal(mi_is_aligned(slice_start + block_start, mi_natural_alignment_of_size(block_size)));
mi_assert_internal(mi_bounds_size(block_size) == block_size && mi_is_aligned(slice_start + block_start, mi_bounds_align(block_size)));
size_t commit_size = 0;
if (!mi_arenas_page_commit_first_block(theap, memid, slice_start, block_start, block_size, page_end, &commit_size)) {
_mi_arenas_free(mi_theap_subproc(theap), slice_start, alloc_size, memid);
return NULL;
}
mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size())));
mi_arenas_page_init_memory(theap, page, slice_start, slice_count, block_start, page_end, &memid);
page = mi_page_meta_memtag(page, slice_start, memid);
mi_arenas_page_init_meta(theap, page, slice_start, block_start, block_size, reserved, commit_size, memid);
#if MI_PAGE_META_IS_ALIGNED
mi_arenas_page_link_aligned_meta(page, slice_start, slice_count, max_page_meta_count);
#endif
mi_assert_internal(page->free==NULL);
mi_assert_internal((mi_addr(page) != mi_addr(slice_start)) == mi_page_meta_is_separated(page));
mi_assert_internal(mi_page_slice_start(page) == slice_start);
mi_assert_internal(mi_page_slice_offset_of(page, mi_page_size(page)) <= page_end);
mi_assert_internal(!memid.guard_end || mi_page_guard_end_offset(page) == page_end);
if (!mi_arenas_page_register(theap, page, arena_pages)) return NULL;
mi_assert_internal(mi_is_aligned(mi_page_slice_start(page),MI_PAGE_ALIGN));
mi_assert_internal(mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(mi_page_block_size(page) == block_size);
#if MI_PAGE_META_SMALL_IS_ALIGNED
if (mi_page_is_small(page)) {
mi_assert_internal(mi_addr(page) == mi_addr(slice_start) && mi_is_aligned(page, MI_SMALL_PAGE_SIZE));
mi_assert_internal(mi_addr(mi_page_start(page)) + mi_page_size(page) <= mi_addr(page) + MI_SMALL_PAGE_SIZE);
}
#endif
mi_assert_internal(mi_page_is_owned(page));
return page;
}
static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_count, size_t block_size)
{
const mi_page_guards_t guards = mi_theap_page_guards(theap, _mi_size_of_slices(slice_count), false, false);
mi_page_t* page = _mi_arenas_page_try_find_abandoned(theap, slice_count, block_size, guards);
if (page != NULL) {
return page; }
const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand);
const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || (slice_count >= _mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0));
page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit, guards);
if (page == NULL) return NULL;
mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count);
if (!_mi_page_init(theap, page)) {
_mi_arenas_page_free_direct(page,theap);
return NULL;
}
return page;
}
static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment)
{
const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
#if MI_PAGE_META_OS_IS_SEPARATED
const size_t info_size = 0;
#else
const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_align_up(mi_page_info_size(), mi_bounds_align(block_size))); #endif
const size_t cache_offset_size = (!os_align && mi_page_block_can_use_large_cache_offset(block_size) ? MI_PAGE_LARGE_CACHE_OFFSET_MAX : 0);
const size_t size = info_size + cache_offset_size + block_size;
const mi_page_guards_t guards = mi_theap_page_guards(theap, _mi_size_of_slices(_mi_slice_count_of_size(size)), true, os_align);
#if MI_MTE
const size_t memtag_slack = (cache_offset_size == 0 && guards.end == 0 ? MI_MEMTAG_UNIT_SIZE : 0);
#else
const size_t memtag_slack = 0;
#endif
const size_t slice_count = _mi_slice_count_of_size(mi_page_guard_front_block_start(guards.front, block_size) + size + memtag_slack + guards.end);
mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true , guards);
if (page == NULL) return NULL;
mi_assert(page->reserved == 1);
if (!_mi_page_init(theap, page)) {
_mi_arenas_page_free_direct(page,theap);
return NULL;
}
return page;
}
mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) {
mi_page_t* page;
mi_assert(_mi_bin(MI_LARGE_MAX_OBJ_SIZE) < MI_ARENA_BIN_COUNT);
if mi_unlikely(block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) {
mi_assert_internal(mi_is_power_of_two(block_alignment));
page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);
}
else if (block_size <= MI_SMALL_MAX_OBJ_SIZE) {
page = mi_arenas_page_regular_alloc(theap, _mi_slice_count_of_size(MI_SMALL_PAGE_SIZE), block_size);
}
else if (mi_block_size_is_medium(block_size)) {
page = mi_arenas_page_regular_alloc(theap, _mi_slice_count_of_size(MI_MEDIUM_PAGE_SIZE), block_size);
}
#if MI_ENABLE_LARGE_PAGES
else if (block_size <= MI_LARGE_MAX_OBJ_SIZE) {
page = mi_arenas_page_regular_alloc(theap, _mi_slice_count_of_size(MI_LARGE_PAGE_SIZE), block_size);
}
#endif
else {
page = mi_arenas_page_singleton_alloc(theap, block_size, block_alignment);
}
if mi_unlikely(page == NULL) {
return NULL;
}
mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_assert_internal(mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(block_alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN || mi_is_aligned(mi_page_start(page), block_alignment));
return page;
}
static size_t mi_page_full_size(mi_page_t* page) {
if (page->memid.memkind == MI_MEM_ARENA) {
return page->memid.mem.arena.slice_count * MI_ARENA_SLICE_SIZE;
}
else if (mi_memid_is_os(page->memid)) {
uint8_t* const slice_start = mi_page_slice_start(page);
mi_assert_internal((uint8_t*)page->memid.mem.os.base <= slice_start);
const size_t presize = (size_t)(slice_start - (uint8_t*)page->memid.mem.os.base);
const size_t os_size = mi_memid_os_size(page->memid);
mi_assert_internal(os_size >= presize);
return (presize > os_size ? 0 : os_size - presize);
}
else {
return 0;
}
}
void _mi_arenas_page_free_prim(mi_page_t* page) {
mi_assert_internal(_mi_page_qsbr_is_clear(page));
mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_assert_internal(mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_all_free(page));
mi_assert_internal(page->next==NULL && page->prev==NULL);
mi_subproc_t* const subproc = mi_page_subproc(page); mi_arena_t* arena = NULL;
mi_arena_pages_t* arena_pages = NULL;
size_t slice_index = 0;
size_t slice_count = 0;
if (page->memid.memkind == MI_MEM_ARENA) {
arena = _mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages);
mi_assert_internal(arena_pages!=NULL);
mi_assert_internal(arena->subproc == subproc);
#if MI_DEBUG>1
if (!mi_page_is_full(page)) {
size_t bin = _mi_bin(mi_page_block_size(page));
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));
}
#endif
mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages, slice_index);
}
_mi_page_map_unregister(page);
if (mi_page_is_memtag(page)) {
mi_memtag_clear(mi_page_start(page), page->capacity * mi_page_block_size(page));
}
uint8_t* const slice_start = mi_page_slice_start(page);
if (page->memid.guard_front) { _mi_os_secure_guard_page_reset_at(subproc, slice_start, page->memid); }
if (page->memid.guard_end) { _mi_os_secure_guard_page_reset_at(subproc, slice_start + mi_page_guard_end_offset(page), page->memid); }
mi_memid_t memid = page->memid;
const size_t slice_committed = mi_page_slice_committed(page);
if (arena != NULL) {
if (slice_committed > 0) {
mi_assert_internal(mi_page_full_size(page) >= slice_committed);
const size_t total_slices = slice_committed / MI_ARENA_SLICE_SIZE; mi_assert_internal(slice_count >= total_slices);
if (total_slices > 0) {
mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL);
}
const size_t extra = slice_committed % MI_ARENA_SLICE_SIZE;
if (extra > 0) {
mi_subproc_stat_decrease(arena->subproc, committed, extra);
}
}
else {
mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
}
}
else {
mi_heap_stat_decrease(page->heap, pages_os_allocated, 1);
mi_atomic_sub_acq_rel(&page->heap->os_page_count, 1); if (slice_committed > 0) { mi_memid_os_set_committed(&memid, slice_committed); } }
if (mi_page_meta_is_separated(page)) { page->block_size = 0; } const size_t full_size = mi_page_full_size(page);
mi_page_meta_memtag_done(page, memid); _mi_arenas_free(subproc, slice_start, full_size, memid);
}
void _mi_arenas_page_free_direct(mi_page_t* page, mi_theap_t* current_theapx) {
mi_assert_internal(_mi_page_qsbr_is_clear(page));
mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_assert_internal(mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_all_free(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(page->next==NULL && page->prev==NULL);
mi_assert_internal(mi_theap_matches_thread(current_theapx));
mi_heap_t* const heap = mi_page_heap(page);
mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1);
mi_theapx_stat_decrease(heap, current_theapx, pages, 1);
_mi_page_free_collect(page,false); _mi_page_update_stats(page); _mi_arenas_page_free_prim(page);
}
bool _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
mi_assert_internal(mi_page_is_owned(page) && mi_page_all_free(page));
mi_assert_internal(mi_page_is_abandoned(page) && !mi_page_is_abandoned_mapped(page));
mi_assert_internal(page->next == NULL && page->prev == NULL);
if (_mi_page_qsbr_should_delay(page)) {
#if MI_QSBR
if (mi_page_is_singleton(page)) {
_mi_heap_qsbr_delayed_enqueue(page);
return false;
}
#endif
_mi_arenas_page_abandon(page, current_theapx);
return false;
}
_mi_page_qsbr_clear(page);
_mi_arenas_page_free_direct(page, current_theapx);
return true;
}