#include "mimalloc.h"
#include "mimalloc/util/memid.h"
#include "mimalloc/alloc/page.h"
#include "mimalloc/arena/arena.h"
#include "mimalloc/heap/subproc.h"
#include "mimalloc/heap/page-map.h"
#include "../arena/bitmap.h"
static void mi_page_map_cannot_commit(void) {
_mi_warning_message("unable to commit the allocation page-map on-demand\n" );
}
#define MI_PAGE_MAP_SUB_SIZE (sizeof(mi_page_submap_t))
static void mi_page_map_set_at(mi_page_map_t* pmap, size_t idx, mi_page_submap_t* sub) {
mi_atomic_ptr_sign_release(&pmap->submaps[idx], sub);
}
static size_t mi_page_map_count_of_size(size_t size) {
return (size < sizeof(mi_page_map_t) ? 0 : 1 + (size - sizeof(mi_page_map_t))/sizeof(mi_ptr_signed_t));
}
static size_t mi_page_map_size_of_count(size_t count) {
return (count == 0 ? 0 : sizeof(mi_page_map_t) + (count - 1)*sizeof(mi_ptr_signed_t));
}
static bool mi_page_map_use_memtag(const mi_memid_t* memid) {
return mi_memtag_use_for_meta(!memid->is_pinned);
}
static void mi_page_map_store_committed(size_t committed_count) {
mi_process_sealed_t* const sealed = _mi_process_unseal();
mi_atomic_store_release(&sealed->page_map_committed, committed_count);
_mi_process_seal();
}
static mi_page_map_t* mi_page_map_vmem(mi_page_map_t* pmap) {
return (mi_page_map_t*)mi_ptr_memtag_meta(mi_ptr_rederive(pmap->memid.mem.os.base, pmap));
}
mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_page_map_t* pmap, size_t required_idx) {
const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size);
if mi_unlikely(required_idx >= reserved_count) {
mi_page_map_cannot_commit();
return false;
}
size_t commit_size = mi_align_up( sizeof(mi_page_map_t) + (required_idx * sizeof(mi_ptr_signed_t)), MI_ARENA_SLICE_SIZE );
if (pmap->reserved_size < commit_size) { commit_size = pmap->reserved_size; }
const size_t commit_count = mi_page_map_count_of_size(commit_size);
mi_assert_internal(commit_count > required_idx);
mi_assert_internal(commit_count <= reserved_count);
const size_t prev_count = mi_page_map_committed();
if (!pmap->memid.initially_committed) { bool is_zero;
if mi_unlikely(!_mi_os_commit(_mi_subproc_main(), mi_page_map_vmem(pmap), commit_size, &is_zero)) {
mi_page_map_cannot_commit();
return false;
}
mi_assert_internal(is_zero || pmap->memid.initially_zero);
}
if (mi_page_map_use_memtag(&pmap->memid)) {
const size_t tag_start = mi_align_down(mi_page_map_size_of_count(prev_count), MI_MEMTAG_UNIT_SIZE);
mi_memtag_store((uint8_t*)pmap + tag_start, commit_size - tag_start);
}
mi_page_map_store_committed(commit_count);
return true;
}
mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, size_t idx, mi_page_submap_t** submap) {
mi_assert_internal(submap!=NULL && *submap==NULL);
if mi_unlikely(idx >= mi_page_map_committed()) {
bool ok = true;
mi_lock(&pmap->lock) {
if (idx >= mi_atomic_load_acquire(&mi_process_get_sealed()->page_map_committed)) {
ok = mi_page_map_commit_entries(pmap,idx);
}
}
if (!ok) return false;
mi_assert_internal(idx < mi_page_map_committed());
}
*submap = mi_page_map_at(pmap, idx);
return true;
}
static bool mi_page_map_init_once(void) {
size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS);
if (vbits == 0) {
vbits = _mi_os_virtual_address_bits();
#if MI_ARCH_X64
if (vbits >= 48) { vbits = 47; }
#endif
}
if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) {
vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT;
}
if (vbits < MI_MIN_VABITS) { vbits = MI_MIN_VABITS;
}
if (vbits > MI_MAX_VABITS) { vbits = MI_MAX_VABITS;
}
mi_assert(MI_MAX_VABITS >= vbits);
mi_assert(MI_MIN_VABITS <= vbits);
const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
const size_t os_page_size = _mi_os_page_size();
const size_t reserve_size = mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_ptr_signed_t)), os_page_size);
const size_t submap_size = MI_PAGE_MAP_SUB_SIZE;
const size_t extra_reserve_size = reserve_size + submap_size;
const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit();
mi_subproc_t* const subproc = _mi_subproc_main();
mi_memid_t memid;
mi_page_map_t* pmap = (mi_page_map_t*)_mi_os_alloc_with_guards(subproc, extra_reserve_size, commit, true , &memid);
if mi_unlikely(pmap==NULL) {
_mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", extra_reserve_size / MI_KiB);
return false;
}
const bool use_memtag = mi_page_map_use_memtag(&memid);
const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
const size_t min_commit_size = mi_align_up( mi_page_map_size_of_count(min_commit_count), os_page_size);
mi_assert_internal(min_commit_size <= reserve_size);
size_t commit_count;
if (memid.initially_committed) {
if (!memid.initially_zero) {
_mi_warning_message("internal: the page map was committed but not zero initialized!\n");
mi_memzero_aligned(pmap, extra_reserve_size);
memid.initially_zero = true;
}
commit_count = mi_page_map_count_of_size(use_memtag ? min_commit_size : reserve_size);
}
else {
bool is_zero;
if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) {
mi_page_map_cannot_commit();
_mi_os_free_with_guards(subproc,memid);
return false;
};
mi_assert_internal(is_zero || memid.initially_zero);
commit_count = mi_page_map_count_of_size(min_commit_size);
}
mi_assert_internal(commit_count >= min_commit_count);
mi_page_submap_t* sub0 = (mi_page_submap_t*)((uint8_t*)pmap + reserve_size); if (!memid.initially_committed) {
if (!_mi_os_commit(subproc, sub0, submap_size, NULL)) { mi_page_map_cannot_commit();
_mi_os_free_with_guards(subproc,memid);
return false;
}
}
if (!memid.initially_zero) { mi_memzero_aligned(sub0, submap_size);
}
if (use_memtag) {
pmap = (mi_page_map_t*)mi_memtag_meta_init_separate(pmap, mi_align_up(mi_page_map_size_of_count(commit_count), MI_MEMTAG_UNIT_SIZE), 0, NULL);
sub0 = (mi_page_submap_t*)mi_memtag_meta_init_separate(sub0, submap_size, 1, pmap);
}
pmap->memid = memid;
pmap->reserved_size = reserve_size;
mi_lock_init(&pmap->lock);
mi_page_map_set_at(pmap, 0, (mi_page_submap_t*)mi_ptr_bounded_data(sub0, submap_size));
mi_process_sealed_once_t* const once = _mi_process_unseal_once();
mi_page_map_t* const pmap_bounded = (mi_page_map_t*)mi_ptr_bounded_data(pmap, reserve_size); mi_atomic_store_ptr_release(mi_page_map_t, &once->page_map, (mi_page_map_t*)mi_ptr_memtag_meta(pmap_bounded)); _mi_process_seal_once();
mi_page_map_store_committed(commit_count);
mi_assert_internal(mi_ptr_page(NULL)==NULL);
return true;
}
bool _mi_page_map_init(void) {
mi_atomic_do_once {
mi_page_map_init_once();
}
return (mi_page_map() != NULL);
}
static mi_page_submap_t* mi_page_map_alloc_submap(mi_subproc_t* subproc, mi_memid_t* memid) {
mi_page_submap_t* sub = (mi_page_submap_t*)_mi_os_alloc_with_guards(subproc, MI_PAGE_MAP_SUB_SIZE, true , false , memid);
if (sub != NULL && !memid->initially_zero) {
mi_memzero_aligned(sub, MI_PAGE_MAP_SUB_SIZE);
}
if (sub != NULL && mi_page_map_use_memtag(memid)) {
sub = (mi_page_submap_t*)mi_memtag_meta_init_separate(sub, MI_PAGE_MAP_SUB_SIZE, 0, NULL);
}
return sub;
}
static mi_memid_t mi_page_map_submap_memid(mi_page_submap_t* sub) {
const size_t guard_size = _mi_os_secure_guard_page_size();
const size_t size = _mi_os_good_alloc_size(MI_PAGE_MAP_SUB_SIZE + 2*guard_size);
return _mi_memid_create_os((uint8_t*)mi_ptr_memtag_meta(sub) - guard_size, size, size, false, false);
}
void _mi_page_map_unsafe_destroy(void) {
mi_page_map_t* const pmap = mi_page_map();
mi_assert_internal(pmap != NULL);
if (pmap == NULL) return;
mi_subproc_t* const subproc = _mi_subproc_main();
mi_lock_done(&pmap->lock);
const size_t committed_count = mi_page_map_committed();
mi_page_map_store_committed(0); for (size_t idx = 1; idx < committed_count; idx++) { mi_page_submap_t* sub = mi_page_map_at(pmap,idx);
if (sub != NULL) {
_mi_os_free_with_guards(subproc, mi_page_map_submap_memid(sub));
mi_page_map_set_at(pmap, idx, NULL);
}
}
_mi_os_free_with_guards(subproc, pmap->memid);
}
mi_decl_nodiscard static mi_decl_noinline mi_page_submap_t* mi_page_map_alloc_submap_at(mi_page_map_t* pmap, size_t idx) {
mi_page_submap_t* sub = NULL;
mi_lock(&pmap->lock)
{
sub = mi_page_map_at(pmap, idx); if (sub==NULL) {
mi_subproc_t* const subproc = _mi_subproc_main();
mi_memid_t memid;
sub = mi_page_map_alloc_submap(subproc, &memid);
if (sub==NULL) {
_mi_warning_message("internal error: unable to extend the page map\n");
}
else {
mi_page_map_set_at(pmap, idx, sub); }
}
}
return sub;
}
mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(mi_page_map_t* pmap, size_t idx, mi_page_submap_t** submap) {
mi_assert_internal(submap!=NULL && *submap==NULL);
mi_page_submap_t* sub = NULL;
if (!mi_page_map_ensure_committed(pmap, idx, &sub)) {
return false;
}
if mi_unlikely(sub==NULL) {
sub = mi_page_map_alloc_submap_at(pmap, idx);
if (sub==NULL) return false; }
mi_assert_internal(sub!=NULL);
*submap = sub;
return true;
}
static bool mi_page_map_set_range_prim(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
while (slice_count > 0) {
mi_page_submap_t* sub = NULL;
if (!mi_page_map_ensure_submap_at(pmap, idx, &sub)) {
return false;
};
mi_assert_internal(sub!=NULL);
while (slice_count > 0 && sub_idx < MI_PAGE_MAP_SUB_COUNT) {
mi_atomic_ptr_sign_relaxed(&sub->page_metas[sub_idx], page);
slice_count--;
sub_idx++;
}
idx++; sub_idx = 0;
}
return true;
}
static bool mi_page_map_set_range(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) {
if mi_unlikely(!mi_page_map_set_range_prim(pmap, page,idx,sub_idx,slice_count)) {
if (page!=NULL) {
mi_page_map_set_range_prim(pmap,NULL,idx,sub_idx,slice_count);
}
return false;
}
return true;
}
static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slice_count) {
size_t page_size;
uint8_t* page_start = mi_page_area(page, &page_size);
if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } mi_assert_internal(page_size > 0);
const mi_addr_t start = mi_addr(page_start);
*slice_count = ((start + page_size - 1) / MI_ARENA_SLICE_SIZE) - (start / MI_ARENA_SLICE_SIZE) + 1;
return mi_page_map_index(page_start, sub_idx);
}
bool _mi_page_map_register(mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_page_map_t* pmap = mi_page_map();
mi_assert_internal(pmap != NULL); if mi_unlikely(pmap == NULL) {
if (!_mi_page_map_init()) return false;
pmap = mi_page_map();
}
mi_assert(pmap!=NULL);
size_t slice_count;
size_t sub_idx;
const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
return mi_page_map_set_range(pmap, page, idx, sub_idx, slice_count);
}
void _mi_page_map_unregister(mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
mi_page_map_t* const pmap = mi_page_map();
if mi_unlikely(pmap == NULL) return;
size_t slice_count;
size_t sub_idx;
const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count);
mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count);
}
void _mi_page_map_unregister_range(void* start, size_t size) {
mi_page_map_t* const pmap = mi_page_map();
if mi_unlikely(pmap == NULL) return;
const size_t slice_count = mi_divide_up(size, MI_ARENA_SLICE_SIZE);
size_t sub_idx;
const size_t idx = mi_page_map_index(start, &sub_idx);
mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); }
mi_page_t* _mi_safe_ptr_page(const void* p) {
return mi_checked_ptr_page(p);
}
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
return (_mi_safe_ptr_page(p) != NULL);
}