#include "mimalloc.h"
#include "mimalloc/prim.h"
#include "mimalloc/track.h"
#include "mimalloc/prim/prim-tls.h"
#include "mimalloc/util/libc.h"
#include "mimalloc/util/options.h"
#include "mimalloc/util/stats.h"
#include "mimalloc/prim/os.h"
#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB
#if MI_SIZE_SIZE < 8
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB
#else
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB
#endif
#endif
static const mi_os_mem_config_t mi_os_mem_config_default = {
4096, 0, 4096, MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB,
MI_MAX_VABITS, true, false, true, false };
static inline const mi_os_mem_config_t* mi_os_mem_config(void) {
const mi_os_mem_config_t* const config = &_mi_process.os_mem_config;
return (mi_likely(config->page_size != 0) ? config : &mi_os_mem_config_default);
}
bool _mi_os_has_overcommit(void) {
return mi_os_mem_config()->has_overcommit;
}
bool _mi_os_has_virtual_reserve(void) {
return mi_os_mem_config()->has_virtual_reserve;
}
bool _mi_os_has_thp(void) {
return mi_os_mem_config()->has_transparent_huge_pages;
}
size_t _mi_os_page_size(void) {
return mi_os_mem_config()->page_size;
}
size_t _mi_os_large_page_size(void) {
const size_t lpsize = mi_os_mem_config()->large_page_size;
return (lpsize != 0 ? lpsize : _mi_os_page_size());
}
size_t _mi_os_minimal_purge_size(void) {
size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);
if (minsize != 0) {
return mi_align_up(minsize, _mi_os_page_size());
}
else if (mi_os_mem_config()->has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {
return _mi_os_large_page_size();
}
else {
return _mi_os_page_size();
}
}
size_t _mi_os_guard_page_size(void) {
const size_t gsize = _mi_os_page_size();
mi_assert(gsize <= (MI_ARENA_SLICE_SIZE/4)); return gsize;
}
size_t _mi_os_virtual_address_bits(void) {
const size_t vbits = mi_os_mem_config()->virtual_address_bits;
mi_assert(vbits <= MI_MAX_VABITS);
return vbits;
}
bool _mi_os_canuse_large_page(size_t size, size_t alignment) {
const size_t lpsize = mi_os_mem_config()->large_page_size;
if (lpsize == 0) return false;
return ((size % lpsize) == 0 && (alignment % lpsize) == 0);
}
size_t _mi_os_good_alloc_size(size_t size) {
size_t align_size;
if (size < 512*MI_KiB) align_size = _mi_os_page_size();
else if (size < 2*MI_MiB) align_size = 64*MI_KiB;
else if (size < 8*MI_MiB) align_size = 256*MI_KiB;
else if (size < 32*MI_MiB) align_size = 1*MI_MiB;
else align_size = 4*MI_MiB;
if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; return mi_align_up(size, align_size);
}
void _mi_os_init(void) {
mi_os_mem_config_t config = mi_os_mem_config_default;
_mi_prim_mem_init(&config);
mi_assert(config.page_size != 0);
_mi_process.os_mem_config = config; if (_mi_os_page_size() > MI_ARENA_SLICE_SIZE) {
_mi_error_message(ENOTSUP, "the OS page size (%zu KiB) is larger than the arena slice size (%zu KiB); increase MI_ARENA_SLICE_SHIFT\n", _mi_os_page_size()/MI_KiB, (size_t)(MI_ARENA_SLICE_SIZE/MI_KiB));
}
}
bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size);
bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero);
#if (MI_SIZE_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT)
#define MI_HINT_BASE (MI_AU(2) << 40)
#define MI_HINT_AREA (MI_AU(4) << 40)
#define MI_HINT_MAX (MI_AU(30) << 40)
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze)
{
if (try_alignment <= mi_os_mem_config()->alloc_granularity || try_alignment > 16*MI_GiB) return NULL;
if (mi_os_mem_config()->virtual_address_bits < 46) return NULL;
size_t req_size = sze + _mi_os_page_size(); req_size += (try_alignment - 1); req_size = mi_align_up(req_size, _mi_os_large_page_size());
#if MI_SECURE
if (req_size > 32*MI_GiB) return NULL; #endif
mi_addr_t hint = mi_atomic_add_acq_rel(&_mi_process.os_aligned_base, req_size);
if (hint == 0 || hint > MI_HINT_MAX) { mi_addr_t init = MI_HINT_BASE;
#if (MI_SECURE || !MI_DEBUG)
mi_theap_t* const theap = _mi_theap_default(); if (!mi_theap_is_initialized(theap)) return NULL; const size_t r = _mi_theap_random_next(theap);
init = init + ((MI_AU(MI_MiB) * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); #endif
mi_addr_t expected = hint + req_size;
mi_atomic_cas_strong_acq_rel(&_mi_process.os_aligned_base, &expected, init);
hint = mi_atomic_add_acq_rel(&_mi_process.os_aligned_base, req_size); if (hint==0) return NULL;
}
const mi_addr_t hint_align = mi_addr_align_up(hint,try_alignment);
mi_assert_internal(hint_align + sze < hint + req_size);
return mi_ptr_from_addr(hint_align);
}
#else
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
MI_UNUSED(try_alignment); MI_UNUSED(size);
return NULL;
}
#endif
size_t _mi_os_secure_guard_page_size(void) {
#if MI_SECURE && MI_OS_HAS_GUARD_PAGES
return _mi_os_guard_page_size();
#else
return 0;
#endif
}
static bool mi_os_secure_guard_page(void* addr, mi_memid_t memid, bool set) {
const size_t size = _mi_os_secure_guard_page_size();
if (addr == NULL || size == 0 || memid.is_pinned) return true;
mi_assert_internal(mi_is_aligned(addr, _mi_os_page_size()));
bool ok;
mi_arena_t* const arena = mi_memid_arena(memid);
if (arena != NULL && arena->commit_fun != NULL) {
ok = (set || memid.initially_committed ? (*(arena->commit_fun))(!set , addr, size, NULL, arena->commit_fun_arg) : true);
}
else {
ok = (set ? _mi_os_guard(addr, size, memid.initially_committed) : _mi_os_unguard(addr, size, memid.initially_committed));
}
if (!ok) {
_mi_error_message(EINVAL, "secure level %d, but failed to %s guard page (at %p of size %zu)\n", MI_SECURE, (set ? "set" : "reset"), addr, size);
}
return ok;
}
bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
MI_UNUSED(subproc);
return mi_os_secure_guard_page(addr, memid, true);
}
bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
return _mi_os_secure_guard_page_set_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
}
bool _mi_os_secure_guard_pages_set_around(mi_subproc_t* subproc, void* addr, size_t size, mi_memid_t memid) {
const bool ok_before = _mi_os_secure_guard_page_set_before(subproc, addr, memid);
const bool ok_after = _mi_os_secure_guard_page_set_at(subproc, mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()), memid);
return (ok_before && ok_after);
}
bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
MI_UNUSED(subproc);
return mi_os_secure_guard_page(addr, memid, false);
}
bool _mi_os_secure_guard_pages_set_within(void* area, size_t area_size, size_t offset, size_t size) {
const size_t gsize = _mi_os_secure_guard_page_size();
if (gsize == 0) return true;
const mi_addr_t start = mi_addr(area);
const mi_addr_t before = mi_addr_align_down(start + offset, _mi_os_page_size()) - gsize;
const mi_addr_t after = mi_addr_align_up(start + offset + size, _mi_os_page_size());
if (before < start || after + gsize > start + area_size) {
_mi_warning_message("secure level %d, but unable to set guard pages around static data at %p as the OS page size (%zu KiB) is too large\n", MI_SECURE, area, gsize / MI_KiB);
return false;
}
mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); memid.initially_committed = true;
const bool ok_before = mi_os_secure_guard_page(mi_ptr_with_addr(area, before), memid, true);
const bool ok_after = mi_os_secure_guard_page(mi_ptr_with_addr(area, after), memid, true);
return (ok_before && ok_after);
}
static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size);
static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) {
mi_assert_internal(subproc!=NULL);
mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL) return; int err = _mi_prim_free(addr, size); if (err != 0) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
}
if (adjust) {
if (commit_size>0) { mi_subproc_stat_adjust_decrease(subproc, committed, commit_size); }
mi_subproc_stat_adjust_decrease(subproc, reserved, size);
}
else {
if (commit_size>0) { mi_subproc_stat_decrease(subproc, committed, commit_size); }
mi_subproc_stat_decrease(subproc, reserved, size);
}
}
void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) {
if (mi_memkind_is_os(memid.memkind)) {
size_t csize = memid.mem.os.size;
if (csize==0) { csize = _mi_os_good_alloc_size(size); }
mi_assert_internal(csize >= size);
size_t commit_size = (still_committed ? csize : 0);
void* base = addr;
if (memid.mem.os.base != base) {
mi_assert(memid.mem.os.base <= addr);
base = memid.mem.os.base;
const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;
if (memid.mem.os.size==0) {
csize += diff;
}
if (still_committed) {
commit_size -= diff; }
}
if (memid.memkind == MI_MEM_OS_HUGE) {
mi_assert(memid.is_pinned);
mi_os_free_huge_os_pages(subproc, base, csize);
}
else {
mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0), false);
}
}
else {
mi_assert(memid.memkind < MI_MEM_OS);
}
}
void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) {
_mi_os_free_ex(subproc, p, size, true, memid);
}
static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(is_zero != NULL);
mi_assert_internal(is_large != NULL);
if (size == 0) return NULL;
if (!commit) { allow_large = false; }
if (try_alignment == 0) { try_alignment = 1; }
const size_t large_page_size = mi_os_mem_config()->large_page_size;
if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) {
try_alignment = large_page_size;
}
*is_zero = false;
void* p = NULL;
int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);
if (err != 0) {
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
}
mi_subproc_stat_counter_increase(subproc, mmap_calls, 1);
if (p != NULL) {
mi_subproc_stat_increase(subproc, reserved, size);
if (commit) {
mi_subproc_stat_increase(subproc, committed, size);
#ifdef MI_TRACK_ASAN
if (*is_zero) { mi_track_mem_defined(p,size); }
else { mi_track_mem_undefined(p,size); }
#endif
}
}
return p;
}
static void* mi_os_prim_alloc(mi_subproc_t* subproc, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
return mi_os_prim_alloc_at(subproc, NULL , size, try_alignment, commit, allow_large, is_large, is_zero);
}
static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert_internal(memid!=NULL);
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
*memid = _mi_memid_none();
if (!commit) allow_large = false;
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = mi_align_up(size, _mi_os_page_size());
#if MI_SIZE_SIZE >= 8
const bool try_direct_alloc = true;
#else
const bool try_direct_alloc = (alignment <= mi_os_mem_config()->alloc_granularity || alignment <= size/4);
#endif
bool os_is_large = false;
bool os_is_zero = false;
void* os_base = NULL;
size_t os_size = size;
void* p = NULL;
if (try_direct_alloc) {
p = mi_os_prim_alloc(subproc, size, alignment, commit, allow_large, &os_is_large, &os_is_zero);
}
if (p != NULL && mi_is_aligned(p,alignment)) {
os_base = p;
}
else {
#if !MI_TRACK_ASAN && !defined(MI_NO_ALIGN_FALLBACK_WARNING)
if (try_direct_alloc) {
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
}
#endif
if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true ); }
if (size >= (SIZE_MAX - alignment)) return NULL; const size_t over_size = size + alignment;
if (!mi_os_mem_config()->has_partial_free) { p = mi_os_prim_alloc(subproc, over_size, 1 , false , false , &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
os_base = p; os_size = over_size; p = mi_align_up_ptr(p, alignment);
if (commit) {
if (!_mi_os_commit(subproc, p, size, NULL)) {
mi_os_prim_free(subproc, os_base, over_size, 0, true);
return NULL;
}
}
}
else { p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large , &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
void* const aligned_p = mi_align_up_ptr(p, alignment);
const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
const size_t mid_size = mi_align_up(size, _mi_os_page_size());
const size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true ); }
if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true ); }
p = aligned_p;
os_base = aligned_p; os_size = mid_size;
}
}
mi_assert_internal(p != NULL && os_base != NULL && mi_is_aligned(p,alignment));
mi_assert_internal(os_base <= p && size <= os_size);
*memid = _mi_memid_create_os(os_base,os_size,commit,os_is_zero,os_is_large);
return p;
}
void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (size == 0) return NULL;
size = _mi_os_good_alloc_size(size);
bool os_is_large = false;
bool os_is_zero = false;
void* p = mi_os_prim_alloc(subproc, size, 0, true, false, &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
*memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large);
mi_assert_internal(memid->mem.os.size >= size);
mi_assert_internal(memid->initially_committed);
return p;
}
void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
{
MI_UNUSED(&_mi_os_get_aligned_hint); *memid = _mi_memid_none();
if (size == 0) return NULL;
size = _mi_os_good_alloc_size(size);
alignment = mi_align_up(alignment, _mi_os_page_size());
void* p = mi_os_prim_alloc_aligned(subproc, size, alignment, commit, allow_large, memid );
if (p == NULL) return NULL;
mi_assert_internal(memid->mem.os.size >= size);
mi_assert_internal(mi_is_aligned(p,alignment));
if (commit) { mi_assert_internal(memid->initially_committed); }
return p;
}
mi_decl_nodiscard static void* mi_os_ensure_zero(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t* memid) {
if (p==NULL || size==0) return p;
if (!memid->initially_committed) {
bool is_zero = false;
if (!_mi_os_commit(subproc, p, size, &is_zero)) {
_mi_os_free(subproc, p, size, *memid);
return NULL;
}
memid->initially_committed = true;
}
if (memid->initially_zero) return p;
mi_memzero_aligned(p,size);
memid->initially_zero = true;
return p;
}
void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {
void* p = _mi_os_alloc(subproc, size,memid);
return mi_os_ensure_zero(subproc, p, size, memid);
}
void* _mi_os_alloc_with_guards(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, mi_memid_t* memid) {
const size_t guard_size = _mi_os_secure_guard_page_size();
size = mi_align_up(size, _mi_os_page_size());
uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, size + 2*guard_size, 1, commit, allow_large, memid);
if (base == NULL) return NULL;
uint8_t* const p = base + guard_size;
if (memid->initially_committed) { _mi_os_secure_guard_pages_set_around(subproc, p, size, *memid);
}
return p;
}
void _mi_os_free_with_guards(mi_subproc_t* subproc, mi_memid_t memid) {
_mi_os_free(subproc, memid.mem.os.base, memid.mem.os.size, memid);
}
void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert(offset <= size);
mi_assert((alignment % _mi_os_page_size()) == 0);
*memid = _mi_memid_none();
if (offset > size) return NULL;
if (offset == 0) {
return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid);
}
else {
const size_t extra = mi_align_up(offset, alignment) - offset;
if (size >= SIZE_MAX - extra) return NULL; const size_t oversize = size + extra;
void* const start = _mi_os_alloc_aligned(subproc, oversize, alignment, commit, allow_large, memid);
if (start == NULL) return NULL;
void* const p = (uint8_t*)start + extra;
mi_assert(mi_is_aligned((uint8_t*)p + offset, alignment));
if (commit && extra >= _mi_os_page_size()) {
_mi_os_decommit(subproc, start, extra);
}
return p;
}
}
static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
mi_assert(addr != NULL && size > 0);
if (newsize != NULL) *newsize = 0;
if (size == 0 || addr == NULL) return NULL;
void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size())
: mi_align_down_ptr(addr, _mi_os_page_size()));
void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
: mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
if (diff <= 0) return NULL;
mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));
if (newsize != NULL) *newsize = (size_t)diff;
return start;
}
static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
return mi_os_page_align_areax(true, addr, size, newsize);
}
bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) {
mi_assert_internal(size >= stat_already_committed);
if (is_zero != NULL) { *is_zero = false; }
mi_subproc_stat_counter_increase(subproc, commit_calls, 1);
size_t csize;
void* start = mi_os_page_align_areax(false , addr, size, &csize);
if (csize == 0) return true;
bool os_is_zero = false;
int err = _mi_prim_commit(start, csize, &os_is_zero);
if (err != 0) {
_mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
return false;
}
if (os_is_zero && is_zero != NULL) {
*is_zero = true;
mi_assert_expensive(mi_mem_is_zero(start, csize));
}
#ifdef MI_TRACK_ASAN
if (os_is_zero) { mi_track_mem_defined(start,csize); }
else { mi_track_mem_undefined(start,csize); }
#endif
mi_subproc_stat_increase(subproc, committed, size - stat_already_committed); return true;
}
bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) {
return _mi_os_commit_ex(subproc, addr, size, is_zero, 0);
}
static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
mi_assert_internal(needs_recommit!=NULL);
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true;
*needs_recommit = true;
int err = _mi_prim_decommit(start,csize,needs_recommit);
if (err != 0) {
_mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
else if (*needs_recommit) {
mi_subproc_stat_decrease(subproc, committed, stat_size);
}
mi_assert_internal(err == 0);
return (err == 0);
}
bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size) {
bool needs_recommit;
return mi_os_decommit_ex(subproc, addr, size, &needs_recommit, size);
}
bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size) {
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true; mi_subproc_stat_counter_increase(subproc, reset, csize);
mi_subproc_stat_counter_increase(subproc, reset_calls, 1);
#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED
memset(start, 0, csize); #endif
int err = _mi_prim_reset(start, csize);
if (err != 0) {
_mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
return (err == 0);
}
void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) {
MI_UNUSED(subproc);
size_t csize = 0;
void* const start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return;
const int err = _mi_prim_reuse(start, csize);
if (err != 0) {
_mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
}
bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)
{
if (mi_option_get(mi_option_purge_delay) < 0) return false; mi_subproc_stat_counter_increase(subproc, purge_calls, 1);
mi_subproc_stat_counter_increase(subproc, purged, size);
if (commit_fun != NULL) {
bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);
return decommitted; }
else if (mi_option_is_enabled(mi_option_purge_decommits) && !_mi_preloading()) {
bool needs_recommit = true;
mi_os_decommit_ex(subproc, p, size, &needs_recommit, stat_size);
return needs_recommit;
}
else {
if (allow_reset) { _mi_os_reset(subproc, p, size);
}
return false; }
}
bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size) {
return _mi_os_purge_ex(subproc, p, size, true, size, NULL, NULL);
}
static bool mi_os_guardx(void* addr, size_t size, bool set, bool is_committed) {
size_t csize = 0;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return false;
int err = _mi_prim_guard(start, csize, set, is_committed);
if (err != 0) {
_mi_warning_message("cannot %s guard OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (set ? "set" : "reset"), err, err, start, csize);
}
return (err == 0);
}
bool _mi_os_guard(void* addr, size_t size, bool is_committed) {
return mi_os_guardx(addr, size, true, is_committed);
}
bool _mi_os_unguard(void* addr, size_t size, bool is_committed) {
return mi_os_guardx(addr, size, false, is_committed);
}
#define MI_HUGE_OS_PAGE_SIZE (MI_GiB)
#if (MI_SIZE_SIZE >= 8) && !MI_CHERI
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
if (total_size != NULL) *total_size = 0;
size_t size = 0;
if (mi_mul_overflow(pages,MI_HUGE_OS_PAGE_SIZE,&size)) {
_mi_warning_message("too many huge pages requested: %zu\n", pages);
return NULL;
}
mi_addr_t start = 0;
mi_addr_t end = 0;
mi_addr_t huge_start = mi_atomic_load_relaxed(&_mi_process.os_huge_start);
do {
start = huge_start;
if (start == 0) {
start = (MI_AU(32) << 40); #if (MI_SECURE || MI_DEBUG==0)
mi_theap_t* const theap = _mi_theap_default(); if (mi_theap_is_initialized(theap)) { const size_t r = _mi_theap_random_next(theap);
start = start + (MI_AU(MI_HUGE_OS_PAGE_SIZE) * ((r>>17) & 0x0FFF)); }
else {
_mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, mi_ptr_from_addr(start));
}
#endif
}
end = start + size;
} while (!mi_atomic_cas_weak_acq_rel(&_mi_process.os_huge_start, &huge_start, end));
if (total_size != NULL) *total_size = size;
return (uint8_t*)mi_ptr_from_addr(start);
}
#else
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
MI_UNUSED(pages);
if (total_size != NULL) *total_size = 0;
return NULL;
}
#endif
void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (psize != NULL) *psize = 0;
if (pages_reserved != NULL) *pages_reserved = 0;
size_t size = 0;
uint8_t* const start = mi_os_claim_huge_pages(pages, &size);
if (start == NULL) return NULL;
mi_msecs_t start_t = _mi_clock_start();
size_t page = 0;
bool all_zero = true;
while (page < pages) {
bool is_zero = false;
void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);
void* p = NULL;
int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);
if (!is_zero) { all_zero = false; }
if (err != 0) {
_mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE);
break;
}
if (p != addr) {
if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, true );
}
break;
}
page++; mi_subproc_stat_increase(subproc, committed, MI_HUGE_OS_PAGE_SIZE);
mi_subproc_stat_increase(subproc, reserved, MI_HUGE_OS_PAGE_SIZE);
if (max_msecs > 0) {
mi_msecs_t elapsed = _mi_clock_end(start_t);
if (page >= 1) {
mi_msecs_t estimate = ((elapsed / (page==0 ? 1 : page)) * pages);
if (estimate > 2*max_msecs) { elapsed = max_msecs + 1;
}
}
if (elapsed > max_msecs) {
_mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page);
break;
}
}
}
const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;
mi_assert_internal(allocated <= size);
if (pages_reserved != NULL) { *pages_reserved = page; }
if (psize != NULL) { *psize = allocated; }
if (page != 0) {
mi_assert(start != NULL);
*memid = _mi_memid_create_os(start, allocated, true , all_zero, true );
memid->memkind = MI_MEM_OS_HUGE;
mi_assert(memid->is_pinned);
#ifdef MI_TRACK_ASAN
if (all_zero) { mi_track_mem_defined(start,allocated); }
#endif
}
return (page == 0 ? NULL : start);
}
static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size) {
if (p==NULL || size==0) return;
uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) {
mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, false );
size -= MI_HUGE_OS_PAGE_SIZE;
base += MI_HUGE_OS_PAGE_SIZE;
}
}
int _mi_os_numa_node_count(void) {
size_t count = mi_atomic_load_acquire(&_mi_process.os_numa_node_count);
if mi_unlikely(count == 0) {
long ncount = mi_option_get(mi_option_use_numa_nodes); if (ncount > 0 && ncount < INT_MAX) {
count = (size_t)ncount;
}
else {
const size_t n = _mi_prim_numa_node_count(); if (n == 0 || n > INT_MAX) { count = 1; }
else { count = n; }
}
mi_atomic_store_release(&_mi_process.os_numa_node_count, count); if (count>1) { _mi_verbose_message("using %zd numa regions\n", count); }
}
mi_assert_internal(count > 0 && count <= INT_MAX);
return (int)count;
}
static int mi_os_numa_node_get(void) {
int numa_count = _mi_os_numa_node_count();
if (numa_count<=1) return 0; const size_t n = _mi_prim_numa_node();
int numa_node = (n < INT_MAX ? (int)n : 0);
if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }
return numa_node;
}
int _mi_os_numa_node(void) {
if mi_likely(mi_atomic_load_relaxed(&_mi_process.os_numa_node_count) == 1) {
return 0;
}
else {
return mi_os_numa_node_get();
}
}
#if 0#endif