#include "mimalloc.h"
#include "mimalloc/prim/prim.h"
#include "mimalloc/track.h"
#include "mimalloc/prim/prim-tls.h"
#include "mimalloc/util/libc.h"
#include "mimalloc/util/options.h"
#include "mimalloc/util/stats.h"
#include "mimalloc/util/memid.h"
#include "mimalloc/prim/os.h"
#include "mimalloc/util/random.h"
#include "mimalloc/prim/prim-pac.h"
#include "mimalloc/arena/arena.h"
#if !defined(_WIN32) && MI_HAS_UNISTDH
#include <unistd.h>
#endif
#ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB
#if MI_SIZE_SIZE < 8
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB
#else
#define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB
#endif
#endif
static const mi_os_mem_config_t mi_os_mem_config_default = {
4096, 0, 4096, MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB,
MI_MAX_VABITS, 0, 0, true, false, true, true, false, false, false, false, false };
static mi_stats_t mi_os_stats_bootstrap;
static inline mi_stats_t* mi_os_stats(mi_subproc_t* subproc) {
return (subproc == NULL ? &mi_os_stats_bootstrap : &subproc->stats);
}
#define mi_os_stat_counter_increase(subproc,stat,amount) __mi_stat_counter_increase_mt( &mi_os_stats(subproc)->stat, amount)
#define mi_os_stat_increase(subproc,stat,amount) __mi_stat_increase_mt( &mi_os_stats(subproc)->stat, amount)
#define mi_os_stat_decrease(subproc,stat,amount) __mi_stat_decrease_mt( &mi_os_stats(subproc)->stat, amount)
#define mi_os_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &mi_os_stats(subproc)->stat, amount)
void _mi_os_stats_bootstrap_merge(mi_subproc_t* subproc) {
_mi_stats_merge_into(&subproc->stats, &mi_os_stats_bootstrap);
}
static inline const mi_os_mem_config_t* mi_os_mem_config(void) {
const mi_os_mem_config_t* const config = &mi_process_get_sealed_once()->os_mem_config;
return (config->page_size != 0 ? config : &mi_os_mem_config_default);
}
bool _mi_os_has_overcommit(void) {
return mi_os_mem_config()->has_overcommit;
}
bool _mi_os_has_virtual_reserve(void) {
return mi_os_mem_config()->has_virtual_reserve;
}
bool _mi_os_has_reset(void) {
return mi_os_mem_config()->has_reset;
}
bool _mi_os_has_thp(void) {
return mi_os_mem_config()->has_transparent_huge_pages;
}
bool _mi_os_has_memtag(void) {
return mi_os_mem_config()->has_memtag;
}
size_t _mi_os_memtag_line_size(void) {
return mi_os_mem_config()->memtag_line_size;
}
static bool mi_os_detect_elevated(void) {
#if defined(_WIN32)
return false; #elif MI_HAS_UNISTDH && (defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__))
return (issetugid() != 0);
#elif MI_HAS_UNISTDH
return (getuid() != geteuid() || getgid() != getegid());
#else
return false; #endif
}
bool _mi_os_is_elevated(void) {
const mi_os_mem_config_t* const config = &mi_process_get_sealed_once()->os_mem_config;
return (config->page_size != 0 ? config->is_elevated : mi_os_detect_elevated());
}
size_t _mi_os_page_size(void) {
return mi_os_mem_config()->page_size;
}
size_t _mi_os_alloc_granularity(void) {
return mi_os_mem_config()->alloc_granularity;
}
size_t _mi_os_large_page_size(void) {
const size_t lpsize = mi_os_mem_config()->large_page_size;
return (lpsize != 0 ? lpsize : _mi_os_page_size());
}
size_t _mi_os_minimal_purge_size(void) {
size_t minsize = mi_option_get_size(mi_option_minimal_purge_size);
if (minsize != 0) {
return mi_align_up(minsize, _mi_os_page_size());
}
else if (mi_os_mem_config()->has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) {
return _mi_os_large_page_size();
}
else {
return _mi_os_page_size();
}
}
size_t _mi_os_guard_page_size(void) {
const size_t gsize = _mi_os_page_size();
mi_assert(gsize <= (MI_ARENA_SLICE_SIZE/4)); return gsize;
}
size_t _mi_os_virtual_address_bits(void) {
const size_t vbits = mi_os_mem_config()->virtual_address_bits;
mi_assert(vbits <= MI_MAX_VABITS);
return vbits;
}
bool _mi_os_canuse_large_page(size_t size, size_t alignment) {
const size_t lpsize = mi_os_mem_config()->large_page_size;
if (lpsize == 0) return false;
return ((size % lpsize) == 0 && (alignment % lpsize) == 0);
}
size_t _mi_os_good_alloc_size(size_t size) {
size_t align_size;
if (size < 512*MI_KiB) align_size = _mi_os_page_size();
else if (size < 2*MI_MiB) align_size = 64*MI_KiB;
else if (size < 8*MI_MiB) align_size = 256*MI_KiB;
else if (size < 32*MI_MiB) align_size = 1*MI_MiB;
else align_size = 4*MI_MiB;
if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; return mi_align_up(size, align_size);
}
static mi_addr_t mi_random_key_new(void) {
mi_random_ctx_t ctx;
_mi_random_init(&ctx);
const mi_addr_t key = (mi_addr_t)_mi_random_next(&ctx);
mi_memzero(&ctx, sizeof(ctx));
return key;
}
void _mi_os_init(void) {
mi_os_mem_config_t config = mi_os_mem_config_default;
_mi_prim_mem_init(&config);
mi_assert(config.page_size != 0);
config.is_elevated = mi_os_detect_elevated();
if (config.has_memtag) {
const size_t line_size = mi_memtag_cpu_line_size();
if (line_size < MI_MEMTAG_LINE_MIN || line_size > MI_MEMTAG_LINE_MAX || !mi_is_power_of_two(line_size)) {
_mi_warning_message("memory tagging is disabled as the CPU tag cache line size (%zu bytes) is not supported\n", line_size);
config.has_memtag = false;
}
else {
config.memtag_line_size = line_size;
}
}
#if MI_MTE >= 2
if (!config.has_memtag) {
_mi_fatal_error("built with MI_MTE=ON but memory tagging is not enabled at runtime\n"
" (on macOS the program needs the MTE entitlement, and the `mte` option must not be disabled)\n");
}
#endif
#if MI_PAC >= 2
if (!config.has_pac) {
_mi_fatal_error("built with MI_PAC=ON but the CPU (or OS) does not support pointer authentication (PAC)\n");
}
#endif
mi_process_sealed_once_t* const once = _mi_process_unseal_once();
once->os_mem_config = config; if (once->process_key == 0) { #if MI_PAC == 1
once->process_key = mi_random_key_new() | MI_PAC_KEY_ALWAYS;
#else
once->process_key = mi_random_key_new() | 1; #endif
}
_mi_process_seal_once();
if (_mi_os_page_size() > MI_ARENA_SLICE_SIZE) {
_mi_error_message(ENOTSUP, "the OS page size (%zu KiB) is larger than the arena slice size (%zu KiB); increase MI_ARENA_SLICE_SHIFT\n", _mi_os_page_size()/MI_KiB, (size_t)(MI_ARENA_SLICE_SIZE/MI_KiB));
}
if (config.has_memtag) { _mi_verbose_message("memory tagging is enabled (for blocks up to %zu KiB)\n", (size_t)(MI_LARGE_MAX_OBJ_SIZE / MI_KiB)); }
#if MI_PAC
_mi_verbose_message("signing is enabled (%s)\n", (config.has_pac && MI_PAC>=2 ? "hardware supported" : "weak, using a process key"));
#endif
}
bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size);
bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero);
#if (MI_SIZE_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT)
#define MI_HINT_BASE (MI_ADDR(2) << 40)
#define MI_HINT_AREA (MI_ADDR(4) << 40)
#define MI_HINT_MAX (MI_ADDR(30) << 40)
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze)
{
if (try_alignment <= mi_os_mem_config()->alloc_granularity || try_alignment > 16*MI_GiB) return NULL;
if (mi_os_mem_config()->virtual_address_bits < 46) return NULL;
size_t req_size = sze + _mi_os_page_size(); req_size += (try_alignment - 1); req_size = mi_align_up(req_size, _mi_os_large_page_size());
#if MI_SECURE
if (req_size > 32*MI_GiB) return NULL; #endif
mi_addr_t base = mi_atomic_load_relaxed(&_mi_process.os.aligned_base);
if (base == 0 || base > MI_HINT_MAX) {
mi_addr_t init = MI_HINT_BASE;
#if (MI_SECURE || !MI_DEBUG)
mi_theap_t* const theap = _mi_theap_default(); if (!mi_theap_is_initialized(theap)) return NULL; const size_t r = _mi_theap_random_next(theap);
init = init + ((MI_ADDR(MI_MiB) * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); #endif
mi_atomic_cas_strong_relaxed(&_mi_process.os.aligned_base, &base, init); }
const mi_addr_t hint = mi_atomic_add_relaxed(&_mi_process.os.aligned_base, req_size);
if (hint == 0 || hint > MI_HINT_MAX) return NULL; const mi_addr_t hint_align = mi_addr_align_up(hint,try_alignment);
mi_assert_internal(hint_align + sze < hint + req_size);
return mi_ptr_from_addr(hint_align);
}
#else
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
MI_UNUSED(try_alignment); MI_UNUSED(size);
return NULL;
}
#endif
size_t _mi_os_secure_guard_page_size(void) {
#if MI_SECURE && MI_OS_HAS_GUARD_PAGES
return _mi_os_guard_page_size();
#else
return 0;
#endif
}
static bool mi_os_secure_guard_page(void* addr, mi_memid_t memid, bool set) {
const size_t size = _mi_os_secure_guard_page_size();
if (addr == NULL || size == 0 || memid.is_pinned) return true;
mi_assert_internal(mi_is_aligned(addr, _mi_os_page_size()));
bool ok;
mi_arena_t* const arena = mi_memid_arena(memid);
mi_commit_fun_t* const commit_fun = (arena != NULL ? mi_fun_auth_at(mi_commit_fun_t, &arena->commit_fun) : NULL);
if (commit_fun != NULL) {
ok = (set || memid.initially_committed ? (*commit_fun)(!set , addr, size, NULL, arena->commit_fun_arg) : true);
}
else {
ok = (set ? _mi_os_guard(addr, size, memid.initially_committed) : _mi_os_unguard(addr, size, memid.initially_committed));
}
if (!ok) {
_mi_error_message(EINVAL, "secure level %d, but failed to %s guard page (at %p of size %zu)\n", MI_SECURE, (set ? "set" : "reset"), addr, size);
}
return ok;
}
bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
MI_UNUSED(subproc);
return mi_os_secure_guard_page(addr, memid, true);
}
bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
return _mi_os_secure_guard_page_set_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid);
}
bool _mi_os_secure_guard_pages_set_around(mi_subproc_t* subproc, void* addr, size_t size, mi_memid_t memid) {
const bool ok_before = _mi_os_secure_guard_page_set_before(subproc, addr, memid);
const bool ok_after = _mi_os_secure_guard_page_set_at(subproc, mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()), memid);
return (ok_before && ok_after);
}
bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) {
MI_UNUSED(subproc);
return mi_os_secure_guard_page(addr, memid, false);
}
bool _mi_os_secure_guard_pages_set_within(void* area, size_t area_size, size_t offset, size_t size) {
const size_t gsize = _mi_os_secure_guard_page_size();
if (gsize == 0) return true;
const mi_addr_t start = mi_addr(area);
const mi_addr_t before = mi_addr_align_down(start + offset, _mi_os_page_size()) - gsize;
const mi_addr_t after = mi_addr_align_up(start + offset + size, _mi_os_page_size());
if (before < start || after + gsize > start + area_size) {
_mi_warning_message("secure level %d, but unable to set guard pages around static data at %p as the OS page size (%zu KiB) is too large\n", MI_SECURE, area, gsize / MI_KiB);
return false;
}
mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); memid.initially_committed = true;
const bool ok_before = mi_os_secure_guard_page(mi_ptr_with_addr(area, before), memid, true);
const bool ok_after = mi_os_secure_guard_page(mi_ptr_with_addr(area, after), memid, true);
return (ok_before && ok_after);
}
static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size);
static void mi_os_prim_free_no_stat(void* addr, size_t size) {
int err = _mi_prim_free(addr, size); if (err != 0) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
}
}
static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) {
mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL) return; mi_os_prim_free_no_stat(addr, size);
if (adjust) {
if (commit_size>0) { mi_os_stat_adjust_decrease(subproc, committed, commit_size); }
mi_os_stat_adjust_decrease(subproc, reserved, size);
}
else {
if (commit_size>0) { mi_os_stat_decrease(subproc, committed, commit_size); }
mi_os_stat_decrease(subproc, reserved, size);
}
}
void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) {
MI_UNUSED(addr); MI_UNUSED(size);
if (mi_memkind_is_os(memid.memkind)) {
void* const base = memid.mem.os.base;
const size_t full_size = mi_memid_os_size(memid);
mi_assert_internal((uint8_t*)base <= (uint8_t*)addr && (uint8_t*)addr + size <= (uint8_t*)base + full_size);
if (memid.memkind == MI_MEM_OS_HUGE) {
mi_assert(memid.is_pinned);
mi_os_free_huge_os_pages(subproc, base, full_size);
}
else {
mi_os_prim_free(subproc, base, full_size, (still_committed ? mi_memid_os_committed(memid) : 0), false);
}
}
else {
mi_assert(memid.memkind < MI_MEM_OS);
}
}
void _mi_os_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) {
_mi_os_free_ex(subproc, p, size, true, memid);
}
static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(is_zero != NULL);
mi_assert_internal(is_large != NULL);
if (size == 0) return NULL;
if (!mi_memid_os_size_fits(size)) {
_mi_warning_message("unable to allocate OS memory as the size is too large (size: 0x%zx bytes)\n", size);
return NULL;
}
if (!commit) { allow_large = false; }
#if MI_SECURE > 0 && MI_OS_HAS_GUARD_PAGES
allow_large = false; #endif
if (try_alignment == 0) { try_alignment = 1; }
const size_t large_page_size = mi_os_mem_config()->large_page_size;
if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) {
try_alignment = large_page_size;
}
*is_zero = false;
void* p = NULL;
int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);
if (err != 0) {
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
}
mi_os_stat_counter_increase(subproc, mmap_calls, 1);
if (p != NULL) {
mi_os_stat_increase(subproc, reserved, size);
if (commit) {
mi_os_stat_increase(subproc, committed, size);
#ifdef MI_TRACK_ASAN
if (*is_zero) { mi_track_mem_defined(p,size); }
else { mi_track_mem_undefined(p,size); }
#endif
}
}
return p;
}
static void* mi_os_prim_alloc(mi_subproc_t* subproc, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
return mi_os_prim_alloc_at(subproc, NULL , size, try_alignment, commit, allow_large, is_large, is_zero);
}
static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert_internal(memid!=NULL);
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
*memid = _mi_memid_none();
if (!commit) allow_large = false;
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = mi_align_up(size, _mi_os_page_size());
#if MI_SIZE_SIZE >= 8
const bool try_direct_alloc = true;
#else
const bool try_direct_alloc = (mi_os_mem_config()->has_aligned_alloc || alignment <= mi_os_mem_config()->alloc_granularity || alignment <= size/4);
#endif
bool os_is_large = false;
bool os_is_zero = false;
void* os_base = NULL;
size_t os_size = size;
void* p = NULL;
if (try_direct_alloc) {
p = mi_os_prim_alloc(subproc, size, alignment, commit, allow_large, &os_is_large, &os_is_zero);
}
if (p != NULL && mi_is_aligned(p,alignment)) {
os_base = p;
}
else {
#if !MI_TRACK_ASAN && !defined(MI_NO_ALIGN_FALLBACK_WARNING)
if (try_direct_alloc) {
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
}
#endif
if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true ); }
if (size >= (SIZE_MAX - alignment)) return NULL; const size_t over_size = size + alignment;
if (!mi_os_mem_config()->has_partial_free) { p = mi_os_prim_alloc(subproc, over_size, 1 , false , false , &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
os_base = p; os_size = over_size;
p = mi_align_up_ptr(p, alignment);
if (commit) {
if (!_mi_os_commit(subproc, p, size, NULL)) {
mi_os_prim_free(subproc, os_base, over_size, 0, true);
return NULL;
}
}
}
else { p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large , &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
void* const aligned_p = mi_align_up_ptr(p, alignment);
const size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
const size_t mid_size = mi_align_up(size, _mi_os_page_size());
const size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true ); }
if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true ); }
p = aligned_p;
os_base = aligned_p; os_size = mid_size;
}
}
mi_assert_internal(p != NULL && os_base != NULL && mi_is_aligned(p,alignment));
mi_assert_internal(os_base <= p && size <= os_size);
*memid = _mi_memid_create_os(os_base, os_size, (commit ? size : 0), os_is_zero, os_is_large);
return p;
}
void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (size == 0) return NULL;
size = _mi_os_good_alloc_size(size);
bool os_is_large = false;
bool os_is_zero = false;
void* p = mi_os_prim_alloc(subproc, size, 0, true, false, &os_is_large, &os_is_zero);
if (p == NULL) return NULL;
*memid = _mi_memid_create_os(p, size, size, os_is_zero, os_is_large);
mi_assert_internal(mi_memid_os_size(*memid) >= size);
mi_assert_internal(memid->initially_committed);
return p;
}
void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
{
MI_UNUSED(&_mi_os_get_aligned_hint); *memid = _mi_memid_none();
if (size == 0) return NULL;
size = _mi_os_good_alloc_size(size);
alignment = mi_align_up(alignment, _mi_os_page_size());
void* p = mi_os_prim_alloc_aligned(subproc, size, alignment, commit, allow_large, memid );
if (p == NULL) return NULL;
mi_assert_internal(mi_memid_os_size(*memid) >= size);
mi_assert_internal(mi_is_aligned(p,alignment));
if (commit) { mi_assert_internal(memid->initially_committed); }
return p;
}
mi_decl_nodiscard static void* mi_os_ensure_zero(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t* memid) {
if (p==NULL || size==0) return p;
if (!memid->initially_committed) {
const size_t commit_size = mi_memid_os_size(*memid);
bool is_zero = false;
if (!_mi_os_commit(subproc, memid->mem.os.base, commit_size, &is_zero)) {
_mi_os_free(subproc, p, size, *memid);
return NULL;
}
mi_memid_os_set_committed(memid, commit_size);
memid->initially_committed = true;
}
if (memid->initially_zero) return p;
mi_memzero_aligned(p,size);
memid->initially_zero = true;
return p;
}
void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) {
void* p = _mi_os_alloc(subproc, size,memid);
return mi_os_ensure_zero(subproc, p, size, memid);
}
void* _mi_os_alloc_with_guards(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, mi_memid_t* memid) {
const size_t guard_size = _mi_os_secure_guard_page_size();
size = mi_align_up(size, _mi_os_page_size());
uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, size + 2*guard_size, 1, commit, allow_large, memid);
if (base == NULL) return NULL;
uint8_t* const p = base + guard_size;
if (memid->initially_committed) { _mi_os_secure_guard_pages_set_around(subproc, p, size, *memid);
}
return p;
}
void _mi_os_free_with_guards(mi_subproc_t* subproc, mi_memid_t memid) {
_mi_os_free(subproc, memid.mem.os.base, mi_memid_os_size(memid), memid);
}
static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size);
void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert(offset <= size);
mi_assert((alignment % _mi_os_page_size()) == 0);
*memid = _mi_memid_none();
if (offset > size) return NULL;
if (offset == 0) {
return _mi_os_alloc_aligned(subproc, size, alignment, commit, allow_large, memid);
}
else {
const size_t extra = mi_align_up(offset, alignment) - offset;
if (size >= SIZE_MAX - extra) return NULL; const size_t oversize = size + extra;
void* const start = _mi_os_alloc_aligned(subproc, oversize, alignment, commit, allow_large, memid);
if (start == NULL) return NULL;
void* const p = (uint8_t*)start + extra;
mi_assert(mi_is_aligned((uint8_t*)p + offset, alignment));
if (commit && extra >= _mi_os_page_size()) {
const size_t decommit_size = mi_align_down(extra, _mi_os_page_size());
bool needs_recommit = false;
if (mi_os_decommit_ex(subproc, start, decommit_size, &needs_recommit, decommit_size) && needs_recommit) {
mi_memid_os_set_committed(memid, mi_memid_os_committed(*memid) - decommit_size);
}
}
return p;
}
}
static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
mi_assert(addr != NULL && size > 0);
if (newsize != NULL) *newsize = 0;
if (size == 0 || addr == NULL) return NULL;
void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size())
: mi_align_down_ptr(addr, _mi_os_page_size()));
void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
: mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
if (diff <= 0) return NULL;
mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));
if (newsize != NULL) *newsize = (size_t)diff;
return start;
}
static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
return mi_os_page_align_areax(true, addr, size, newsize);
}
bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) {
mi_assert_internal(size >= stat_already_committed);
if (is_zero != NULL) { *is_zero = false; }
mi_os_stat_counter_increase(subproc, commit_calls, 1);
size_t csize;
void* start = mi_os_page_align_areax(false , addr, size, &csize);
if (csize == 0) return true;
bool os_is_zero = false;
int err = _mi_prim_commit(start, csize, &os_is_zero);
if (err != 0) {
_mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
return false;
}
if (os_is_zero && is_zero != NULL) {
*is_zero = true;
mi_assert_expensive(mi_mem_is_zero(start, csize));
}
#ifdef MI_TRACK_ASAN
if (os_is_zero) { mi_track_mem_defined(start,csize); }
else { mi_track_mem_undefined(start,csize); }
#endif
mi_os_stat_increase(subproc, committed, size - stat_already_committed); return true;
}
bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) {
return _mi_os_commit_ex(subproc, addr, size, is_zero, 0);
}
static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
mi_assert_internal(needs_recommit!=NULL);
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true;
*needs_recommit = true;
int err = _mi_prim_decommit(start,csize,needs_recommit);
if (err != 0) {
_mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
else if (*needs_recommit) {
mi_os_stat_decrease(subproc, committed, stat_size);
}
mi_assert_internal(err == 0);
return (err == 0);
}
bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size) {
bool needs_recommit;
return mi_os_decommit_ex(subproc, addr, size, &needs_recommit, size);
}
bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size) {
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true; mi_os_stat_counter_increase(subproc, reset, csize);
mi_os_stat_counter_increase(subproc, reset_calls, 1);
#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED
memset(start, 0, csize); #endif
int err = _mi_prim_reset(start, csize);
if (err != 0) {
_mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
return (err == 0);
}
void _mi_os_reuse( mi_subproc_t* subproc, void* addr, size_t size ) {
MI_UNUSED(subproc);
size_t csize = 0;
void* const start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return;
const int err = _mi_prim_reuse(start, csize);
if (err != 0) {
_mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
}
}
bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg)
{
if (mi_option_get(mi_option_purge_delay) < 0) return false; mi_os_stat_counter_increase(subproc, purge_calls, 1);
mi_os_stat_counter_increase(subproc, purged, size);
if (commit_fun != NULL) {
bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);
return decommitted; }
else if (mi_option_is_enabled(mi_option_purge_decommits) && !_mi_preloading()) {
bool needs_recommit = true;
mi_os_decommit_ex(subproc, p, size, &needs_recommit, stat_size);
return needs_recommit;
}
else {
if (allow_reset) { _mi_os_reset(subproc, p, size);
}
return false; }
}
bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size) {
return _mi_os_purge_ex(subproc, p, size, true, size, NULL, NULL);
}
static bool mi_os_guardx(void* addr, size_t size, bool set, bool is_committed) {
size_t csize = 0;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return false;
int err = _mi_prim_guard(start, csize, set, is_committed);
if (err != 0) {
_mi_warning_message("cannot %s guard OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (set ? "set" : "reset"), err, err, start, csize);
}
return (err == 0);
}
bool _mi_os_guard(void* addr, size_t size, bool is_committed) {
return mi_os_guardx(addr, size, true, is_committed);
}
bool _mi_os_unguard(void* addr, size_t size, bool is_committed) {
return mi_os_guardx(addr, size, false, is_committed);
}
#define MI_HUGE_OS_PAGE_SIZE (MI_GiB)
#if (MI_SIZE_SIZE >= 8) && !MI_CHERI
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
if (total_size != NULL) *total_size = 0;
size_t size = 0;
if (mi_mul_overflow(pages,MI_HUGE_OS_PAGE_SIZE,&size) || !mi_memid_os_size_fits(size)) {
_mi_warning_message("too many huge pages requested: %zu\n", pages);
return NULL;
}
mi_addr_t start = 0;
mi_addr_t end = 0;
mi_addr_t huge_start = mi_atomic_load_relaxed(&_mi_process.os.huge_start);
do {
start = huge_start;
if (start == 0) {
start = (MI_ADDR(32) << 40); #if (MI_SECURE || MI_DEBUG==0)
mi_theap_t* const theap = _mi_theap_default(); if (mi_theap_is_initialized(theap)) { const size_t r = _mi_theap_random_next(theap);
start = start + (MI_ADDR(MI_HUGE_OS_PAGE_SIZE) * ((r>>17) & 0x0FFF)); }
else {
_mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, mi_ptr_from_addr(start));
}
#endif
}
end = start + size;
} while (!mi_atomic_cas_weak_relaxed(&_mi_process.os.huge_start, &huge_start, end));
if (total_size != NULL) *total_size = size;
return (uint8_t*)mi_ptr_from_addr(start);
}
#else
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
MI_UNUSED(pages);
if (total_size != NULL) *total_size = 0;
return NULL;
}
#endif
void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (psize != NULL) *psize = 0;
if (pages_reserved != NULL) *pages_reserved = 0;
#if MI_SECURE > 0 && MI_OS_HAS_GUARD_PAGES
_mi_warning_message("cannot allocate huge OS pages in secure mode as pinned memory cannot support guard pages\n");
return NULL;
#endif
size_t size = 0;
uint8_t* const start = mi_os_claim_huge_pages(pages, &size);
if (start == NULL) return NULL;
mi_msecs_t start_t = _mi_clock_start();
size_t page = 0;
bool all_zero = true;
while (page < pages) {
bool is_zero = false;
void* addr = start + (page * MI_HUGE_OS_PAGE_SIZE);
void* p = NULL;
int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);
if (!is_zero) { all_zero = false; }
if (err != 0) {
_mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE);
break;
}
if (p != addr) {
if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
mi_os_prim_free_no_stat(p, MI_HUGE_OS_PAGE_SIZE); }
break;
}
page++; mi_os_stat_increase(subproc, committed, MI_HUGE_OS_PAGE_SIZE);
mi_os_stat_increase(subproc, reserved, MI_HUGE_OS_PAGE_SIZE);
if (max_msecs > 0) {
mi_msecs_t elapsed = _mi_clock_end(start_t);
if (page >= 1) {
mi_msecs_t estimate = ((elapsed / (page==0 ? 1 : page)) * pages);
if (estimate > 2*max_msecs) { elapsed = max_msecs + 1;
}
}
if (elapsed > max_msecs) {
_mi_warning_message("huge OS page allocation timed out (after allocating %zu page(s))\n", page);
break;
}
}
}
const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;
mi_assert_internal(allocated <= size);
if (pages_reserved != NULL) { *pages_reserved = page; }
if (psize != NULL) { *psize = allocated; }
if (page != 0) {
mi_assert(start != NULL);
*memid = _mi_memid_create_os(start, allocated, allocated, all_zero, true );
memid->memkind = MI_MEM_OS_HUGE;
mi_assert(memid->is_pinned);
#ifdef MI_TRACK_ASAN
if (all_zero) { mi_track_mem_defined(start,allocated); }
#endif
}
return (page == 0 ? NULL : start);
}
static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size) {
if (p==NULL || size==0) return;
uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) {
mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, false );
size -= MI_HUGE_OS_PAGE_SIZE;
base += MI_HUGE_OS_PAGE_SIZE;
}
}
int _mi_os_numa_node_count(void) {
size_t count = mi_atomic_load_acquire(&_mi_process.os.numa_node_count);
if mi_unlikely(count == 0) {
long ncount = mi_option_get(mi_option_use_numa_nodes); if (ncount > 0 && ncount < INT_MAX) {
count = (size_t)ncount;
}
else {
const size_t n = _mi_prim_numa_node_count(); if (n == 0 || n > INT_MAX) { count = 1; }
else { count = n; }
}
mi_atomic_store_release(&_mi_process.os.numa_node_count, count); if (count>1) { _mi_verbose_message("using %zd numa regions\n", count); }
}
mi_assert_internal(count > 0 && count <= INT_MAX);
return (int)count;
}
static int mi_os_numa_node_get(void) {
int numa_count = _mi_os_numa_node_count();
if (numa_count<=1) return 0; const size_t n = _mi_prim_numa_node();
int numa_node = (n < INT_MAX ? (int)n : 0);
if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }
return numa_node;
}
int _mi_os_numa_node(void) {
if mi_likely(mi_atomic_load_relaxed(&_mi_process.os.numa_node_count) == 1) {
return 0;
}
else {
return mi_os_numa_node_get();
}
}
#if 0#endif