#ifndef _DEFAULT_SOURCE
#define _DEFAULT_SOURCE
#endif
#if defined(__sun)
#undef _XOPEN_SOURCE
#undef _POSIX_C_SOURCE
#endif
#include "mimalloc.h"
#include "mimalloc/prim/prim.h"
#include "mimalloc/track.h"
#include "mimalloc/prim/os.h"
#include "mimalloc/util/libc.h"
#include "mimalloc/util/options.h"
#include "mimalloc/heap/process.h"
#include "mimalloc/prim/prim-tls.h"
#include <sys/mman.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdlib.h>
#if defined(__linux__)
#include <features.h>
#include <sys/prctl.h>
#include <sys/sysinfo.h>
#if defined(__GLIBC__) && !defined(PR_SET_VMA)
#include <linux/prctl.h>
#endif
#if defined(__GLIBC__)
#include <linux/mman.h>
#endif
#if defined(__riscv) || defined(_M_RISCV)
#if defined(MI_HAS_SYS_HWPROBEH)
#include <sys/hwprobe.h>
#elif defined(MI_HAS_ASM_HWPROBEH)
#include <asm/hwprobe.h>
#endif
#endif
#if MI_PAC >= 2 && defined(__aarch64__) && !defined(__APPLE__)
#include <sys/auxv.h>
#define MI_UNIX_LINUX_PAC 1
#ifndef HWCAP_PACG
#define HWCAP_PACG (1UL << 31)
#endif
#endif
#if MI_MTE && defined(__aarch64__)
#include <sys/auxv.h>
#define MI_UNIX_LINUX_MTE 1
#ifndef HWCAP2_MTE
#define HWCAP2_MTE (1UL << 18)
#endif
#ifndef PROT_MTE
#define PROT_MTE (0x20)
#endif
#ifndef PR_SET_TAGGED_ADDR_CTRL
#define PR_SET_TAGGED_ADDR_CTRL 55
#define PR_GET_TAGGED_ADDR_CTRL 56
#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
#endif
#ifndef PR_MTE_TCF_SHIFT
#define PR_MTE_TCF_SHIFT 1
#define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
#define PR_MTE_TCF_ASYNC (2UL << PR_MTE_TCF_SHIFT)
#define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
#define PR_MTE_TAG_SHIFT 3
#define PR_MTE_TAG_MASK (0xFFFFUL << PR_MTE_TAG_SHIFT)
#endif
#endif
#elif defined(__APPLE__)
#include <AvailabilityMacros.h>
#include <TargetConditionals.h>
#if !defined(TARGET_OS_OSX) || TARGET_OS_OSX || MI_MTE
#include <mach/vm_statistics.h>
#endif
#if !defined(MAC_OS_X_VERSION_10_7)
#define MAC_OS_X_VERSION_10_7 1070
#endif
#include <sys/sysctl.h>
#if MI_MTE && defined(VM_FLAGS_MTE) && __has_include(<os/security_config.h>)
#include <os/security_config.h>
#include <mach/mach.h>
#include <mach/vm_map.h>
#define MI_UNIX_MACH_MTE 1
#endif
#elif defined(__FreeBSD__) || defined(__DragonFly__)
#include <sys/param.h>
#if __FreeBSD_version >= 1200000
#include <sys/cpuset.h>
#include <sys/domainset.h>
#endif
#include <sys/sysctl.h>
#endif
#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__FreeBSD__)
#define MI_HAS_SYSCALL_H
#include <sys/syscall.h>
#endif
#if defined(__has_feature)
#if __has_feature(memory_sanitizer)
#include <sanitizer/msan_interface.h>
#define mi_msan_unpoison(p,size) __msan_unpoison(p,size)
#endif
#endif
#ifndef mi_msan_unpoison
#define mi_msan_unpoison(p,size)
#endif
#if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED)
#define MADV_DONTNEED POSIX_MADV_DONTNEED
#endif
#if !defined(MADV_FREE) && defined(POSIX_MADV_FREE)
#define MADV_FREE POSIX_MADV_FREE
#endif
#define MI_UNIX_MAX_THP_PAGE_SIZE (8*MI_MiB)
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) && !defined(__ANDROID__)
static inline int mi_prim_open(const char* fpath, int open_flags) {
return syscall(SYS_open,fpath,open_flags,0);
}
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
ssize_t nread = syscall(SYS_read,fd,buf,bufsize);
if (nread > 0) { mi_msan_unpoison(buf, (size_t)nread); }
return nread;
}
static inline int mi_prim_close(int fd) {
return syscall(SYS_close,fd);
}
static inline int mi_prim_access(const char *fpath, int mode) {
return syscall(SYS_access,fpath,mode);
}
#else
static inline int mi_prim_open(const char* fpath, int open_flags) {
return open(fpath,open_flags);
}
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
return read(fd,buf,bufsize);
}
static inline int mi_prim_close(int fd) {
return close(fd);
}
static inline int mi_prim_access(const char *fpath, int mode) {
return access(fpath,mode);
}
#endif
static bool unix_detect_overcommit(void) {
bool os_overcommit = true;
#if defined(__linux__)
int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY);
if (fd >= 0) {
char buf[64];
ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
mi_prim_close(fd);
if (nread >= 1 && nread < (ssize_t)sizeof(buf)) {
buf[nread] = 0;
os_overcommit = (buf[0] == '0' || buf[0] == '1');
}
}
#elif defined(__FreeBSD__)
int val = 0;
size_t olen = sizeof(val);
if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) {
os_overcommit = (val != 0);
}
#else
#endif
return os_overcommit;
}
static bool unix_detect_thp(void) {
bool thp_enabled = false;
#if defined(__linux__)
int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
if (fd >= 0) {
char buf[64];
ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
mi_prim_close(fd);
if (nread >= 1 && nread < (ssize_t)sizeof(buf)) {
buf[nread] = 0;
thp_enabled = (_mi_strnstr(buf,nread,"[never]") == NULL);
}
}
#endif
return thp_enabled;
}
static size_t unix_detect_large_page_size(void) {
size_t large_page_size = 2*MI_MiB; #if defined(__linux__)
const int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY);
if (fd >= 0) {
char buf[64];
const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
mi_prim_close(fd);
if (nread >= 1 && nread < (ssize_t)sizeof(buf)) {
buf[nread] = 0;
long size = 0;
if (_mi_strntol(buf, nread, NULL, &size) && size > 0 && (unsigned long)size <= SIZE_MAX) {
large_page_size = (size_t)size;
}
}
}
#endif
return large_page_size;
}
static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) {
#if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE))
MI_UNUSED(page_size);
int64_t physical_memory = 0;
size_t length = sizeof(int64_t);
#if defined(HW_PHYSMEM64)
int mib[2] = { CTL_HW, HW_PHYSMEM64 };
#else
int mib[2] = { CTL_HW, HW_MEMSIZE };
#endif
const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0);
if (err==0 && physical_memory > 0) {
const int64_t phys_in_kib = physical_memory / MI_KiB;
if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) {
*physical_memory_in_kib = (size_t)phys_in_kib;
}
}
#elif defined(__linux__)
MI_UNUSED(page_size);
struct sysinfo info; _mi_memzero_var(info);
const int err = sysinfo(&info);
if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) {
if (info.mem_unit==MI_KiB) {
*physical_memory_in_kib = (size_t)info.totalram;
}
else {
size_t total = 0;
if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) {
*physical_memory_in_kib = (total / MI_KiB);
}
}
}
#elif defined(_SC_PHYS_PAGES)
const long pphys = sysconf(_SC_PHYS_PAGES);
const size_t psize_in_kib = page_size / MI_KiB;
if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) {
*physical_memory_in_kib = (size_t)pphys * psize_in_kib;
}
#endif
}
static size_t unix_detect_virtual_address_bits(void) {
#if defined(__riscv) || defined(_M_RISCV)
#if defined(RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS)
struct riscv_hwprobe probe = { .key = RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS, };
#if defined(MI_HAS_SYS_HWPROBEH)
if (__riscv_hwprobe(&probe, 1, 0, NULL, 0) == 0)
#else
if (syscall(__NR_riscv_hwprobe, &probe, 1, 0, NULL, 0) == 0)
#endif
{
if (probe.key != -1) { return (MI_SIZE_BITS - mi_clz((size_t)probe.value));
}
}
#endif
const int fd = mi_prim_open("/proc/cpuinfo", O_RDONLY);
if (fd >= 0) {
char buf[2048];
const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
mi_prim_close(fd);
if ((nread >= 1) && (nread < (ssize_t)sizeof(buf))) {
buf[nread] = 0;
if (_mi_strnstr(buf, nread, "sv39")) { return 39; }
else if (_mi_strnstr(buf, nread, "sv48")) { return 48; }
else if (_mi_strnstr(buf, nread, "sv57")) { return 57; }
}
}
#endif return MI_MAX_VABITS;
}
#if MI_UNIX_LINUX_MTE
#if MI_MTE >= 2
#define unix_memtag_thread_error(...) _mi_fatal_error(__VA_ARGS__)
#else
#define unix_memtag_thread_error(...) _mi_warning_message(__VA_ARGS__)
#endif
static bool unix_memtag_thread_enable(unsigned long ctrl, unsigned long tcf) {
const unsigned long tags = (ctrl & PR_MTE_TAG_MASK) | (0xFFFEUL << PR_MTE_TAG_SHIFT);
return (prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE | tcf | tags, 0, 0, 0) == 0);
}
static size_t unix_thread_count(void) {
const int fd = mi_prim_open("/proc/self/stat", O_RDONLY);
if (fd < 0) return 0;
char buf[1024];
const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
mi_prim_close(fd);
if (nread <= 0 || nread >= (ssize_t)sizeof(buf)) return 0;
buf[nread] = 0;
const char* s = NULL;
for (const char* p = buf; *p != 0; p++) { if (*p == ')') { s = p + 1; } }
if (s == NULL) return 0;
for (int field = 3; field < 20; field++) {
while (*s == ' ') { s++; }
while (*s != ' ' && *s != 0) { s++; }
}
while (*s == ' ') { s++; }
size_t count = 0;
while (*s >= '0' && *s <= '9') { count = 10*count + (size_t)(*s - '0'); s++; }
return count;
}
#endif
#if MI_MTE >= 2
static bool unix_detect_memtag_cpu(void) {
#if MI_UNIX_LINUX_MTE
return ((getauxval(AT_HWCAP2) & HWCAP2_MTE) != 0);
#elif defined(__APPLE__) && defined(__aarch64__)
int val = 0;
size_t len = sizeof(val);
return (sysctlbyname("hw.optional.arm.FEAT_MTE", &val, &len, NULL, 0) == 0 && val != 0);
#else
return true;
#endif
}
#endif
#if MI_PAC >= 2
static bool unix_detect_pac_cpu(void) {
#if MI_UNIX_LINUX_PAC
return ((getauxval(AT_HWCAP) & HWCAP_PACG) != 0);
#elif defined(__APPLE__) && defined(__aarch64__)
int val = 0;
size_t len = sizeof(val);
return (sysctlbyname("hw.optional.arm.FEAT_PAuth", &val, &len, NULL, 0) == 0 && val != 0);
#else
return true;
#endif
}
#endif
static bool unix_detect_memtag(mi_os_mem_config_t* config) {
MI_UNUSED(config);
#if MI_UNIX_MACH_MTE
if (__builtin_available(macOS 26.0, iOS 26.0, *)) {
return ((os_security_config_get() & OS_SECURITY_CONFIG_MTE) != 0); }
#elif MI_UNIX_LINUX_MTE
if ((getauxval(AT_HWCAP2) & HWCAP2_MTE) == 0) return false;
const int ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
if (ctrl < 0) return false;
unsigned long tcf;
switch (mi_option_get(mi_option_mte)) {
case MI_MEMTAG_MODE_SYNC: tcf = PR_MTE_TCF_SYNC; break;
case MI_MEMTAG_MODE_ASYNC: tcf = PR_MTE_TCF_ASYNC; break;
case MI_MEMTAG_MODE_PREFERRED: tcf = PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC; break;
default: tcf = ((unsigned long)ctrl & PR_MTE_TCF_MASK); if (tcf == 0) { tcf = PR_MTE_TCF_SYNC; } break;
}
if (!unix_memtag_thread_enable((unsigned long)ctrl, tcf)) {
_mi_warning_message("unable to enable memory tagging (error %d)\n", errno);
return false;
}
config->memtag_mode = tcf;
_mi_verbose_message("memory tag check mode: %s\n", (tcf == PR_MTE_TCF_SYNC ? "sync" : (tcf == PR_MTE_TCF_ASYNC ? "async" : "preferred")));
const size_t nthreads = unix_thread_count();
if (nthreads > 1) {
unix_memtag_thread_error("memory tagging was enabled after %zu other thread(s) were created; these may not use tag checking\n"
" (and system calls with tagged pointers may fail in those threads until they call mimalloc)\n", nthreads - 1);
}
return true;
#endif
return false;
}
void _mi_prim_mem_init( mi_os_mem_config_t* config )
{
long psize = sysconf(_SC_PAGESIZE);
if (psize > 0 && (unsigned long)psize < SIZE_MAX) {
config->page_size = (size_t)psize;
config->alloc_granularity = (size_t)psize;
unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib);
}
config->has_partial_free = true; config->has_virtual_reserve = true; #if defined(MAP_ALIGNED) || defined(MAP_ALIGN)
config->has_aligned_alloc = true; #endif
config->large_page_size = unix_detect_large_page_size();
config->has_transparent_huge_pages = (config->large_page_size <= MI_UNIX_MAX_THP_PAGE_SIZE) && unix_detect_thp();
config->has_overcommit = unix_detect_overcommit();
config->virtual_address_bits = unix_detect_virtual_address_bits();
#if MI_MTE >= 2
if (!unix_detect_memtag_cpu()) {
_mi_fatal_error("built with MI_MTE=ON but the CPU does not support memory tagging (MTE)\n");
}
#endif
config->has_memtag = (mi_option_is_enabled(mi_option_mte) && unix_detect_memtag(config));
#if MI_PAC >= 2
config->has_pac = unix_detect_pac_cpu();
#endif
#if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)
if (!mi_option_is_enabled(mi_option_allow_thp)) {
config->has_transparent_huge_pages = false;
if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) { (void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);
}
}
#endif
}
int _mi_prim_free(void* addr, size_t size ) {
if (size==0) return 0;
bool err = (munmap(addr, size) == -1);
return (err ? errno : 0);
}
static int unix_madvise(void* addr, size_t size, int advice) {
#if defined(__sun) || defined(_AIX)
const int res = madvise((caddr_t)addr, size, advice); return (res==0 ? 0 : errno);
#elif defined(__QNX__)
return posix_madvise(addr, size, advice); #else
const int res = madvise(addr, size, advice); return (res==0 ? 0 : errno);
#endif
}
static void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) {
void* p = mmap(addr, size, protect_flags, flags, fd, 0 );
#if defined(__linux__) && defined(PR_SET_VMA)
static _Atomic(size_t) no_set_vma_name; if (p!=MAP_FAILED && p!=NULL && mi_atomic_load_relaxed(&no_set_vma_name)==0) {
const int err = errno;
if (prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, "mimalloc") != 0) {
if (errno == EINVAL) { mi_atomic_store_relaxed(&no_set_vma_name, (size_t)1); }
errno = err; }
}
#endif
return p;
}
static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
MI_UNUSED(try_alignment);
void* p = NULL;
#if defined(MAP_ALIGNED)
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
size_t n = 0;
mi_bsr(try_alignment, &n);
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd);
if (p==MAP_FAILED || !mi_is_aligned(p,try_alignment)) {
int err = errno;
_mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
}
if (p!=MAP_FAILED) return p;
}
}
#elif defined(MAP_ALIGN)
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd); if (p!=MAP_FAILED) return p;
}
#endif
#if (MI_SIZE_SIZE >= 8) && !defined(MAP_ALIGNED)
if (addr == NULL) {
void* hint = _mi_os_get_aligned_hint(try_alignment, size);
if (hint != NULL) {
p = unix_mmap_prim(hint, size, protect_flags, flags, fd);
if (p==MAP_FAILED || !mi_is_aligned(p,try_alignment)) {
#if MI_TRACK_ENABLED
int err = 0;
#else
int err = errno;
#endif
_mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
}
if (p!=MAP_FAILED) return p;
}
}
#endif
p = unix_mmap_prim(addr, size, protect_flags, flags, fd);
if (p!=MAP_FAILED) return p;
return NULL;
}
static int unix_mmap_fd(void) {
#if defined(VM_MAKE_TAG)
int os_tag = (int)mi_option_get(mi_option_os_tag);
if (os_tag < 100 || os_tag > 255) { os_tag = 254; }
return VM_MAKE_TAG(os_tag);
#else
return -1;
#endif
}
#if defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) && !(defined(__APPLE__) && (defined(__aarch64__) || defined(__arm64__)))
#define MI_OS_HAS_SUPERPAGES 1
#endif
#if defined(MAP_ALIGNED_SUPER) || defined(MAP_HUGETLB) || defined(MAP_HUGE_1GB) || defined(MAP_HUGE_2MB) || MI_OS_HAS_SUPERPAGES
#define MI_OS_HAS_HUGE_PAGES 1
#endif
#if MI_UNIX_MACH_MTE
static void* unix_mmap_memtag(size_t size, size_t try_alignment, int protect_flags, int fd) {
const vm_offset_t mask = (mi_is_power_of_two(try_alignment) && try_alignment <= MI_GiB ? try_alignment - 1 : 0);
const vm_prot_t prot = ((protect_flags & PROT_WRITE) != 0 ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_NONE);
vm_address_t addr = 0;
const kern_return_t err = vm_map(mach_task_self(), &addr, size, mask, VM_FLAGS_ANYWHERE | VM_FLAGS_MTE | (fd != -1 ? fd : 0),
MEMORY_OBJECT_NULL, 0, FALSE, prot, VM_PROT_READ | VM_PROT_WRITE, VM_INHERIT_DEFAULT);
if (err != KERN_SUCCESS) {
_mi_trace_message("unable to allocate memory tagged OS memory (error: %d, size: 0x%zx bytes, alignment: 0x%zx)\n", err, size, try_alignment);
errno = ENOMEM;
return NULL;
}
return (void*)addr;
}
#endif
static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
#if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON
#endif
#if !defined(MAP_NORESERVE)
#define MAP_NORESERVE 0
#endif
void* p = NULL;
const int fd = unix_mmap_fd();
bool allow_huge = allow_large;
#if MI_UNIX_MACH_MTE || MI_UNIX_LINUX_MTE
if (_mi_os_has_memtag()) { *is_large = false;
if (large_only) return NULL;
#if MI_UNIX_MACH_MTE
return unix_mmap_memtag(size, try_alignment, protect_flags, fd);
#else
allow_huge = false;
protect_flags |= PROT_MTE; #endif
}
#endif
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
if (_mi_os_has_overcommit()) {
flags |= MAP_NORESERVE;
}
#if defined(PROT_MAX)
protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); #endif
#if MI_OS_HAS_HUGE_PAGES
if (allow_huge && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {
size_t backoff = mi_atomic_load_relaxed(&_mi_process.os.large_page_backoff);
if (!large_only && backoff > 0) {
mi_atomic_cas_strong_relaxed(&_mi_process.os.large_page_backoff, &backoff, backoff - 1);
}
else {
int lflags = flags & ~MAP_NORESERVE; int lfd = fd;
#ifdef MAP_ALIGNED_SUPER
lflags |= MAP_ALIGNED_SUPER;
#endif
#ifdef MAP_HUGETLB
lflags |= MAP_HUGETLB;
#endif
#ifdef MAP_HUGE_1GB
if (large_only && (size % MI_GiB) == 0 && (mi_atomic_load_relaxed(&_mi_process.os.huge_page_unavailable)==0)) {
lflags |= MAP_HUGE_1GB;
}
else
#endif
{
#ifdef MAP_HUGE_2MB
lflags |= MAP_HUGE_2MB;
#endif
}
#if MI_OS_HAS_SUPERPAGES
lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
#endif
if (large_only || lflags != flags) {
*is_large = true;
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
#ifdef MAP_HUGE_1GB
if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {
mi_atomic_store_relaxed(&_mi_process.os.huge_page_unavailable,1); if (large_only) {
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno);
}
lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
}
#endif
if (large_only) return p;
if (p == NULL) {
mi_atomic_store_relaxed(&_mi_process.os.large_page_backoff, (size_t)8); }
}
}
} #else
MI_UNUSED(allow_huge);
if (large_only) { *is_large = false; errno = ENOMEM; return NULL; } #endif
if (p == NULL) {
*is_large = false;
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd);
if (p!=NULL) {
#if defined(MADV_NOHUGEPAGE)
const size_t large_page_size = _mi_os_large_page_size();
if (large_page_size > MI_UNIX_MAX_THP_PAGE_SIZE && size >= large_page_size) {
(void)unix_madvise(p, size, MADV_NOHUGEPAGE); }
else
#endif
{ #if !defined(MI_NO_THP)
if (allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size,try_alignment)) {
#if defined(MADV_HUGEPAGE)
if (_mi_os_has_thp()) {
if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
};
}
#elif defined(__sun)
struct memcntl_mha cmd = {0};
cmd.mha_pagesize = _mi_os_large_page_size();
cmd.mha_cmd = MHA_MAPSIZE_VA;
if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
}
#endif
}
#endif
}
}
}
return p;
}
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(commit || !allow_large);
mi_assert_internal(try_alignment > 0);
*is_zero = true;
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
*addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large);
return (*addr != NULL ? 0 : errno);
}
static void unix_mprotect_hint(int err) {
#if defined(__linux__) && (MI_SECURE>=5 || MI_GUARDED)
if (err == ENOMEM) {
_mi_warning_message("The next warning may be caused by a low memory map limit.\n"
" On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"
" For example: sudo sysctl -w vm.max_map_count=262144\n");
}
#else
MI_UNUSED(err);
#endif
}
int _mi_prim_commit(void* start, size_t size, bool* is_zero) {
*is_zero = false;
int err = mprotect(start, size, (PROT_READ | PROT_WRITE));
if (err != 0) {
err = errno;
unix_mprotect_hint(err);
}
return err;
}
int _mi_prim_reuse(void* start, size_t size) {
MI_UNUSED(start); MI_UNUSED(size);
#if defined(__APPLE__) && defined(MADV_FREE_REUSE)
return unix_madvise(start, size, MADV_FREE_REUSE);
#endif
return 0;
}
static int unix_madvise_release(void* start, size_t size) {
#if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
int err = unix_madvise(start, size, MADV_FREE_REUSABLE);
if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
return err;
#else
return unix_madvise(start, size, MADV_DONTNEED);
#endif
}
int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
int err = 0;
#if 1
err = unix_madvise_release(start, size);
#if MI_DEBUG || MI_SECURE>=4
*needs_recommit = true;
mprotect(start, size, PROT_NONE);
#else
*needs_recommit = false;
#endif
#else #endif
return err;
}
int _mi_prim_reset(void* start, size_t size) {
int err = 0;
#if 0 #endif
#if defined(MADV_FREE)
const bool lazy = (mi_atomic_load_relaxed(&_mi_process.os.lazy_reset_unavailable) == 0);
while ((err = unix_madvise(start, size, (lazy ? MADV_FREE : MADV_DONTNEED))) != 0 && err == EAGAIN) { };
if (err == EINVAL && lazy) {
mi_atomic_store_release(&_mi_process.os.lazy_reset_unavailable, (size_t)1);
err = unix_madvise(start, size, MADV_DONTNEED);
}
#else
err = unix_madvise(start, size, MADV_DONTNEED);
#endif
return err;
}
#if defined(__OpenBSD__)
#include <sys/param.h>
#endif
static void unix_mseal(void* addr, size_t size) {
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mseal)
syscall(SYS_mseal, addr, size, 0UL); #elif defined(__OpenBSD__) && defined(OpenBSD) && (OpenBSD >= 202304)
mimmutable(addr, size); #else
MI_UNUSED(addr); MI_UNUSED(size);
#endif
}
int _mi_prim_protect(void* addr, size_t size, mi_protect_t protect) {
const int prot = (protect == MI_PROTECT_NONE ? PROT_NONE : (protect == MI_PROTECT_READWRITE ? (PROT_READ | PROT_WRITE) : PROT_READ));
int err = mprotect(addr, size, prot);
if (err != 0) { err = errno; }
unix_mprotect_hint(err);
if (err == 0 && protect == MI_PROTECT_READ_SEALED) { unix_mseal(addr, size); }
return err;
}
int _mi_prim_guard(void* start, size_t size, bool set_guard, bool is_committed) {
if (set_guard) {
unix_madvise_release(start, size); return _mi_prim_protect(start, size, MI_PROTECT_NONE);
}
else if (is_committed) {
const int err = _mi_prim_protect(start, size, MI_PROTECT_READWRITE);
return (err != 0 ? err : _mi_prim_reuse(start, size));
}
else {
return 0; }
}
#if (MI_SIZE_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)
#ifndef MPOL_PREFERRED
#define MPOL_PREFERRED 1
#endif
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind)
static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags);
}
#else
static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags);
return 0;
}
#endif
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
bool is_large = true;
*is_zero = true;
*addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large);
if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_SIZE_SIZE - 1)) { unsigned long numa_mask = (1UL << numa_node);
long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_SIZE_SIZE, 0);
if (err != 0) {
err = errno;
_mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err);
}
}
return (*addr != NULL ? 0 : errno);
}
#else
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
*is_zero = false;
*addr = NULL;
return ENOMEM;
}
#endif
#if defined(__linux__)
size_t _mi_prim_numa_node(void) {
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)
unsigned int node = 0;
unsigned int ncpu = 0;
int err = syscall(SYS_getcpu, &ncpu, &node, NULL);
if (err != 0) return 0;
return node;
#else
return 0;
#endif
}
size_t _mi_prim_numa_node_count(void) {
char buf[128];
unsigned last_found = 0;
for(unsigned node = 1; node < 256; node++) {
_mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node);
if (mi_prim_access(buf,R_OK) != 0) {
if (node - last_found > 4) break; }
else { last_found = node; } }
return last_found + 1;
}
#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000
size_t _mi_prim_numa_node(void) {
domainset_t dom;
size_t node;
int policy;
if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul;
for (node = 0; node < MAXMEMDOM; node++) {
if (DOMAINSET_ISSET(node, &dom)) return node;
}
return 0ul;
}
size_t _mi_prim_numa_node_count(void) {
size_t ndomains = 0;
size_t len = sizeof(ndomains);
if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul;
return ndomains;
}
#elif defined(__DragonFly__)
size_t _mi_prim_numa_node(void) {
return 0ul;
}
size_t _mi_prim_numa_node_count(void) {
size_t ncpus = 0, nvirtcoresperphys = 0;
size_t len = sizeof(size_t);
if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul;
if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul;
return nvirtcoresperphys * ncpus;
}
#else
size_t _mi_prim_numa_node(void) {
return 0;
}
size_t _mi_prim_numa_node_count(void) {
return 1;
}
#endif
#include <time.h>
static mi_msecs_t mi_prim_clock_now_lowres(void) {
const int64_t ticks = (int64_t)clock();
#if !defined(CLOCKS_PER_SEC)
return ticks;
#else
if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) {
return ticks;
}
else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) {
return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
}
else {
return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
}
#endif
}
mi_msecs_t _mi_prim_clock_now(void) {
#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
#ifdef CLOCK_MONOTONIC
const clockid_t clockid = CLOCK_MONOTONIC;
#else
const clockid_t clockid = CLOCK_REALTIME;
#endif
struct timespec t;
if (clock_gettime(clockid,&t) == 0) {
return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L);
}
#endif
return mi_prim_clock_now_lowres();
}
#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)
#include <stdio.h>
#include <sys/resource.h>
#if defined(__APPLE__)
#include <mach/mach.h>
#endif
#if defined(__HAIKU__)
#include <kernel/OS.h>
#endif
static mi_msecs_t timeval_secs(const struct timeval* tv) {
return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L);
}
void _mi_prim_process_info(mi_process_info_t* pinfo)
{
struct rusage rusage;
if (getrusage(RUSAGE_SELF, &rusage) == 0) {
pinfo->utime = timeval_secs(&rusage.ru_utime);
pinfo->stime = timeval_secs(&rusage.ru_stime);
#if !defined(__HAIKU__)
pinfo->page_faults = rusage.ru_majflt;
#endif
#if defined(__APPLE__)
pinfo->peak_rss = rusage.ru_maxrss; #else
pinfo->peak_rss = rusage.ru_maxrss * 1024; #endif
}
#if defined(__HAIKU__)
thread_info tid;
if (get_thread_info(find_thread(0), &tid) == B_OK) {
area_info mem;
ssize_t c;
while (get_next_area_info(tid.team, &c, &mem) == B_OK) {
pinfo->peak_rss += mem.ram_size;
}
}
pinfo->page_faults = 0;
#elif defined(__APPLE__)
#ifdef MACH_TASK_BASIC_INFO
struct mach_task_basic_info info;
mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
pinfo->current_rss = (size_t)info.resident_size;
}
#else
struct task_basic_info info;
mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;
if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
pinfo->current_rss = (size_t)info.resident_size;
}
#endif
#endif
}
#else
#ifndef __wasi__
#pragma message("define a way to get process info")
#endif
void _mi_prim_process_info(mi_process_info_t* pinfo)
{
MI_UNUSED(pinfo);
}
#endif
void _mi_prim_out_stderr( const char* msg ) {
fputs(msg,stderr);
}
#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0)
#if defined(__APPLE__) && defined(__has_include) && __has_include(<crt_externs.h>)
#include <crt_externs.h>
static char** mi_get_environ(void) {
return (*_NSGetEnviron());
}
#else
extern char** environ;
static char** mi_get_environ(void) {
return environ;
}
#endif
int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
return _mi_prim_getenv_by_visit(name, result, result_size);
}
bool _mi_prim_env_visit(mi_env_visit_fun* visit, void* arg) {
char** env = mi_get_environ();
if (env == NULL) return false;
for (int i = 0; i < 10000 && env[i] != NULL; i++) {
visit(env[i], arg);
}
return true;
}
#else
int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
if (_mi_preloading()) return -1; const char* s = getenv(name);
if (s == NULL) {
char buf[64+1];
size_t len = _mi_strnlen(name,sizeof(buf)-1);
for (size_t i = 0; i < len; i++) {
buf[i] = _mi_toupper(name[i]);
}
buf[len] = 0;
s = getenv(buf);
}
if (s == NULL) return 0; if (!_mi_strlcpy(result, s, result_size)) return -2; return 1; }
bool _mi_prim_env_visit(mi_env_visit_fun* visit, void* arg) {
MI_UNUSED(visit); MI_UNUSED(arg);
return false; }
#endif
#if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15)
#include <CommonCrypto/CommonCryptoError.h>
#include <CommonCrypto/CommonRandom.h>
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);
}
#elif defined(__ANDROID__) || defined(__DragonFly__) || \
defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
defined(__sun) || \
(defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7))
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
arc4random_buf(buf, buf_len);
return true;
}
#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) || defined(__CYGWIN__)
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom)
#ifndef GRND_NONBLOCK
#define GRND_NONBLOCK (1)
#endif
#ifndef GRND_INSECURE
#define GRND_INSECURE (4)
#endif
static _Atomic(size_t) no_getrandom; if (mi_atomic_load_acquire(&no_getrandom)==0) {
ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK);
if (ret < 0 && errno == EAGAIN) {
ret = syscall(SYS_getrandom, buf, buf_len, GRND_INSECURE);
}
if (ret > 0) { mi_msan_unpoison(buf, (size_t)ret); }
if (ret >= 0) return (buf_len == (size_t)ret);
if (errno == ENOSYS) { mi_atomic_store_release(&no_getrandom, (size_t)1); } }
#endif
int flags = O_RDONLY;
#if defined(O_CLOEXEC)
flags |= O_CLOEXEC;
#endif
int fd = mi_prim_open("/dev/urandom", flags);
if (fd < 0) return false;
size_t count = 0;
while(count < buf_len) {
ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);
if (ret==0) {
break;
}
else if (ret<0) {
if (errno!=EAGAIN && errno!=EINTR) break;
}
else {
count += ret;
}
}
mi_prim_close(fd);
return (count==buf_len);
}
#else
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
return false;
}
#endif
#if defined(MI_USE_PTHREADS)
static _Atomic(pthread_key_t) _mi_heap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID);
static void mi_pthread_done(void* value) {
if (value!=NULL) {
_mi_thread_done((mi_theap_t*)value);
}
}
void _mi_prim_thread_init_auto_done(void) {
mi_assert_internal(mi_atomic_load_relaxed(&_mi_heap_default_key) == MI_PTHREAD_KEY_INVALID);
pthread_key_t key;
if (pthread_key_create(&key, &mi_pthread_done) == 0) {
mi_atomic_store_release(&_mi_heap_default_key, key);
}
}
void _mi_prim_thread_done_auto_done(void) {
pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_heap_default_key, MI_PTHREAD_KEY_INVALID);
if (key != MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key);
}
}
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
pthread_key_t key = mi_atomic_load_relaxed(&_mi_heap_default_key);
if (key != MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap);
}
}
#else
void _mi_prim_thread_init_auto_done(void) {
}
void _mi_prim_thread_done_auto_done(void) {
}
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
MI_UNUSED(theap);
}
#endif
void _mi_prim_thread_init(void) {
#if MI_UNIX_LINUX_MTE
if (_mi_os_has_memtag()) {
const int ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
if (ctrl >= 0 && ((ctrl & PR_TAGGED_ADDR_ENABLE) == 0 || (ctrl & PR_MTE_TCF_MASK) == 0)) {
if (!unix_memtag_thread_enable((unsigned long)ctrl, (unsigned long)mi_process_get_sealed_once()->os_mem_config.memtag_mode)) {
unix_memtag_thread_error("unable to enable memory tagging on thread 0x%zx (error %d)\n", _mi_thread_id(), errno);
}
}
}
#endif
}
bool _mi_prim_thread_is_in_threadpool(void) {
return false;
}
void _mi_prim_thread_yield(void) {
sleep(0);
}
bool _mi_prim_process_is_killed(void) {
return false; }