#include "mimalloc.h"
#include "mimalloc/prim/prim.h"
#include "mimalloc/prim/os.h"
#include "mimalloc/util/libc.h"
#include "mimalloc/util/options.h"
#include "mimalloc/heap/process.h"
#include <stdio.h>
#include <stdlib.h>
#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM)
#define MI_WIN_DESKTOP 1
#endif
typedef enum MI_MEM_EXTENDED_PARAMETER_TYPE_E {
MiMemExtendedParameterInvalidType = 0,
MiMemExtendedParameterAddressRequirements,
MiMemExtendedParameterNumaNode,
MiMemExtendedParameterPartitionHandle,
MiMemExtendedParameterUserPhysicalHandle,
MiMemExtendedParameterAttributeFlags,
MiMemExtendedParameterMax
} MI_MEM_EXTENDED_PARAMETER_TYPE;
typedef struct DECLSPEC_ALIGN(8) MI_MEM_EXTENDED_PARAMETER_S {
struct { DWORD64 Type : 8; DWORD64 Reserved : 56; } Type;
union { DWORD64 ULong64; PVOID Pointer; SIZE_T Size; HANDLE Handle; DWORD ULong; } Arg;
} MI_MEM_EXTENDED_PARAMETER;
typedef struct MI_MEM_ADDRESS_REQUIREMENTS_S {
PVOID LowestStartingAddress;
PVOID HighestEndingAddress;
SIZE_T Alignment;
} MI_MEM_ADDRESS_REQUIREMENTS;
#define MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE 0x00000010
#include <winternl.h>
typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG);
typedef LONG (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, MI_MEM_EXTENDED_PARAMETER*, ULONG); static void* pVirtualAlloc2 = NULL; static void* pNtAllocateVirtualMemoryEx = NULL;
typedef struct MI_PROCESSOR_NUMBER_S { WORD Group; BYTE Number; BYTE Reserved; } MI_PROCESSOR_NUMBER;
typedef VOID (__stdcall *PGetCurrentProcessorNumberEx)(MI_PROCESSOR_NUMBER* ProcNumber);
typedef BOOL (__stdcall *PGetNumaProcessorNodeEx)(MI_PROCESSOR_NUMBER* Processor, PUSHORT NodeNumber);
typedef BOOL (__stdcall* PGetNumaNodeProcessorMaskEx)(USHORT Node, PGROUP_AFFINITY ProcessorMask);
typedef BOOL (__stdcall *PGetNumaProcessorNode)(UCHAR Processor, PUCHAR NodeNumber);
typedef BOOL (__stdcall* PGetNumaNodeProcessorMask)(UCHAR Node, PULONGLONG ProcessorMask);
typedef BOOL (__stdcall* PGetNumaHighestNodeNumber)(PULONG Node);
typedef BOOLEAN (__stdcall* PRtlDllShutdownInProgress)(VOID);
static void* pGetCurrentProcessorNumberEx = NULL; static void* pGetNumaProcessorNodeEx = NULL;
static void* pGetNumaNodeProcessorMaskEx = NULL;
static void* pGetNumaProcessorNode = NULL;
static void* pGetNumaNodeProcessorMask = NULL;
static void* pGetNumaHighestNodeNumber = NULL;
static void* pRtlDllShutdownInProgress = NULL;
typedef VOID (__stdcall* PRtlAcquirePebLock)(VOID);
typedef VOID (__stdcall* PRtlReleasePebLock)(VOID);
static void* pRtlAcquirePebLock = NULL; static void* pRtlReleasePebLock = NULL;
typedef SIZE_T(__stdcall* PGetLargePageMinimum)(VOID);
static void* pGetLargePageMinimum = NULL;
typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalMemoryInKilobytes );
static HMODULE mi_win_loadlibrary(const wchar_t* library) {
#if 1 return LoadLibraryW(library);
#else #endif
}
static HMODULE mi_win_getlibrary(const wchar_t* library, bool* should_free) {
#if MI_WIN_DESKTOP
HMODULE mod = GetModuleHandleW(library);
if (mod!=NULL) {
*should_free = false;
return mod;
}
#endif
*should_free = true;
return mi_win_loadlibrary(library);
}
static void mi_win_freelibrary(HMODULE mod, bool should_free) {
if (should_free) {
FreeLibrary(mod);
}
}
static void* mi_win_encode_pointer(void* p) {
return (p == NULL ? NULL : EncodePointer(p));
}
static void* mi_win_decode_pointer(void* p) {
return (p == NULL ? NULL : DecodePointer(p));
}
static void* mi_win_get_proc_encoded(HMODULE mod, const char* name) {
return mi_win_encode_pointer((void*)GetProcAddress(mod, name));
}
#define mi_win_call_proc_decoded(proc) ((P##proc)mi_win_decode_pointer(p##proc))
mi_decl_maybe_unused static bool win_version_at_least(ULONG major, ULONG minor) {
const uint8_t* const peb = (const uint8_t*)NtCurrentTeb()->ProcessEnvironmentBlock;
const ULONG* const version = (const ULONG*)(peb + (MI_SIZE_BITS == 32 ? 0xA4 : 0x118));
return (version[0] > major || (version[0] == major && version[1] >= minor));
}
static bool win_enable_large_os_pages_once(void)
{
if (pGetLargePageMinimum==NULL) return false;
unsigned long err = 0;
HANDLE token = NULL;
BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
err = GetLastError();
if (ok) {
TOKEN_PRIVILEGES tp;
ok = LookupPrivilegeValueW(NULL, L"SeLockMemoryPrivilege", &tp.Privileges[0].Luid);
err = GetLastError();
if (ok) {
tp.PrivilegeCount = 1;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);
err = GetLastError();
if (ok) {
ok = (err == ERROR_SUCCESS);
}
}
CloseHandle(token);
}
if (!ok) {
if (err == 0) { err = GetLastError(); }
_mi_warning_message("cannot enable large OS page support, error %lu\n", err);
}
return (ok!=0);
}
static bool win_enable_large_os_pages(void) {
size_t access = mi_atomic_load_acquire(&_mi_process.os.large_page_access);
if mi_unlikely(access == 0) {
access = (win_enable_large_os_pages_once() ? 1 : 2);
mi_atomic_store_release(&_mi_process.os.large_page_access, access);
}
return (access == 1);
}
void _mi_prim_mem_init( mi_os_mem_config_t* config )
{
config->has_overcommit = false;
config->has_partial_free = false;
config->has_virtual_reserve = true;
SYSTEM_INFO si; _mi_memzero_var(si);
GetSystemInfo(&si);
if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; }
if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; }
if ((uintptr_t)si.lpMaximumApplicationAddress > 0) {
const size_t vbits = MI_SIZE_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);
config->virtual_address_bits = vbits;
}
bool hDllFree;
HINSTANCE hDll = mi_win_getlibrary(L"kernelbase.dll", &hDllFree);
if (hDll != NULL) {
pVirtualAlloc2 = mi_win_get_proc_encoded(hDll, "VirtualAlloc2FromApp");
if (pVirtualAlloc2==NULL) pVirtualAlloc2 = mi_win_get_proc_encoded(hDll, "VirtualAlloc2");
mi_win_freelibrary(hDll, hDllFree);
}
config->has_aligned_alloc = (pVirtualAlloc2 != NULL);
hDll = mi_win_getlibrary(L"ntdll.dll", &hDllFree);
if (hDll != NULL) {
pNtAllocateVirtualMemoryEx = mi_win_get_proc_encoded(hDll, "NtAllocateVirtualMemoryEx");
pRtlDllShutdownInProgress = mi_win_get_proc_encoded(hDll, "RtlDllShutdownInProgress");
pRtlAcquirePebLock = mi_win_get_proc_encoded(hDll, "RtlAcquirePebLock");
pRtlReleasePebLock = mi_win_get_proc_encoded(hDll, "RtlReleasePebLock");
mi_win_freelibrary(hDll, hDllFree);
}
hDll = mi_win_getlibrary(L"kernel32.dll", &hDllFree);
if (hDll != NULL) {
pGetCurrentProcessorNumberEx = mi_win_get_proc_encoded(hDll, "GetCurrentProcessorNumberEx");
pGetNumaProcessorNodeEx = mi_win_get_proc_encoded(hDll, "GetNumaProcessorNodeEx");
pGetNumaNodeProcessorMaskEx = mi_win_get_proc_encoded(hDll, "GetNumaNodeProcessorMaskEx");
pGetNumaProcessorNode = mi_win_get_proc_encoded(hDll, "GetNumaProcessorNode");
pGetNumaNodeProcessorMask = mi_win_get_proc_encoded(hDll, "GetNumaNodeProcessorMask");
pGetNumaHighestNodeNumber = mi_win_get_proc_encoded(hDll, "GetNumaHighestNodeNumber");
pGetLargePageMinimum = mi_win_get_proc_encoded(hDll, "GetLargePageMinimum");
PGetPhysicallyInstalledSystemMemory pGetPhysicallyInstalledSystemMemory = (PGetPhysicallyInstalledSystemMemory)(void (*)(void))GetProcAddress(hDll,"GetPhysicallyInstalledSystemMemory");
if (pGetPhysicallyInstalledSystemMemory != NULL) {
ULONGLONG memInKiB = 0;
if ((*pGetPhysicallyInstalledSystemMemory)(&memInKiB)) {
if (memInKiB > 0 && memInKiB <= SIZE_MAX) {
config->physical_memory_in_kib = (size_t)memInKiB;
}
}
}
mi_win_freelibrary(hDll, hDllFree);
}
if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
if (win_enable_large_os_pages()) { config->large_page_size = mi_win_call_proc_decoded(GetLargePageMinimum)(); }
}
}
int _mi_prim_free(void* addr, size_t size ) {
MI_UNUSED(size);
DWORD errcode = 0;
bool err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
if (err) { errcode = GetLastError(); }
if (errcode == ERROR_INVALID_ADDRESS) {
MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info);
err = (VirtualQuery(addr, &info, sizeof(info)) == 0);
if (err) { errcode = GetLastError(); }
if (!err && info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) {
errcode = 0;
err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0);
if (err) { errcode = GetLastError(); }
}
}
return (int)errcode;
}
static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {
#if (MI_SIZE_SIZE >= 8)
if (addr == NULL) {
void* hint = _mi_os_get_aligned_hint(try_alignment,size);
if (hint != NULL) {
void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE);
if (p != NULL) return p;
_mi_verbose_message("warning: unable to allocate hinted aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), hint, try_alignment, flags);
}
}
#endif
if (addr == NULL && try_alignment > _mi_os_alloc_granularity() && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };
reqs.Alignment = try_alignment;
MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };
param.Type.Type = MiMemExtendedParameterAddressRequirements;
param.Arg.Pointer = &reqs;
void* p = mi_win_call_proc_decoded(VirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1);
if (p != NULL) return p;
_mi_warning_message("unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
}
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
}
static bool win_is_out_of_memory_error(DWORD err) {
switch (err) {
case ERROR_COMMITMENT_MINIMUM:
case ERROR_COMMITMENT_LIMIT:
case ERROR_PAGEFILE_QUOTA:
case ERROR_NOT_ENOUGH_MEMORY:
return true;
default:
return false;
}
}
static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000); if (max_retry_msecs == 1) { max_retry_msecs = 100; } for (long tries = 1; tries <= 10; tries++) { void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags);
if (p != NULL) {
return p;
}
else if (max_retry_msecs > 0 && (try_alignment <= 8*MI_MiB) &&
(flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 &&
win_is_out_of_memory_error(GetLastError())) {
_mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags);
long sleep_msecs = tries*40; if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; }
max_retry_msecs -= sleep_msecs;
Sleep(sleep_msecs);
}
else {
break;
}
}
return NULL;
}
static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
mi_assert_internal(!(large_only && !allow_large));
void* p = NULL;
if ((large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))
&& allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0)
{
size_t backoff = mi_atomic_load_relaxed(&_mi_process.os.large_page_backoff);
if (!large_only && backoff > 0) {
mi_atomic_cas_strong_relaxed(&_mi_process.os.large_page_backoff, &backoff, backoff - 1);
}
else {
*is_large = true;
p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES);
if (large_only) return p;
if (p == NULL) {
mi_atomic_store_relaxed(&_mi_process.os.large_page_backoff, (size_t)10); }
}
}
if (p == NULL) {
*is_large = ((flags&MEM_LARGE_PAGES) != 0);
p = win_virtual_alloc_prim(addr, size, try_alignment, flags);
}
return p;
}
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(commit || !allow_large);
mi_assert_internal(try_alignment > 0);
*is_zero = true;
int flags = MEM_RESERVE;
if (commit) { flags |= MEM_COMMIT; }
*addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large);
return (*addr != NULL ? 0 : (int)GetLastError());
}
#ifdef _MSC_VER
#pragma warning(disable:6250)
#endif
int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
*is_zero = false;
void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
if (p == NULL) return (int)GetLastError();
return 0;
}
int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {
BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT);
*needs_recommit = true; return (ok ? 0 : (int)GetLastError());
}
int _mi_prim_reset(void* addr, size_t size) {
void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
mi_assert_internal(p == addr);
#if 0 #endif
return (p != NULL ? 0 : (int)GetLastError());
}
int _mi_prim_reuse(void* addr, size_t size) {
MI_UNUSED(addr); MI_UNUSED(size);
return 0;
}
int _mi_prim_protect(void* addr, size_t size, mi_protect_t protect) {
const ULONG prot = (protect == MI_PROTECT_NONE ? PAGE_NOACCESS : (protect == MI_PROTECT_READWRITE ? PAGE_READWRITE : PAGE_READONLY));
#if !MI_WIN_DESKTOP && WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)
ULONG oldprotect = 0;
BOOL ok = VirtualProtectFromApp(addr, size, prot, &oldprotect);
#else
DWORD oldprotect = 0;
BOOL ok = VirtualProtect(addr, size, prot, &oldprotect);
#endif
return (ok ? 0 : (int)GetLastError());
}
int _mi_prim_guard(void* addr, size_t size, bool set_guard, bool is_committed) {
if (is_committed) {
if (set_guard) { VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); }
return _mi_prim_protect(addr, size, (set_guard ? MI_PROTECT_NONE : MI_PROTECT_READWRITE));
}
else if (set_guard) {
bool needs_recommit;
return _mi_prim_decommit(addr, size, &needs_recommit);
}
else {
return 0; }
}
static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node)
{
const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE;
if (!win_enable_large_os_pages()) {
SetLastError(ERROR_PRIVILEGE_NOT_HELD);
return NULL;
}
MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };
if (pNtAllocateVirtualMemoryEx != NULL && mi_atomic_load_acquire(&_mi_process.os.huge_page_unavailable) == 0) {
params[0].Type.Type = MiMemExtendedParameterAttributeFlags;
params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
ULONG param_count = 1;
if (numa_node >= 0) {
param_count++;
params[1].Type.Type = MiMemExtendedParameterNumaNode;
params[1].Arg.ULong = (unsigned)numa_node;
}
SIZE_T psize = size;
void* base = hint_addr;
LONG err = mi_win_call_proc_decoded(NtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, params, param_count);
if (err == 0 && base != NULL) {
return base;
}
else {
mi_atomic_store_release(&_mi_process.os.huge_page_unavailable, (size_t)1); _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err);
}
}
if (pVirtualAlloc2 != NULL && numa_node >= 0) {
params[0].Type.Type = MiMemExtendedParameterNumaNode;
params[0].Arg.ULong = (unsigned)numa_node;
return mi_win_call_proc_decoded(VirtualAlloc2)(GetCurrentProcess(), hint_addr, size, flags, PAGE_READWRITE, params, 1);
}
return VirtualAlloc(hint_addr, size, flags, PAGE_READWRITE);
}
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
*is_zero = true;
*addr = _mi_prim_alloc_huge_os_pagesx(hint_addr,size,numa_node);
return (*addr != NULL ? 0 : (int)GetLastError());
}
size_t _mi_prim_numa_node(void) {
USHORT numa_node = 0;
if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) {
MI_PROCESSOR_NUMBER pnum;
mi_win_call_proc_decoded(GetCurrentProcessorNumberEx)(&pnum);
USHORT nnode = 0;
BOOL ok = mi_win_call_proc_decoded(GetNumaProcessorNodeEx)(&pnum, &nnode);
if (ok) { numa_node = nnode; }
}
else if (pGetNumaProcessorNode != NULL) {
DWORD pnum = GetCurrentProcessorNumber();
UCHAR nnode = 0;
BOOL ok = mi_win_call_proc_decoded(GetNumaProcessorNode)((UCHAR)pnum, &nnode);
if (ok) { numa_node = nnode; }
}
return numa_node;
}
size_t _mi_prim_numa_node_count(void) {
ULONG numa_max = 0;
if (pGetNumaHighestNodeNumber!=NULL) {
mi_win_call_proc_decoded(GetNumaHighestNodeNumber)(&numa_max);
}
while (numa_max > 0) {
if (pGetNumaNodeProcessorMaskEx != NULL) {
GROUP_AFFINITY affinity;
if (mi_win_call_proc_decoded(GetNumaNodeProcessorMaskEx)((USHORT)numa_max, &affinity)) {
if (affinity.Mask != 0) break; }
}
else {
ULONGLONG mask;
if (pGetNumaNodeProcessorMask != NULL) {
if (mi_win_call_proc_decoded(GetNumaNodeProcessorMask)((UCHAR)numa_max, &mask)) {
if (mask != 0) break; }
};
}
numa_max--;
}
return ((size_t)numa_max + 1);
}
static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {
int64_t ticks_per_msec = mi_process_get_sealed_once()->clock_ticks_per_msec;
if (ticks_per_msec == 0) {
LARGE_INTEGER f;
if (QueryPerformanceFrequency(&f)) {
ticks_per_msec = f.QuadPart/1000LL;
}
if (ticks_per_msec == 0) { ticks_per_msec = 1; }
((mi_process_sealed_once_t*)mi_process_get_sealed_once())->clock_ticks_per_msec = ticks_per_msec; }
return (mi_msecs_t)(t.QuadPart / ticks_per_msec);
}
mi_msecs_t _mi_prim_clock_now(void) {
LARGE_INTEGER t;
if (!QueryPerformanceCounter(&t)) { t.QuadPart = 0; }
return mi_to_msecs(t); }
#include <psapi.h>
static mi_msecs_t filetime_msecs(const FILETIME* ftime) {
ULARGE_INTEGER i;
i.LowPart = ftime->dwLowDateTime;
i.HighPart = ftime->dwHighDateTime;
mi_msecs_t msecs = (i.QuadPart / 10000); return msecs;
}
typedef BOOL (WINAPI *PGetProcessMemoryInfo)(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD);
static void* pGetProcessMemoryInfo = NULL;
void _mi_prim_process_info(mi_process_info_t* pinfo)
{
FILETIME ct;
FILETIME ut;
FILETIME st;
FILETIME et;
if (GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut)) {
pinfo->utime = filetime_msecs(&ut);
pinfo->stime = filetime_msecs(&st);
}
mi_atomic_do_once{
HINSTANCE hDll = mi_win_loadlibrary(L"psapi.dll");
if (hDll != NULL) {
pGetProcessMemoryInfo = mi_win_get_proc_encoded(hDll, "GetProcessMemoryInfo");
}
}
if (pGetProcessMemoryInfo != NULL) {
PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);
if (mi_win_call_proc_decoded(GetProcessMemoryInfo)(GetCurrentProcess(), &info, sizeof(info))) {
pinfo->current_rss = (size_t)info.WorkingSetSize;
pinfo->peak_rss = (size_t)info.PeakWorkingSetSize;
pinfo->current_commit = (size_t)info.PagefileUsage;
pinfo->peak_commit = (size_t)info.PeakPagefileUsage;
pinfo->page_faults = (size_t)info.PageFaultCount;
}
}
}
void _mi_prim_out_stderr( const char* msg )
{
if (!_mi_preloading()) {
static HANDLE hcon = INVALID_HANDLE_VALUE;
#if MI_WIN_DESKTOP
static bool hconIsConsole = false;
#endif
if (hcon == INVALID_HANDLE_VALUE) {
hcon = GetStdHandle(STD_ERROR_HANDLE); #if MI_WIN_DESKTOP
CONSOLE_SCREEN_BUFFER_INFO sbi;
hconIsConsole = ((hcon != NULL && hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi));
#endif
}
const size_t len = _mi_strlen(msg);
if (len > 0 && len < UINT32_MAX) {
DWORD written = 0;
if (hcon != NULL && hcon != INVALID_HANDLE_VALUE) {
#if MI_WIN_DESKTOP
if (hconIsConsole) {
WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL);
}
else
#endif
{
WriteFile(hcon, msg, (DWORD)len, &written, NULL);
}
}
#if MI_WIN_DESKTOP
else if (hcon == NULL) {
OutputDebugStringA(msg);
}
#endif
}
}
}
int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
result[0] = 0;
const size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);
return (len < result_size ? (len > 0 ? 1 : 0 ) : -2 );
}
bool _mi_prim_env_visit(mi_env_visit_fun* visit, void* arg) {
if (pRtlAcquirePebLock == NULL || pRtlReleasePebLock == NULL) return false;
char entry[512];
mi_win_call_proc_decoded(RtlAcquirePebLock)();
const RTL_USER_PROCESS_PARAMETERS* const params = NtCurrentTeb()->ProcessEnvironmentBlock->ProcessParameters;
const WCHAR* const env = (params == NULL ? NULL : *(const WCHAR* const*)((const uint8_t*)¶ms->CommandLine + sizeof(UNICODE_STRING)));
if (env != NULL) {
int entries = 0;
for (const WCHAR* s = env; *s != 0 && entries < 10000; entries++) {
size_t len = 0;
while (s[len] != 0 && len < 32*MI_KiB) { len++; } if (s[len] != 0) break; const int n = WideCharToMultiByte(CP_ACP, 0, s, (int)len + 1, entry, (int)sizeof(entry), NULL, NULL);
if (n > 0) { visit(entry, arg); }
s += len + 1;
}
}
mi_win_call_proc_decoded(RtlReleasePebLock)();
return (env != NULL);
}
#if defined(MI_USE_RTLGENRANDOM) || MI_WIN_DESKTOP
#if defined(_MSC_VER)
#pragma comment (lib,"advapi32.lib")
#endif
#define RtlGenRandom SystemFunction036
mi_decl_externc BOOLEAN NTAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength);
static bool mi_win_rtl_gen_random(void* buf, size_t buf_len) {
mi_assert(buf_len <= ULONG_MAX);
if (buf_len > ULONG_MAX) return false;
return (RtlGenRandom(buf, (ULONG)buf_len) != 0);
}
#endif
#if defined(MI_USE_RTLGENRANDOM)
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
return mi_win_rtl_gen_random(buf, buf_len);
}
#elif !MI_WIN_DESKTOP
#include <bcrypt.h>
#if defined(_MSC_VER)
#pragma comment (lib,"bcrypt.lib")
#endif
#ifndef BCRYPT_USE_SYSTEM_PREFERRED_RNG
#define BCRYPT_USE_SYSTEM_PREFERRED_RNG 0x00000002
#endif
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
mi_assert(buf_len <= ULONG_MAX);
if (buf_len > ULONG_MAX) return false;
return (BCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0);
}
#else
typedef BOOL (WINAPI *PProcessPrng)(PBYTE pbData, SIZE_T cbData);
static void* pProcessPrng = NULL;
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
mi_atomic_do_once {
if (win_version_at_least(6,2)) { bool should_free = false;
HMODULE hDll = mi_win_getlibrary(L"bcryptprimitives.dll", &should_free);
if (hDll != NULL) {
pProcessPrng = mi_win_get_proc_encoded(hDll, "ProcessPrng");
if (pProcessPrng == NULL) { mi_win_freelibrary(hDll, should_free); } }
}
}
if (pProcessPrng != NULL) {
return (mi_win_call_proc_decoded(ProcessPrng)((PBYTE)buf, buf_len) != FALSE);
}
return mi_win_rtl_gen_random(buf, buf_len);
}
#endif
void _mi_prim_thread_init(void) {
}
bool _mi_prim_thread_is_in_threadpool(void) {
#if (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64)
if (win_version_at_least(6,0)) { struct _TEB* const teb = NtCurrentTeb();
void* const pool_data = *((void**)((uint8_t*)teb + (MI_SIZE_BITS == 32 ? 0x0F90 : 0x1778)));
return (pool_data != NULL);
}
#endif
return false;
}
void _mi_prim_thread_yield(void) {
SwitchToThread();
}
bool _mi_prim_process_is_killed(void) {
if (pRtlDllShutdownInProgress != NULL) {
return mi_win_call_proc_decoded(RtlDllShutdownInProgress)();
}
else {
return false;
}
}
static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
MI_UNUSED(reserved);
MI_UNUSED(module);
if (reason==DLL_PROCESS_ATTACH) {
_mi_auto_process_init();
}
else if (reason==DLL_PROCESS_DETACH) {
_mi_auto_process_done();
}
else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) {
_mi_thread_done(NULL);
}
}
#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS)
#if defined(__INTEL_LLVM_COMPILER) || defined(__INTEL_COMPILER)
#define MI_WIN_INIT_USE_TLS_DLLMAIN 1
#else
#define MI_WIN_INIT_USE_CRT_TLS 1
#endif
#endif
#if defined(MI_WIN_INIT_USE_CRT_TLS)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
void _mi_prim_thread_init_auto_done(void) {}
void _mi_prim_thread_done_auto_done(void) {}
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
MI_UNUSED(theap);
}
static mi_decl_noinline bool mi_module_is_dll(PVOID mod) {
if (mod==NULL) return false;
PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;
PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);
return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);
}
#if defined(__cplusplus)
extern "C" {
#endif
extern IMAGE_DOS_HEADER __ImageBase; #if defined(__cplusplus)
}
#endif
static bool mi_current_module_is_dll(void) {
const HMODULE mod = (HMODULE)(&__ImageBase);
return mi_module_is_dll(mod);
}
static void mi_cdecl mi_crt_done(void) {
mi_win_main(NULL, DLL_PROCESS_DETACH, 0);
}
static int mi_cdecl mi_crt_init(void) {
if (mi_current_module_is_dll()) {
atexit(&mi_crt_done);
}
return 0;
}
static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_THREAD_ATTACH) {
mi_win_main(module, reason, reserved);
}
else if (reason == DLL_PROCESS_ATTACH) {
mi_win_main(module, reason, reserved);
}
}
static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_THREAD_DETACH) {
mi_win_main(module, reason, reserved);
}
else if (reason == DLL_PROCESS_DETACH) {
if (!mi_module_is_dll(module)) {
mi_win_main(module, reason, reserved);
}
}
}
#if defined(__cplusplus)
extern "C" {
#endif
typedef int (mi_cdecl* mi_crt_callback_t)(void);
#if (defined(_WIN64) || defined(_M_ARM)) && defined(_MSC_VER) #pragma comment(linker, "/INCLUDE:_tls_used")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
#pragma comment(linker, "/INCLUDE:_mi_crt_callback_init")
#pragma const_seg(".CRT$XLB")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
#pragma const_seg()
#pragma const_seg(".CRT$XLY")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
#pragma const_seg()
#pragma const_seg(".CRT$XIB")
extern const mi_crt_callback_t _mi_crt_callback_init[];
const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
#pragma const_seg()
#elif defined(_MSC_VER)
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
#pragma comment(linker, "/INCLUDE:__mi_crt_callback_init")
#pragma data_seg(".CRT$XLB")
PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
#pragma data_seg()
#pragma data_seg(".CRT$XLY")
PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
#pragma data_seg()
#pragma data_seg(".CRT$XIB")
mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
#pragma data_seg()
#elif defined(__MINGW32__)
extern const IMAGE_TLS_DIRECTORY _tls_used;
__attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach;
__attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach;
__attribute__((used, section(".CRT$XIB"))) mi_crt_callback_t _mi_crt_callback_init = &mi_crt_init;
#endif
#if defined(__cplusplus)
}
#endif
#elif defined(MI_WIN_INIT_USE_RAW_DLLMAIN)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
void _mi_prim_thread_init_auto_done(void) {}
void _mi_prim_thread_done_auto_done(void) {}
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
MI_UNUSED(theap);
}
static BOOL NTAPI mi_dll_main_raw(PVOID module, DWORD reason, LPVOID reserved) {
mi_win_main(module, reason, reserved);
return TRUE;
}
#if defined(__cplusplus)
extern "C"
#endif
PVOID _pRawDllMain = (PVOID)&mi_dll_main_raw;
static bool mi_module_is_dll(PVOID mod) {
if (mod==NULL) return false;
PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;
PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);
return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);
}
static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {
if (!mi_module_is_dll(module)) {
mi_win_main(module, reason, reserved);
}
}
}
static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {
if (!mi_module_is_dll(module)) {
mi_win_main(module, reason, reserved);
}
}
}
#if defined(__cplusplus)
extern "C" {
#endif
#if (defined(_WIN64) || defined(_M_ARM)) && defined(_MSC_VER) #pragma comment(linker, "/INCLUDE:_tls_used")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
#pragma const_seg(".CRT$XLB")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
#pragma const_seg()
#pragma const_seg(".CRT$XLY")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
#pragma const_seg()
#elif defined(_MSC_VER)
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
#pragma data_seg(".CRT$XLB")
PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
#pragma data_seg()
#pragma data_seg(".CRT$XLY")
PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
#pragma data_seg()
#elif defined(__MINGW32__)
extern const IMAGE_TLS_DIRECTORY _tls_used;
__attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach;
__attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach;
#endif
#if defined(__cplusplus)
}
#endif
#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && defined(MI_SHARED_LIB)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
mi_win_main((PVOID)inst,reason,reserved);
return TRUE;
}
void _mi_prim_thread_init_auto_done(void) { }
void _mi_prim_thread_done_auto_done(void) { }
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
MI_UNUSED(theap);
}
#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {
mi_win_main(module, reason, reserved);
}
}
static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {
mi_win_main(module, reason, reserved);
}
}
#if defined(__cplusplus)
extern "C" {
#endif
#if (defined(_WIN64) || defined(_M_ARM)) && defined(_MSC_VER) #pragma comment(linker, "/INCLUDE:_tls_used")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
#pragma const_seg(".CRT$XLB")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
#pragma const_seg()
#pragma const_seg(".CRT$XLY")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
#pragma const_seg()
#elif defined(_MSC_VER)
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
#pragma data_seg(".CRT$XLB")
PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
#pragma data_seg()
#pragma data_seg(".CRT$XLY")
PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
#pragma data_seg()
#elif defined(__MINGW32__)
extern const IMAGE_TLS_DIRECTORY _tls_used;
__attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_win_main_attach;
__attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_win_main_detach;
#endif
#if defined(__cplusplus)
}
#endif
void _mi_prim_thread_init_auto_done(void) { }
void _mi_prim_thread_done_auto_done(void) { }
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
MI_UNUSED(theap);
}
#elif defined(MI_WIN_INIT_USE_FLS)
#if defined(_MSC_VER)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
static int mi_cdecl mi_crt_init(void) {
mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL);
atexit(&_mi_auto_process_done);
return 0;
}
#if defined(__cplusplus)
extern "C" {
#endif
typedef int (mi_cdecl* mi_crt_callback_t)(void);
#if defined(_WIN64) #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init")
#pragma const_seg(".CRT$XIU")
extern const mi_crt_callback_t _mi_crt_callback_init[];
const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
#pragma const_seg()
#else
#pragma comment(linker, "/INCLUDE:__mi_crt_callback_init")
#pragma data_seg(".CRT$XIU")
mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
#pragma data_seg()
#endif
#if defined(__cplusplus)
}
#endif
#endif
#include <fibersapi.h>
#if (_WIN32_WINNT < 0x600)
WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );
WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );
WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );
WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);
#endif
static DWORD mi_fls_key = (DWORD)(-1);
static void NTAPI mi_fls_done(PVOID value) {
mi_theap_t* theap = (mi_theap_t*)value;
if (theap != NULL) {
_mi_thread_done(theap);
FlsSetValue(mi_fls_key, NULL); }
}
void _mi_prim_thread_init_auto_done(void) {
mi_fls_key = FlsAlloc(&mi_fls_done);
}
void _mi_prim_thread_done_auto_done(void) {
FlsFree(mi_fls_key);
}
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
mi_assert_internal(mi_fls_key != (DWORD)(-1));
FlsSetValue(mi_fls_key, theap);
}
#else
#error "define windows process and thread auto initialization"
#endif
#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT)
#define MI_PRIM_HAS_ALLOCATOR_INIT 1
bool _mi_is_redirected(void) {
return _mi_process.malloc_is_redirected;
}
#ifdef __cplusplus
extern "C" {
#endif
mi_decl_export void _mi_redirect_entry(DWORD reason) {
if (reason == DLL_PROCESS_ATTACH) {
_mi_process.malloc_is_redirected = true;
}
else if (reason == DLL_PROCESS_DETACH) {
_mi_process.malloc_is_redirected = false;
}
else if (reason == DLL_THREAD_DETACH) {
_mi_thread_done(NULL);
}
}
__declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message);
__declspec(dllimport) void mi_cdecl mi_allocator_done(void);
#ifdef __cplusplus
}
#endif
bool _mi_allocator_init(const char** message) {
return mi_allocator_init(message);
}
void _mi_allocator_done(void) {
mi_allocator_done();
}
#endif