vey-mimalloc-sys 0.3.2+3.7.0

Rust FFI binding to mimalloc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
/* ----------------------------------------------------------------------------
Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/

// This file is included in `src/prim/prim.c`

#ifndef _DEFAULT_SOURCE
#define _DEFAULT_SOURCE   // ensure mmap flags and syscall are defined
#endif

#if defined(__sun)
// illumos provides new mman.h api when any of these are defined
// otherwise the old api based on caddr_t which predates the void pointers one.
// stock solaris provides only the former, chose to atomically to discard those
// flags only here rather than project wide tough.
#undef _XOPEN_SOURCE
#undef _POSIX_C_SOURCE
#endif

#include "mimalloc.h"
#include "mimalloc/prim/prim.h"
#include "mimalloc/track.h"
#include "mimalloc/prim/os.h"
#include "mimalloc/util/libc.h"
#include "mimalloc/util/options.h"
#include "mimalloc/heap/process.h"
#include "mimalloc/prim/prim-tls.h"  // _mi_thread_id

#include <sys/mman.h>  // mmap
#include <unistd.h>    // sysconf, sleep
#include <fcntl.h>     // open, close, read, access
#include <stdlib.h>    // getenv, arc4random_buf

#if defined(__linux__)
  #include <features.h>
  #include <sys/prctl.h>    // THP disable, PR_SET_VMA
  #include <sys/sysinfo.h>  // sysinfo
  #if defined(__GLIBC__) && !defined(PR_SET_VMA)
  #include <linux/prctl.h>
  #endif
  #if defined(__GLIBC__)
  #include <linux/mman.h>   // linux mmap flags
  #endif
  #if defined(__riscv) || defined(_M_RISCV)
    #if defined(MI_HAS_SYS_HWPROBEH)
      #include <sys/hwprobe.h>
    #elif defined(MI_HAS_ASM_HWPROBEH)
      #include <asm/hwprobe.h>
    #endif
  #endif
  #if MI_PAC >= 2 && defined(__aarch64__) && !defined(__APPLE__)
    #include <sys/auxv.h>   // getauxval
    #define MI_UNIX_LINUX_PAC  1
    #ifndef HWCAP_PACG
    #define HWCAP_PACG  (1UL << 31)
    #endif
  #endif
  #if MI_MTE && defined(__aarch64__)
    #include <sys/auxv.h>   // getauxval
    #define MI_UNIX_LINUX_MTE  1
    #ifndef HWCAP2_MTE
    #define HWCAP2_MTE  (1UL << 18)
    #endif
    #ifndef PROT_MTE
    #define PROT_MTE    (0x20)
    #endif
    #ifndef PR_SET_TAGGED_ADDR_CTRL
    #define PR_SET_TAGGED_ADDR_CTRL  55
    #define PR_GET_TAGGED_ADDR_CTRL  56
    #define PR_TAGGED_ADDR_ENABLE    (1UL << 0)
    #endif
    #ifndef PR_MTE_TCF_SHIFT
    #define PR_MTE_TCF_SHIFT         1
    #define PR_MTE_TCF_SYNC          (1UL << PR_MTE_TCF_SHIFT)
    #define PR_MTE_TCF_ASYNC         (2UL << PR_MTE_TCF_SHIFT)
    #define PR_MTE_TCF_MASK          (3UL << PR_MTE_TCF_SHIFT)
    #define PR_MTE_TAG_SHIFT         3
    #define PR_MTE_TAG_MASK          (0xFFFFUL << PR_MTE_TAG_SHIFT)
    #endif
  #endif
#elif defined(__APPLE__)
  #include <AvailabilityMacros.h>
  #include <TargetConditionals.h>
  #if !defined(TARGET_OS_OSX) || TARGET_OS_OSX || MI_MTE  // see issue #879 (on iOS only included for memory tagging)
  #include <mach/vm_statistics.h>    // VM_MAKE_TAG, VM_FLAGS_SUPERPAGE_SIZE_2MB, VM_FLAGS_MTE, etc.
  #endif
  #if !defined(MAC_OS_X_VERSION_10_7)
  #define MAC_OS_X_VERSION_10_7   1070
  #endif
  #include <sys/sysctl.h>
  #if MI_MTE && defined(VM_FLAGS_MTE) && __has_include(<os/security_config.h>)
  #include <os/security_config.h>   // os_security_config_get
  #include <mach/mach.h>
  #include <mach/vm_map.h>          // vm_map (as `mach/mach_vm.h` is not supported on iOS)
  #define MI_UNIX_MACH_MTE  1
  #endif
#elif defined(__FreeBSD__) || defined(__DragonFly__)
  #include <sys/param.h>
  #if __FreeBSD_version >= 1200000
  #include <sys/cpuset.h>
  #include <sys/domainset.h>
  #endif
  #include <sys/sysctl.h>
#endif

#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__FreeBSD__)
  #define MI_HAS_SYSCALL_H
  #include <sys/syscall.h>
#endif

// MSan does not intercept raw syscalls, so memory they write must be marked as initialized
#if defined(__has_feature)
  #if __has_feature(memory_sanitizer)
  #include <sanitizer/msan_interface.h>  // __msan_unpoison
  #define mi_msan_unpoison(p,size)  __msan_unpoison(p,size)
  #endif
#endif
#ifndef mi_msan_unpoison
#define mi_msan_unpoison(p,size)
#endif

#if !defined(MADV_DONTNEED) && defined(POSIX_MADV_DONTNEED)  // QNX
#define MADV_DONTNEED  POSIX_MADV_DONTNEED
#endif
#if !defined(MADV_FREE) && defined(POSIX_MADV_FREE)  // QNX
#define MADV_FREE  POSIX_MADV_FREE
#endif

#define MI_UNIX_MAX_THP_PAGE_SIZE (8*MI_MiB)        // above this we do not use madvise for transparent huge pages (see issue #1431)

//------------------------------------------------------------------------------------
// Use syscalls for some primitives to allow for libraries that override open/read/close etc.
// and do allocation themselves; using syscalls prevents recursion when mimalloc is
// still initializing (issue #713)
// Not used on Android, see issue python/cpython/pull/158530
// Declare inline to avoid unused function warnings.
//------------------------------------------------------------------------------------

#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)  && !defined(__ANDROID__)

static inline int mi_prim_open(const char* fpath, int open_flags) {
  return syscall(SYS_open,fpath,open_flags,0);
}
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
  ssize_t nread = syscall(SYS_read,fd,buf,bufsize);
  if (nread > 0) { mi_msan_unpoison(buf, (size_t)nread); }
  return nread;
}
static inline int mi_prim_close(int fd) {
  return syscall(SYS_close,fd);
}
static inline int mi_prim_access(const char *fpath, int mode) {
  return syscall(SYS_access,fpath,mode);
}

#else

static inline int mi_prim_open(const char* fpath, int open_flags) {
  return open(fpath,open_flags);
}
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
  return read(fd,buf,bufsize);
}
static inline int mi_prim_close(int fd) {
  return close(fd);
}
static inline int mi_prim_access(const char *fpath, int mode) {
  return access(fpath,mode);
}

#endif



//---------------------------------------------
// init
//---------------------------------------------

static bool unix_detect_overcommit(void) {
  bool os_overcommit = true;
  #if defined(__linux__)
    int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY);
    if (fd >= 0) {
      char buf[64];
      ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
      mi_prim_close(fd);
      // <https://www.kernel.org/doc/Documentation/vm/overcommit-accounting>
      // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE)
      if (nread >= 1 && nread < (ssize_t)sizeof(buf)) {
        buf[nread] = 0;
        os_overcommit = (buf[0] == '0' || buf[0] == '1');
      }
    }
  #elif defined(__FreeBSD__)
    int val = 0;
    size_t olen = sizeof(val);
    if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) {
      os_overcommit = (val != 0);
    }
  #else
    // default: overcommit is true
  #endif
  return os_overcommit;
}

static bool unix_detect_thp(void) {
  bool thp_enabled = false;
  #if defined(__linux__)
  int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
  if (fd >= 0) {
    char buf[64];
    ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
    mi_prim_close(fd);
    // <https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html>
    // between brackets is the current value, for example: always [madvise] never
    if (nread >= 1 && nread < (ssize_t)sizeof(buf)) {
      buf[nread] = 0;
      thp_enabled = (_mi_strnstr(buf,nread,"[never]") == NULL);
    }
  }
  #endif
  return thp_enabled;
}

static size_t unix_detect_large_page_size(void) {
  size_t large_page_size = 2*MI_MiB;  // assume 512*4KiB by default
  #if defined(__linux__)
  // get the transparent huge page size from the pmd size
  const int fd = mi_prim_open("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY);
  if (fd >= 0) {
    char buf[64];
    const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
    mi_prim_close(fd);
    if (nread >= 1 && nread < (ssize_t)sizeof(buf)) {
      buf[nread] = 0;
      long size = 0;
      if (_mi_strntol(buf, nread, NULL, &size) && size > 0 && (unsigned long)size <= SIZE_MAX) {
        large_page_size = (size_t)size;
      }
    }
  }
  #endif
  return large_page_size;
}

// try to detect the physical memory dynamically (if possible)
static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) {
  #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE))  // freeBSD, macOS
    MI_UNUSED(page_size);
    int64_t physical_memory = 0;
    size_t length = sizeof(int64_t);
    #if defined(HW_PHYSMEM64)
    int mib[2] = { CTL_HW, HW_PHYSMEM64 };
    #else
    int mib[2] = { CTL_HW, HW_MEMSIZE };
    #endif
    const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0);
    if (err==0 && physical_memory > 0) {
      const int64_t phys_in_kib = physical_memory / MI_KiB;
      if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) {
        *physical_memory_in_kib = (size_t)phys_in_kib;
      }
    }
  #elif defined(__linux__)
    MI_UNUSED(page_size);
    struct sysinfo info; _mi_memzero_var(info);
    const int err = sysinfo(&info);
    if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) {
      if (info.mem_unit==MI_KiB) {
        *physical_memory_in_kib = (size_t)info.totalram;
      }
      else {
        size_t total = 0;
        if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) {
          *physical_memory_in_kib = (total / MI_KiB);
        }
      }
    }
  #elif defined(_SC_PHYS_PAGES)  // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100)
    const long pphys = sysconf(_SC_PHYS_PAGES);
    const size_t psize_in_kib = page_size / MI_KiB;
    if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) {
      *physical_memory_in_kib = (size_t)pphys * psize_in_kib;
    }
  #endif
}

// Detect the virtual address bits (currently Linux/RISC-V only)
static size_t unix_detect_virtual_address_bits(void) {
  #if defined(__riscv) || defined(_M_RISCV)
    #if defined(RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS)
      struct riscv_hwprobe probe = { .key = RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS, };
      // Prefer the GNU libc interface if available, as it can also use the VDSO
      #if defined(MI_HAS_SYS_HWPROBEH)
      if (__riscv_hwprobe(&probe, 1, 0, NULL, 0) == 0)
      #else
      if (syscall(__NR_riscv_hwprobe, &probe, 1, 0, NULL, 0) == 0)
      #endif
      {
        if (probe.key != -1) { // If a key is unknown to the kernel, its key field will be cleared to -1.
          return (MI_SIZE_BITS - mi_clz((size_t)probe.value));
        }
      }
    #endif
    // Fallback to checking /proc/cpuinfo for older kernels
    const int fd = mi_prim_open("/proc/cpuinfo", O_RDONLY);
    if (fd >= 0) {
      char buf[2048];
      const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
      mi_prim_close(fd);
      if ((nread >= 1) && (nread < (ssize_t)sizeof(buf))) {
        buf[nread] = 0;
        if (_mi_strnstr(buf, nread, "sv39")) { return 39; }
        else if (_mi_strnstr(buf, nread, "sv48")) { return 48; }
        else if (_mi_strnstr(buf, nread, "sv57")) { return 57; }
      }
    }
  #endif // riscv
  // default
  return MI_MAX_VABITS;
}

#if MI_UNIX_LINUX_MTE
// With `-DMI_MTE=ON` it is a fatal error if tag checking cannot be enabled for all threads
#if MI_MTE >= 2
#define unix_memtag_thread_error(...)   _mi_fatal_error(__VA_ARGS__)
#else
#define unix_memtag_thread_error(...)   _mi_warning_message(__VA_ARGS__)
#endif

// Enable the tagged address ABI with the tag check mode `tcf` and allow tags 1 to 15 for `irg` (for the current thread)
static bool unix_memtag_thread_enable(unsigned long ctrl, unsigned long tcf) {
  const unsigned long tags = (ctrl & PR_MTE_TAG_MASK) | (0xFFFEUL << PR_MTE_TAG_SHIFT);
  return (prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE | tcf | tags, 0, 0, 0) == 0);
}

// Return the current number of threads in the process (or 0 if unknown)
// (we use `/proc/self/stat` instead of `/proc/self/status` as it is emulated by qemu-user to only count guest threads)
static size_t unix_thread_count(void) {
  const int fd = mi_prim_open("/proc/self/stat", O_RDONLY);
  if (fd < 0) return 0;
  char buf[1024];
  const ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)-1);
  mi_prim_close(fd);
  if (nread <= 0 || nread >= (ssize_t)sizeof(buf)) return 0;
  buf[nread] = 0;
  // skip to the last `)` (as the executable name can contain spaces and parenthesis) which ends field 2
  const char* s = NULL;
  for (const char* p = buf; *p != 0; p++) { if (*p == ')') { s = p + 1; } }
  if (s == NULL) return 0;
  // and skip fields 3 to 19 to get to field 20 (`num_threads`)
  for (int field = 3; field < 20; field++) {
    while (*s == ' ') { s++; }
    while (*s != ' ' && *s != 0) { s++; }
  }
  while (*s == ' ') { s++; }
  size_t count = 0;
  while (*s >= '0' && *s <= '9') { count = 10*count + (size_t)(*s - '0'); s++; }
  return count;
}
#endif

#if MI_MTE >= 2
// Does the CPU support the memory tagging (MTE) instructions? (only checked with `-DMI_MTE=ON`;
// on other systems we assume it does and fault on the first use of a tag instruction otherwise)
static bool unix_detect_memtag_cpu(void) {
  #if MI_UNIX_LINUX_MTE
  return ((getauxval(AT_HWCAP2) & HWCAP2_MTE) != 0);
  #elif defined(__APPLE__) && defined(__aarch64__)
  int val = 0;
  size_t len = sizeof(val);
  return (sysctlbyname("hw.optional.arm.FEAT_MTE", &val, &len, NULL, 0) == 0 && val != 0);
  #else
  return true;
  #endif
}
#endif

#if MI_PAC >= 2
// Does the CPU (and OS) support generic pointer authentication (`pacga`)? (only detected with `-DMI_PAC=ON`)
static bool unix_detect_pac_cpu(void) {
  #if MI_UNIX_LINUX_PAC
  return ((getauxval(AT_HWCAP) & HWCAP_PACG) != 0);
  #elif defined(__APPLE__) && defined(__aarch64__)
  int val = 0;
  size_t len = sizeof(val);
  return (sysctlbyname("hw.optional.arm.FEAT_PAuth", &val, &len, NULL, 0) == 0 && val != 0);
  #else
  return true;
  #endif
}
#endif

// Detect if memory tagging (MTE) is enabled for this process (only called if the `mte` option is enabled)
static bool unix_detect_memtag(mi_os_mem_config_t* config) {
  MI_UNUSED(config);
  #if MI_UNIX_MACH_MTE
  if (__builtin_available(macOS 26.0, iOS 26.0, *)) {
    return ((os_security_config_get() & OS_SECURITY_CONFIG_MTE) != 0);   // set through process entitlements
  }
  #elif MI_UNIX_LINUX_MTE
  // These settings are per thread and inherited by threads created afterwards (see also `_mi_prim_thread_init`)
  if ((getauxval(AT_HWCAP2) & HWCAP2_MTE) == 0) return false;
  const int ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
  if (ctrl < 0) return false;
  unsigned long tcf;
  switch (mi_option_get(mi_option_mte)) {
    case MI_MEMTAG_MODE_SYNC:      tcf = PR_MTE_TCF_SYNC; break;
    case MI_MEMTAG_MODE_ASYNC:     tcf = PR_MTE_TCF_ASYNC; break;
    case MI_MEMTAG_MODE_PREFERRED: tcf = PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC; break;
    default: tcf = ((unsigned long)ctrl & PR_MTE_TCF_MASK); if (tcf == 0) { tcf = PR_MTE_TCF_SYNC; } break;
  }
  if (!unix_memtag_thread_enable((unsigned long)ctrl, tcf)) {
    _mi_warning_message("unable to enable memory tagging (error %d)\n", errno);
    return false;
  }
  config->memtag_mode = tcf;
  _mi_verbose_message("memory tag check mode: %s\n", (tcf == PR_MTE_TCF_SYNC ? "sync" : (tcf == PR_MTE_TCF_ASYNC ? "async" : "preferred")));
  // tag checking is only enabled for this thread and threads created from it afterwards (or that allocate)
  const size_t nthreads = unix_thread_count();
  if (nthreads > 1) {
    unix_memtag_thread_error("memory tagging was enabled after %zu other thread(s) were created; these may not use tag checking\n"
                        "  (and system calls with tagged pointers may fail in those threads until they call mimalloc)\n", nthreads - 1);
  }
  return true;
  #endif
  return false;
}

void _mi_prim_mem_init( mi_os_mem_config_t* config )
{
  long psize = sysconf(_SC_PAGESIZE);
  if (psize > 0 && (unsigned long)psize < SIZE_MAX) {
    config->page_size = (size_t)psize;
    config->alloc_granularity = (size_t)psize;
    unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib);
  }
  config->has_partial_free = true;    // mmap can free in parts
  config->has_virtual_reserve = true; // todo: check if this true for NetBSD?  (for anonymous mmap with PROT_NONE)
  #if defined(MAP_ALIGNED) || defined(MAP_ALIGN)
  config->has_aligned_alloc = true;   // see `unix_mmap_prim_aligned`
  #endif
  config->large_page_size = unix_detect_large_page_size();
  config->has_transparent_huge_pages = (config->large_page_size <= MI_UNIX_MAX_THP_PAGE_SIZE) && unix_detect_thp();
  config->has_overcommit = unix_detect_overcommit();
  config->virtual_address_bits = unix_detect_virtual_address_bits();
  #if MI_MTE >= 2
  if (!unix_detect_memtag_cpu()) {
    _mi_fatal_error("built with MI_MTE=ON but the CPU does not support memory tagging (MTE)\n");
  }
  #endif
  config->has_memtag = (mi_option_is_enabled(mi_option_mte) && unix_detect_memtag(config));
  #if MI_PAC >= 2
  config->has_pac = unix_detect_pac_cpu();
  #endif

  // disable transparent huge pages for this process?
  #if (defined(__linux__) || defined(__ANDROID__)) && defined(PR_GET_THP_DISABLE)
  if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option
  {
    config->has_transparent_huge_pages = false;
    if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) {   // -1 on error, 1 if already disabled
      // Most likely since distros often come with always/madvise settings.
      // Disabling only for mimalloc process rather than touching system wide settings
      (void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);
    }
  }
  #endif
}


//---------------------------------------------
// free
//---------------------------------------------

int _mi_prim_free(void* addr, size_t size ) {
  if (size==0) return 0;
  bool err = (munmap(addr, size) == -1);
  return (err ? errno : 0);
}


//---------------------------------------------
// mmap
//---------------------------------------------

// return errno on failure
static int unix_madvise(void* addr, size_t size, int advice) {
  #if defined(__sun) || defined(_AIX)
  const int res = madvise((caddr_t)addr, size, advice);  // Solaris needs cast (issue #520)
  return (res==0 ? 0 : errno);
  #elif defined(__QNX__)
  return posix_madvise(addr, size, advice);              // posix returns errno
  #else
  const int res = madvise(addr, size, advice);           // linux returns -1 on failure and sets errno
  return (res==0 ? 0 : errno);
  #endif
}

static void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) {
  void* p = mmap(addr, size, protect_flags, flags, fd, 0 /* offset */);
  #if defined(__linux__) && defined(PR_SET_VMA)
  static _Atomic(size_t) no_set_vma_name; // = 0
  if (p!=MAP_FAILED && p!=NULL && mi_atomic_load_relaxed(&no_set_vma_name)==0) {
    const int err = errno;
    if (prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, "mimalloc") != 0) {
      // `EINVAL` means naming is not supported (Linux < 5.17, or without `CONFIG_ANON_VMA_NAME`)
      // as our arguments are always valid; in that case don't call again.
      if (errno == EINVAL) { mi_atomic_store_relaxed(&no_set_vma_name, (size_t)1); }
      errno = err;  // don't modify errno on a successful mmap
    }
  }
  #endif
  return p;
}

static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
  MI_UNUSED(try_alignment);
  void* p = NULL;
  #if defined(MAP_ALIGNED)  // BSD
  if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
    size_t n = 0;
    mi_bsr(try_alignment, &n);
    if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) {  // alignment is a power of 2 and 4096 <= alignment <= 1GiB
      p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd);
      if (p==MAP_FAILED || !mi_is_aligned(p,try_alignment)) {
        int err = errno;
        _mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
      }
      if (p!=MAP_FAILED) return p;
      // fall back to regular mmap
    }
  }
  #elif defined(MAP_ALIGN)  // Solaris
  if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
    p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd);  // addr parameter is the required alignment
    if (p!=MAP_FAILED) return p;
    // fall back to regular mmap
  }
  #endif
  #if (MI_SIZE_SIZE >= 8) && !defined(MAP_ALIGNED)
  // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations
  if (addr == NULL) {
    void* hint = _mi_os_get_aligned_hint(try_alignment, size);
    if (hint != NULL) {
      p = unix_mmap_prim(hint, size, protect_flags, flags, fd);
      if (p==MAP_FAILED || !mi_is_aligned(p,try_alignment)) {
        #if MI_TRACK_ENABLED  // asan sometimes does not instrument errno correctly?
        int err = 0;
        #else
        int err = errno;
        #endif
        _mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
      }
      if (p!=MAP_FAILED) return p;
      // fall back to regular mmap
    }
  }
  #endif
  // regular mmap
  p = unix_mmap_prim(addr, size, protect_flags, flags, fd);
  if (p!=MAP_FAILED) return p;
  // failed to allocate
  return NULL;
}

static int unix_mmap_fd(void) {
  #if defined(VM_MAKE_TAG)
  // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
  int os_tag = (int)mi_option_get(mi_option_os_tag);
  if (os_tag < 100 || os_tag > 255) { os_tag = 254; }
  return VM_MAKE_TAG(os_tag);
  #else
  return -1;
  #endif
}

// macOS superpages are not supported on arm64 (with 16KiB pages)
#if defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) && !(defined(__APPLE__) && (defined(__aarch64__) || defined(__arm64__)))
#define MI_OS_HAS_SUPERPAGES  1
#endif

#if defined(MAP_ALIGNED_SUPER) || defined(MAP_HUGETLB) || defined(MAP_HUGE_1GB) || defined(MAP_HUGE_2MB) || MI_OS_HAS_SUPERPAGES
#define MI_OS_HAS_HUGE_PAGES  1
#endif

#if MI_UNIX_MACH_MTE
// Allocate memory that supports memory tagging (as `mmap` has no MTE flag on macOS/iOS)
static void* unix_mmap_memtag(size_t size, size_t try_alignment, int protect_flags, int fd) {
  const vm_offset_t mask = (mi_is_power_of_two(try_alignment) && try_alignment <= MI_GiB ? try_alignment - 1 : 0);
  const vm_prot_t prot = ((protect_flags & PROT_WRITE) != 0 ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_NONE);
  vm_address_t addr = 0;
  const kern_return_t err = vm_map(mach_task_self(), &addr, size, mask, VM_FLAGS_ANYWHERE | VM_FLAGS_MTE | (fd != -1 ? fd : 0),
                                   MEMORY_OBJECT_NULL, 0, FALSE, prot, VM_PROT_READ | VM_PROT_WRITE, VM_INHERIT_DEFAULT);
  if (err != KERN_SUCCESS) {
    _mi_trace_message("unable to allocate memory tagged OS memory (error: %d, size: 0x%zx bytes, alignment: 0x%zx)\n", err, size, try_alignment);
    errno = ENOMEM;
    return NULL;
  }
  return (void*)addr;
}
#endif

static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
  #if !defined(MAP_ANONYMOUS)
  #define MAP_ANONYMOUS  MAP_ANON
  #endif
  #if !defined(MAP_NORESERVE)
  #define MAP_NORESERVE  0
  #endif
  void* p = NULL;
  const int fd = unix_mmap_fd();
  bool allow_huge = allow_large;
  #if MI_UNIX_MACH_MTE || MI_UNIX_LINUX_MTE
  if (_mi_os_has_memtag()) {   // no large OS pages with memory tagging
    *is_large = false;
    if (large_only) return NULL;
    #if MI_UNIX_MACH_MTE
    return unix_mmap_memtag(size, try_alignment, protect_flags, fd);
    #else
    allow_huge = false;
    protect_flags |= PROT_MTE;  // also on `PROT_NONE` reservations as it is sticky with later `mprotect` calls
    #endif
  }
  #endif
  int flags = MAP_PRIVATE | MAP_ANONYMOUS;
  if (_mi_os_has_overcommit()) {
    flags |= MAP_NORESERVE;
  }
  #if defined(PROT_MAX)
  protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
  #endif
  // huge page allocation
  #if MI_OS_HAS_HUGE_PAGES
  if (allow_huge && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) {
    size_t backoff = mi_atomic_load_relaxed(&_mi_process.os.large_page_backoff);
    if (!large_only && backoff > 0) {
      // If the OS is not configured for large OS pages, or the user does not have
      // enough permission, the `mmap` will always fail (but it might also fail for other reasons).
      // Therefore, once a large page allocation failed, we don't try again for `os_large_page_backoff` times
      // to avoid too many failing calls to mmap.
      mi_atomic_cas_strong_relaxed(&_mi_process.os.large_page_backoff, &backoff, backoff - 1);
    }
    else {
      int lflags = flags & ~MAP_NORESERVE;  // using NORESERVE on huge pages seems to fail on Linux
      int lfd = fd;
      #ifdef MAP_ALIGNED_SUPER
      lflags |= MAP_ALIGNED_SUPER;
      #endif
      #ifdef MAP_HUGETLB
      lflags |= MAP_HUGETLB;
      #endif
      #ifdef MAP_HUGE_1GB
      if (large_only && (size % MI_GiB) == 0 && (mi_atomic_load_relaxed(&_mi_process.os.huge_page_unavailable)==0)) {
        lflags |= MAP_HUGE_1GB;
      }
      else
      #endif
      {
        #ifdef MAP_HUGE_2MB
        lflags |= MAP_HUGE_2MB;
        #endif
      }
      #if MI_OS_HAS_SUPERPAGES
      lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
      #endif
      if (large_only || lflags != flags) {
        // try large OS page allocation
        *is_large = true;
        p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
        #ifdef MAP_HUGE_1GB
        if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {
          mi_atomic_store_relaxed(&_mi_process.os.huge_page_unavailable,1); // don't try huge 1GiB pages again
          if (large_only) {
            _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno);
          }
          lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
          p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
        }
        #endif
        if (large_only) return p;
        if (p == NULL) {
          mi_atomic_store_relaxed(&_mi_process.os.large_page_backoff, (size_t)8);  // on error, don't try again for the next N allocations
        }
      }
    }
  } // huge pages
  #else
  MI_UNUSED(allow_huge);
  if (large_only) { *is_large = false; errno = ENOMEM; return NULL; }  // no huge OS pages available
  #endif
  // regular allocation
  if (p == NULL) {
    *is_large = false;
    p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd);
    if (p!=NULL) {
      #if defined(MADV_NOHUGEPAGE)
      const size_t large_page_size = _mi_os_large_page_size();
      if (large_page_size > MI_UNIX_MAX_THP_PAGE_SIZE && size >= large_page_size) {
        (void)unix_madvise(p, size, MADV_NOHUGEPAGE);  // prevent using large transparent huge pages, see issue #1431
      }
      else
      #endif
      {  // keep the `else` above well-formed when MI_NO_THP is defined
      #if !defined(MI_NO_THP)
      if (allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size,try_alignment)) {
        #if defined(MADV_HUGEPAGE)
        if (_mi_os_has_thp()) {
          // Many Linux systems don't allow MAP_HUGETLB but they support instead
          // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE
          // though since properly aligned allocations will already use large pages if available
          // in that case -- in particular for our large regions (in `memory.c`).
          // However, some systems only allow THP if called with explicit `madvise`, so
          // when large OS pages are enabled for mimalloc, we call `madvise` anyways.
          if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
            // *is_large = true; // possibly
          };
        }
        #elif defined(__sun)
        struct memcntl_mha cmd = {0};
        cmd.mha_pagesize = _mi_os_large_page_size();
        cmd.mha_cmd = MHA_MAPSIZE_VA;
        if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
          // *is_large = true; // possibly
        }
        #endif
      }
      #endif
      }
    }
  }
  return p;
}

// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
  mi_assert_internal(commit || !allow_large);
  mi_assert_internal(try_alignment > 0);
  *is_zero = true;
  int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
  *addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large);
  return (*addr != NULL ? 0 : errno);
}


//---------------------------------------------
// Commit/Reset
//---------------------------------------------

static void unix_mprotect_hint(int err) {
  #if defined(__linux__) && (MI_SECURE>=5 || MI_GUARDED) // guard page around every mimalloc page
  if (err == ENOMEM) {
    _mi_warning_message("The next warning may be caused by a low memory map limit.\n"
                        "  On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"
                        "  For example: sudo sysctl -w vm.max_map_count=262144\n");
  }
  #else
  MI_UNUSED(err);
  #endif
}





int _mi_prim_commit(void* start, size_t size, bool* is_zero) {
  // commit: ensure we can access the area
  // note: we may think that *is_zero can be true since the memory
  // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but
  // we sometimes call commit on a range with still partially committed
  // memory and `mprotect` does not zero the range.
  *is_zero = false;
  int err = mprotect(start, size, (PROT_READ | PROT_WRITE));
  if (err != 0) {
    err = errno;
    unix_mprotect_hint(err);
  }
  return err;
}

int _mi_prim_reuse(void* start, size_t size) {
  MI_UNUSED(start); MI_UNUSED(size);
  #if defined(__APPLE__) && defined(MADV_FREE_REUSE)
  return unix_madvise(start, size, MADV_FREE_REUSE);
  #endif
  return 0;
}

// Release the physical memory of a range (while keeping it mapped)
static int unix_madvise_release(void* start, size_t size) {
  #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
  // use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097)
  int err = unix_madvise(start, size, MADV_FREE_REUSABLE);
  if (err) { err = unix_madvise(start, size, MADV_DONTNEED); }
  return err;
  #else
  // use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
  return unix_madvise(start, size, MADV_DONTNEED);
  #endif
}

int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
  int err = 0;
  #if 1
    err = unix_madvise_release(start, size);
    #if MI_DEBUG || MI_SECURE>=4
      *needs_recommit = true;
      mprotect(start, size, PROT_NONE);
    #else
      *needs_recommit = false;
    #endif
  #else
    // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss)
    *needs_recommit = true;
    const int fd = unix_mmap_fd();
    void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
    if (p != start) { err = errno; }
  #endif
  return err;
}

int _mi_prim_reset(void* start, size_t size) {
  int err = 0;

  // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower)
  #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE)
  err = unix_madvise(start, size, MADV_FREE_REUSABLE);
  if (err==0) return 0;
  // fall through
  #endif

  #if defined(MADV_FREE)
  // Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it
  // will not reduce the `rss` stats in tools like `top` even though the memory is available
  // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by
  // default `MADV_DONTNEED` is used though.
  const bool lazy = (mi_atomic_load_relaxed(&_mi_process.os.lazy_reset_unavailable) == 0);
  while ((err = unix_madvise(start, size, (lazy ? MADV_FREE : MADV_DONTNEED))) != 0 && err == EAGAIN) { /* try again */ };
  if (err == EINVAL && lazy) {
    // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
    mi_atomic_store_release(&_mi_process.os.lazy_reset_unavailable, (size_t)1);
    err = unix_madvise(start, size, MADV_DONTNEED);
  }
  #else
  err = unix_madvise(start, size, MADV_DONTNEED);
  #endif
  return err;
}

#if defined(__OpenBSD__)
#include <sys/param.h>  // OpenBSD version
#endif

// Make a read-only range immutable (best effort)
static void unix_mseal(void* addr, size_t size) {
  #if defined(MI_HAS_SYSCALL_H) && defined(SYS_mseal)
  syscall(SYS_mseal, addr, size, 0UL);  // Linux 6.10+
  #elif defined(__OpenBSD__) && defined(OpenBSD) && (OpenBSD >= 202304)
  mimmutable(addr, size);               // OpenBSD 7.3+
  #else
  MI_UNUSED(addr); MI_UNUSED(size);
  #endif
}

int _mi_prim_protect(void* addr, size_t size, mi_protect_t protect) {
  const int prot = (protect == MI_PROTECT_NONE ? PROT_NONE : (protect == MI_PROTECT_READWRITE ? (PROT_READ | PROT_WRITE) : PROT_READ));
  int err = mprotect(addr, size, prot);
  if (err != 0) { err = errno; }
  unix_mprotect_hint(err);
  if (err == 0 && protect == MI_PROTECT_READ_SEALED) { unix_mseal(addr, size); }
  return err;
}

int _mi_prim_guard(void* start, size_t size, bool set_guard, bool is_committed) {
  if (set_guard) {
    // `mprotect` also works on reserved memory, so committed or not, we release the physical memory and protect
    unix_madvise_release(start, size);  // best effort
    return _mi_prim_protect(start, size, MI_PROTECT_NONE);
  }
  else if (is_committed) {
    const int err = _mi_prim_protect(start, size, MI_PROTECT_READWRITE);
    return (err != 0 ? err : _mi_prim_reuse(start, size));
  }
  else {
    return 0;  // stays inaccessible until committed
  }
}



//---------------------------------------------
// Huge page allocation
//---------------------------------------------

#if (MI_SIZE_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__)

#ifndef MPOL_PREFERRED
#define MPOL_PREFERRED 1
#endif

#if defined(MI_HAS_SYSCALL_H) && defined(SYS_mbind)
static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
  return syscall(SYS_mbind, start, len, mode, nmask, maxnode, flags);
}
#else
static long mi_prim_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) {
  MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags);
  return 0;
}
#endif

int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
  bool is_large = true;
  *is_zero = true;
  *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large);
  if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_SIZE_SIZE - 1)) { // at most 63 nodes
    unsigned long numa_mask = (1UL << numa_node);
    // todo: does `mbind` work correctly for huge OS pages? should we
    // use `set_mempolicy` before calling mmap instead?
    // see: <https://lkml.org/lkml/2017/2/9/875>
    long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_SIZE_SIZE, 0);
    if (err != 0) {
      err = errno;
      _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err);
    }
  }
  return (*addr != NULL ? 0 : errno);
}

#else

int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr) {
  MI_UNUSED(hint_addr); MI_UNUSED(size); MI_UNUSED(numa_node);
  *is_zero = false;
  *addr = NULL;
  return ENOMEM;
}

#endif

//---------------------------------------------
// NUMA nodes
//---------------------------------------------

#if defined(__linux__)

size_t _mi_prim_numa_node(void) {
  #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)
    unsigned int node = 0;
    unsigned int ncpu = 0;
    int err = syscall(SYS_getcpu, &ncpu, &node, NULL);
    if (err != 0) return 0;
    return node;
  #else
    return 0;
  #endif
}

size_t _mi_prim_numa_node_count(void) {
  char buf[128];
  unsigned last_found = 0;
  for(unsigned node = 1; node < 256; node++) {
    // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
    _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node);
    if (mi_prim_access(buf,R_OK) != 0) {
      if (node - last_found > 4) break; // allow some sparseness of nodes but not more than 4
    }
    else { last_found = node; }         // highest found node
  }
  return last_found + 1;
}

#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000

size_t _mi_prim_numa_node(void) {
  domainset_t dom;
  size_t node;
  int policy;
  if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul;
  for (node = 0; node < MAXMEMDOM; node++) {
    if (DOMAINSET_ISSET(node, &dom)) return node;
  }
  return 0ul;
}

size_t _mi_prim_numa_node_count(void) {
  size_t ndomains = 0;
  size_t len = sizeof(ndomains);
  if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul;
  return ndomains;
}

#elif defined(__DragonFly__)

size_t _mi_prim_numa_node(void) {
  // todo: DragonFly does not seem to provide any userland means to get this information.
  return 0ul;
}

size_t _mi_prim_numa_node_count(void) {
  size_t ncpus = 0, nvirtcoresperphys = 0;
  size_t len = sizeof(size_t);
  if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul;
  if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul;
  return nvirtcoresperphys * ncpus;
}

#else

size_t _mi_prim_numa_node(void) {
  return 0;
}

size_t _mi_prim_numa_node_count(void) {
  return 1;
}

#endif

// ----------------------------------------------------------------
// Clock
// ----------------------------------------------------------------

#include <time.h>

// low resolution timer
static mi_msecs_t mi_prim_clock_now_lowres(void) {
  const int64_t ticks = (int64_t)clock();
  #if !defined(CLOCKS_PER_SEC)
    return ticks;
  #else
    if (CLOCKS_PER_SEC <= 0 || CLOCKS_PER_SEC == 1000) {
      return ticks;
    }
    else if (CLOCKS_PER_SEC > 0 && CLOCKS_PER_SEC < 1000) {
      return ticks * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
    }
    else {
      return ticks / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
    }
  #endif
}

mi_msecs_t _mi_prim_clock_now(void) {
  #if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC)
    #ifdef CLOCK_MONOTONIC
    const clockid_t clockid = CLOCK_MONOTONIC;
    #else
    const clockid_t clockid = CLOCK_REALTIME;
    #endif
    struct timespec t;
    if (clock_gettime(clockid,&t) == 0) {
      return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000L);
    }
  #endif
  return mi_prim_clock_now_lowres();
}


//----------------------------------------------------------------
// Process info
//----------------------------------------------------------------

#if defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)
#include <stdio.h>
#include <sys/resource.h>

#if defined(__APPLE__)
#include <mach/mach.h>
#endif

#if defined(__HAIKU__)
#include <kernel/OS.h>
#endif

static mi_msecs_t timeval_secs(const struct timeval* tv) {
  return ((mi_msecs_t)tv->tv_sec * 1000L) + ((mi_msecs_t)tv->tv_usec / 1000L);
}

void _mi_prim_process_info(mi_process_info_t* pinfo)
{
  struct rusage rusage;
  if (getrusage(RUSAGE_SELF, &rusage) == 0) {
    pinfo->utime = timeval_secs(&rusage.ru_utime);
    pinfo->stime = timeval_secs(&rusage.ru_stime);
    #if !defined(__HAIKU__)
      pinfo->page_faults = rusage.ru_majflt;
    #endif
    #if defined(__APPLE__)
      pinfo->peak_rss = rusage.ru_maxrss;         // macos reports in bytes
    #else
      pinfo->peak_rss = rusage.ru_maxrss * 1024;  // Linux/BSD report in KiB
    #endif
  }

  #if defined(__HAIKU__)
    // Haiku does not have (yet?) a way to
    // get these stats per process
    thread_info tid;
    if (get_thread_info(find_thread(0), &tid) == B_OK) {
      area_info mem;
      ssize_t c;
      while (get_next_area_info(tid.team, &c, &mem) == B_OK) {
        pinfo->peak_rss += mem.ram_size;
      }
    }
    pinfo->page_faults = 0;
  #elif defined(__APPLE__)
    #ifdef MACH_TASK_BASIC_INFO
    struct mach_task_basic_info info;
    mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
    if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
      pinfo->current_rss = (size_t)info.resident_size;
    }
    #else
    struct task_basic_info info;
    mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT;
    if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) {
      pinfo->current_rss = (size_t)info.resident_size;
    }
    #endif
  #endif
  // use defaults for commit
}

#else

#ifndef __wasi__
// WebAssembly instances are not processes
#pragma message("define a way to get process info")
#endif

void _mi_prim_process_info(mi_process_info_t* pinfo)
{
  // use defaults
  MI_UNUSED(pinfo);
}

#endif


//----------------------------------------------------------------
// Output
//----------------------------------------------------------------

void _mi_prim_out_stderr( const char* msg ) {
  fputs(msg,stderr);
}


//----------------------------------------------------------------
// Environment
//----------------------------------------------------------------

#if !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0)
// On Posix systemsr use `environ` to access environment variables
// even before the C runtime is initialized.
#if defined(__APPLE__) && defined(__has_include) && __has_include(<crt_externs.h>)
#include <crt_externs.h>
static char** mi_get_environ(void) {
  return (*_NSGetEnviron());
}
#else
extern char** environ;
static char** mi_get_environ(void) {
  return environ;
}
#endif
int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
  return _mi_prim_getenv_by_visit(name, result, result_size);
}
bool _mi_prim_env_visit(mi_env_visit_fun* visit, void* arg) {
  char** env = mi_get_environ();
  if (env == NULL) return false;
  for (int i = 0; i < 10000 && env[i] != NULL; i++) {
    visit(env[i], arg);
  }
  return true;
}
#else
// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime
int _mi_prim_getenv(const char* name, char* result, size_t result_size) {
  // cannot call getenv() when still initializing the C runtime.
  if (_mi_preloading()) return -1;  // error, try again later
  const char* s = getenv(name);
  if (s == NULL) {
    // we check the upper case name too.
    char buf[64+1];
    size_t len = _mi_strnlen(name,sizeof(buf)-1);
    for (size_t i = 0; i < len; i++) {
      buf[i] = _mi_toupper(name[i]);
    }
    buf[len] = 0;
    s = getenv(buf);
  }
  if (s == NULL) return 0; // not found
  if (!_mi_strlcpy(result, s, result_size)) return -2;  // too long
  return 1;  // success
}
bool _mi_prim_env_visit(mi_env_visit_fun* visit, void* arg) {
  MI_UNUSED(visit); MI_UNUSED(arg);
  return false;  // cannot iterate the environment
}
#endif  // !MI_USE_ENVIRON


//----------------------------------------------------------------
// Random
//----------------------------------------------------------------

#if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_15)
#include <CommonCrypto/CommonCryptoError.h>
#include <CommonCrypto/CommonRandom.h>

bool _mi_prim_random_buf(void* buf, size_t buf_len) {
  // We prefer CCRandomGenerateBytes as it returns an error code while arc4random_buf
  // may fail silently on macOS. See PR #390, and <https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html>
  return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);
}

#elif defined(__ANDROID__) || defined(__DragonFly__) || \
      defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
      defined(__sun) || \
      (defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7))

bool _mi_prim_random_buf(void* buf, size_t buf_len) {
  arc4random_buf(buf, buf_len);
  return true;
}

#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) || defined(__CYGWIN__)  // also for old apple versions < 10.7 (issue #829)

#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>

bool _mi_prim_random_buf(void* buf, size_t buf_len) {
  // Modern Linux provides `getrandom` but different distributions either use `sys/random.h` or `linux/random.h`
  // and for the latter the actual `getrandom` call is not always defined.
  // (see <https://stackoverflow.com/questions/45237324/why-doesnt-getrandom-compile>)
  // We therefore use a syscall directly and fall back dynamically to /dev/urandom when needed.
  #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getrandom)
    #ifndef GRND_NONBLOCK
    #define GRND_NONBLOCK (1)
    #endif
    #ifndef GRND_INSECURE
    #define GRND_INSECURE (4)
    #endif
    static _Atomic(size_t) no_getrandom; // = 0
    if (mi_atomic_load_acquire(&no_getrandom)==0) {
      ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK);
      if (ret < 0 && errno == EAGAIN) {
        // early during boot the entropy pool may not be initialized yet; use the best available randomness (Linux 5.6+)
        ret = syscall(SYS_getrandom, buf, buf_len, GRND_INSECURE);
      }
      if (ret > 0) { mi_msan_unpoison(buf, (size_t)ret); }
      if (ret >= 0) return (buf_len == (size_t)ret);
      if (errno == ENOSYS) { mi_atomic_store_release(&no_getrandom, (size_t)1); } // don't call again
      // and fall back to /dev/urandom (which does not block early during boot)
    }
  #endif
  int flags = O_RDONLY;
  #if defined(O_CLOEXEC)
  flags |= O_CLOEXEC;
  #endif
  int fd = mi_prim_open("/dev/urandom", flags);
  if (fd < 0) return false;
  size_t count = 0;
  while(count < buf_len) {
    ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count);
    if (ret==0) {
      break;
    }
    else if (ret<0) {
      if (errno!=EAGAIN && errno!=EINTR) break;
    }
    else {
      count += ret;
    }
  }
  mi_prim_close(fd);
  return (count==buf_len);
}

#else

bool _mi_prim_random_buf(void* buf, size_t buf_len) {
  return false;
}

#endif


//----------------------------------------------------------------
// Thread init/done
//----------------------------------------------------------------

#if defined(MI_USE_PTHREADS)

// use pthread local storage keys to detect thread ending
// (and used with MI_TLS_PTHREADS for the default theap)
// atomic as a thread may initialize while the process is exiting and deleting the key
static _Atomic(pthread_key_t) _mi_heap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID);

static void mi_pthread_done(void* value) {
  if (value!=NULL) {
    _mi_thread_done((mi_theap_t*)value);
  }
}

void _mi_prim_thread_init_auto_done(void) {
  mi_assert_internal(mi_atomic_load_relaxed(&_mi_heap_default_key) == MI_PTHREAD_KEY_INVALID);
  pthread_key_t key;
  if (pthread_key_create(&key, &mi_pthread_done) == 0) {
    mi_atomic_store_release(&_mi_heap_default_key, key);
  }
}

void _mi_prim_thread_done_auto_done(void) {
  pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_heap_default_key, MI_PTHREAD_KEY_INVALID);
  if (key != MI_PTHREAD_KEY_INVALID) {  // do not leak the key, see issue #809
    pthread_key_delete(key);
  }
}

void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
  pthread_key_t key = mi_atomic_load_relaxed(&_mi_heap_default_key);
  if (key != MI_PTHREAD_KEY_INVALID) {  // can happen during recursive invocation on freeBSD
    pthread_setspecific(key, theap);
  }
}


#else

void _mi_prim_thread_init_auto_done(void) {
  // nothing
}

void _mi_prim_thread_done_auto_done(void) {
  // nothing
}

void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
  MI_UNUSED(theap);
}

#endif

void _mi_prim_thread_init(void) {
  #if MI_UNIX_LINUX_MTE
  // ensure tag checking is enabled on threads that existed before mimalloc was initialized (and keep any mode that is already set)
  if (_mi_os_has_memtag()) {
    const int ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
    if (ctrl >= 0 && ((ctrl & PR_TAGGED_ADDR_ENABLE) == 0 || (ctrl & PR_MTE_TCF_MASK) == 0)) {
      if (!unix_memtag_thread_enable((unsigned long)ctrl, (unsigned long)mi_process_get_sealed_once()->os_mem_config.memtag_mode)) {
        unix_memtag_thread_error("unable to enable memory tagging on thread 0x%zx (error %d)\n", _mi_thread_id(), errno);
      }
    }
  }
  #endif
}

bool _mi_prim_thread_is_in_threadpool(void) {
  return false;
}

void _mi_prim_thread_yield(void) {
  sleep(0);
}

bool _mi_prim_process_is_killed(void) {
  return false;  // todo: can we detect when exit_group has been called and threads have been killed (potentially holding locks?)
}