vey-mimalloc-sys 0.3.2+3.7.0

Rust FFI binding to mimalloc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
/* ----------------------------------------------------------------------------
Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/
#include "mimalloc.h"
#include "mimalloc/prim/prim.h"
#include "mimalloc/track.h"
#include "mimalloc/prim/os.h"
#include "mimalloc/prim/prim-tls.h"
#include "mimalloc/util/random.h"
#include "mimalloc/util/stats.h"
#include "mimalloc/util/memid.h"
#include "mimalloc/heap/page-map.h"
#include "mimalloc/profile/pprof.h"
#include "mimalloc/profile/sample-profile.h"

#include <stdlib.h>  // atexit
#include "mimalloc/heap/process.h"

// Empty page used to initialize the small free pages array
static const mi_page_t mi_page_empty = {
  #if MI_PAGE_META_IS_ALIGNED
  MI_ATOMIC_VAR_INIT(NULL),  // self
  #endif
  MI_ATOMIC_VAR_INIT(0),  // xthread_id
  NULL,                   // free
  NULL,                   // local_free
  0,                      // block_size
  {0},                    // xused
  #if MI_SIZE_SIZE < 8
  0,                      // xlast_used
  0,                      // xlast_alloc
  #endif
  NULL,                   // page_start
  #if MI_FREE_IS_CHECKED
  MI_ATOMIC_VAR_INIT(0),  // capacity_size
  #endif
  0,                      // capacity
  0,                      // reserved capacity
  0,                      // slice_pcommitted
  0,                      // retire_expire
  false,                  // is_zero
  MI_ATOMIC_VAR_INIT(0),  // xthread_free
  NULL,                   // theap
  NULL,                   // heap
  NULL, NULL,             // next, prev
  MI_MEMID_STATIC,        // memid
  #if (MI_PADDING || MI_ENCODE_FREELIST)
  { 0, 0 },               // keys
  #endif
  #if MI_FREE_IS_CHECKED
  0,                      // block_recip
  #endif
  #if MI_QSBR
  MI_ATOMIC_VAR_INIT(0),  // qsbr_xthread_id
  NULL, NULL,             // qsbr_next, qsbr_prev
  mi_init_struct_zero,    // qsbr
  #endif
  #if MI_DEBUG>0
  MI_ATOMIC_VAR_INIT(0),  // debug_fill_preserve
  #endif
};

#define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty)

#if MI_SMALL_WSIZE_MAX == 128
#define MI_INIT_PAGES_DIRECT(p)  MI_INIT128(p)
#elif MI_SMALL_WSIZE_MAX == 256
#define MI_INIT_PAGES_DIRECT(p)  MI_INIT128(p), MI_INIT128(p)
#else
#error define initializer for direct pages
#endif

#if (MI_PADDING>0) && (MI_SIZE_SIZE >= 8)
#define MI_SMALL_PAGES_EMPTY  { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#elif (MI_PADDING>0)
#define MI_SMALL_PAGES_EMPTY  { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#else
#define MI_SMALL_PAGES_EMPTY  { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }
#endif


// Empty page queues for every bin
#define QNULL(sz)  { NULL, NULL, 0, (sz)*sizeof(size_t), 0 }
#define MI_PAGE_QUEUES_EMPTY \
  { QNULL(1), \
    QNULL(     1), QNULL(     2), QNULL(     3), QNULL(     4), QNULL(     5), QNULL(     6), QNULL(     7), QNULL(     8), /* 8 */ \
    QNULL(    10), QNULL(    12), QNULL(    14), QNULL(    16), QNULL(    20), QNULL(    24), QNULL(    28), QNULL(    32), /* 16 */ \
    QNULL(    40), QNULL(    48), QNULL(    56), QNULL(    64), QNULL(    80), QNULL(    96), QNULL(   112), QNULL(   128), /* 24 */ \
    QNULL(   160), QNULL(   192), QNULL(   224), QNULL(   256), QNULL(   320), QNULL(   384), QNULL(   448), QNULL(   512), /* 32 */ \
    QNULL(   640), QNULL(   768), QNULL(   896), QNULL(  1024), QNULL(  1280), QNULL(  1536), QNULL(  1792), QNULL(  2048), /* 40 */ \
    QNULL(  2560), QNULL(  3072), QNULL(  3584), QNULL(  4096), QNULL(  5120), QNULL(  6144), QNULL(  7168), QNULL(  8192), /* 48 */ \
    QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \
    QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \
    QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \
    QNULL(MI_LARGE_MAX_OBJ_WSIZE + 1  /* 655360, Huge queue */), \
    QNULL(MI_LARGE_MAX_OBJ_WSIZE + 2) /* Full queue */ }

#define MI_STAT_COUNT_NULL()  {0,0,0}

// Empty statistics
#define MI_STAT_COUNT(stat)     {0,0,0},
#define MI_STAT_COUNTER(stat)   {0},

#define MI_STATS_FIELDS_NULL  \
  MI_STAT_FIELDS()                    /* regular stat fields */ \
  { MI_INIT4(MI_STAT_COUNT_NULL) },   /* stat reserved */ \
  { { 0 }, { 0 }, { 0 }, { 0 } },     /* stat counter reserved */ \
  { MI_INIT74(MI_STAT_COUNT_NULL) },  /* malloc_bins */ \
  { MI_INIT74(MI_STAT_COUNT_NULL) },  /* page bins   */ \
  { MI_INIT5(MI_STAT_COUNT_NULL) }    /* chunk bins  */

#define MI_STATS_NULL \
  { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_FIELDS_NULL }

// --------------------------------------------------------
// Statically allocate an empty theap as the initial
// thread local value for the default theap,
// and statically allocate the backing theap for the main
// thread so it can function without doing any allocation
// itself (as accessing a thread local for the first time
// may lead to allocation itself on some platforms)
// --------------------------------------------------------

// read-only detached tld of the empty theap
static mi_decl_cache_align const mi_tld_t mi_tld_empty = {
  MI_THREAD_ID_DETACHED,   // thread_id
  0,                      // thread_seq
  0,                      // fork_epoch
  0,                      // default numa node
  NULL,                   // subproc
  NULL,                   // theaps list
  MI_LOCK_INITIALIZER,    // theaps lock
  false,                  // recurse
  false,                  // in_profiler
  false,                  // is_in_threadpool
  MI_MEMID_STATIC         // memid
};

#ifdef __cplusplus
extern // fix warning in C++ for const variables with internal linkage
#endif
mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = {
  MI_SMALL_PAGES_EMPTY,   // direct small pages
  (mi_tld_t*)&mi_tld_empty, // tld
  MI_ATOMIC_VAR_INIT(NULL), // heap
  MI_ATOMIC_VAR_INIT(NULL), // subproc
  MI_ATOMIC_VAR_INIT(1),  // refcount
  0,                      // cookie (never valid; empty theaps are uninitialized)
  0,                      // full page retain
  false,                  // allow reclaim
  true,                   // allow abandon
  true,                   // is_detached
  true,                   // profile_never: this static empty theap must never be sampled
  false, false,           // guard_front, guard_end
  ~MI_ZU(0),              // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2)
  0, 0,                   // sample rate, requested
  0, 0,                   // profile rate, countdown
  0, 0, 0, 0,             // guarded rate, countdown, min, max
  0, 0,                   // profiler_reserved1, profiler_reserved2
  0,                      // heartbeat
  { {0}, {0}, 0, true },  // random
  0, 0,                   // page count, OS page count
  MI_BIN_FULL, 0,         // page retired min/max
  0,                      // pages_full_size
  0, 0,                   // generic count
  NULL, NULL,             // tnext, tprev
  NULL, NULL,             // hnext, hprev
  MI_PAGE_QUEUES_EMPTY,
  MI_MEMID_STATIC,
  MI_STATS_NULL,          // stats
  #if MI_QSBR
  NULL, NULL,            // qsbr_delayed, qsbr_delayed_last
  #endif
};

#undef MI_STAT_COUNT
#undef MI_STAT_COUNTER


// pre-allocate the process data
mi_decl_hidden mi_decl_process_align mi_process_t _mi_process = mi_init_struct_zero;

// The sealed lock is static as it can be used before process initialization (and a zero lock is invalid on some platforms)
static mi_lock_t mi_sealed_lock = MI_LOCK_INITIALIZER;

// Make the sealed process data writable; this takes the sealed lock until `_mi_process_seal`.
mi_process_sealed_t* _mi_process_unseal(void) {
  mi_lock_acquire(&mi_sealed_lock);
  mi_process_sealed_t* const sealed = (mi_process_sealed_t*)mi_process_get_sealed();
  if (mi_process_get_sealed_once()->is_protected) {
    _mi_prim_protect(sealed, MI_OS_PROBABLE_PAGE_SIZE, MI_PROTECT_READWRITE);  // on failure we fault on the first write
  }
  return sealed;
}

// Make the sealed process data read-only again and release the sealed lock.
void _mi_process_seal(void) {
  mi_process_sealed_t* const sealed = (mi_process_sealed_t*)mi_process_get_sealed();
  const int err = (mi_process_get_sealed_once()->is_protected ? _mi_prim_protect(sealed, MI_OS_PROBABLE_PAGE_SIZE, MI_PROTECT_READ) : 0);
  mi_lock_release(&mi_sealed_lock);
  if (err != 0) {
    _mi_error_message(EINVAL, "unable to seal the process data (error: %d)\n", err);
  }
}

// Set a callback `fun` with `arg` in the sealed fields `pfun` and `parg` (using a seqlock so readers always see a consistent pair).
// The fields must be in the sealed process data (and are writable once unsealed).
void _mi_process_set_callback(const _Atomic(void*)* pfun, const _Atomic(void*)* parg, void* fun, void* arg) {
  mi_process_sealed_t* const sealed = _mi_process_unseal();
  const size_t seq = mi_atomic_load_relaxed(&sealed->callbacks_seq);
  mi_atomic_store_relaxed(&sealed->callbacks_seq, seq + 1);           // odd while updating
  mi_atomic_store_ptr_release(void, (_Atomic(void*)*)parg, arg);      // release stores keep the odd sequence visible first
  mi_atomic_store_ptr_release(void, (_Atomic(void*)*)pfun, fun);
  mi_atomic_store_release(&sealed->callbacks_seq, seq + 2);
  _mi_process_seal();
}

// Get a consistent callback pair from the sealed fields `pfun` and `parg`.
void* _mi_process_get_callback(const _Atomic(void*)* pfun, const _Atomic(void*)* parg, void** arg) {
  const mi_process_sealed_t* const sealed = mi_process_get_sealed();
  size_t seq;
  void* fun;
  do {
    seq = mi_atomic_load_acquire(&sealed->callbacks_seq);
    fun = mi_atomic_load_ptr_acquire(void, pfun);
    *arg = mi_atomic_load_ptr_acquire(void, parg);
  } while ((seq & 1) != 0 || mi_atomic_load_relaxed(&sealed->callbacks_seq) != seq);
  return fun;
}

// Write the sealed-once process data; this takes the sealed lock until `_mi_process_seal_once`.
// This is only allowed before `mi_process_sealed_init` (as the data is never made writable again).
mi_process_sealed_once_t* _mi_process_unseal_once(void) {
  mi_lock_acquire(&mi_sealed_lock);
  mi_process_sealed_once_t* const once = (mi_process_sealed_once_t*)mi_process_get_sealed_once();
  if mi_unlikely(once->is_sealed) {
    mi_lock_release(&mi_sealed_lock);
    _mi_fatal_error("attempt to write sealed process data after process initialization\n");
  }
  return once;
}

void _mi_process_seal_once(void) {
  mi_lock_release(&mi_sealed_lock);
}

// Seal the process data and, in secure mode, make it read-only (and the sealed-once data immutable if possible).
// With memory tagging, secure level > 1, or in debug mode, failing to seal is a fatal error.
#if MI_MTE || (MI_SECURE > 1) || MI_DEBUG
#define MI_SEALED_REQUIRED  (1)
#else
#define MI_SEALED_REQUIRED  (0)
#endif
static void mi_process_sealed_init(void) {
  bool protect = false;
  #if MI_SECURE_STATIC_SEALED
  mi_assert_internal(sizeof(mi_process_sealed_once_t) <= MI_OS_PROBABLE_PAGE_SIZE);
  mi_assert_internal(sizeof(mi_process_sealed_t) <= MI_OS_PROBABLE_PAGE_SIZE);
  if ((MI_OS_PROBABLE_PAGE_SIZE % _mi_os_page_size()) == 0) {
    protect = true;
  }
  else if (MI_SEALED_REQUIRED) {  // fail early as the sealed data (like the page map root and user callbacks) would be writable
    _mi_fatal_error("unable to seal the process data as the OS page size (%zu KiB) is larger than the probable OS page size (%zu KiB); rebuild with `-DMI_OS_PROBABLE_PAGE_SIZE=%zu`\n", _mi_os_page_size() / MI_KiB, (size_t)(MI_OS_PROBABLE_PAGE_SIZE / MI_KiB), _mi_os_page_size());
  }
  else {
    _mi_warning_message("unable to seal the process data as the OS page size (%zu KiB) is too large (rebuild with `-DMI_OS_PROBABLE_PAGE_SIZE=%zu`?)\n", _mi_os_page_size() / MI_KiB, _mi_os_page_size());
  }
  #endif
  mi_process_sealed_once_t* const once = _mi_process_unseal_once();
  once->is_sealed = true;
  once->is_protected = protect;
  int err = 0;
  if (protect) {
    err = _mi_prim_protect((void*)mi_process_get_sealed(), MI_OS_PROBABLE_PAGE_SIZE, MI_PROTECT_READ);
    if (err == 0) { err = _mi_prim_protect(once, MI_OS_PROBABLE_PAGE_SIZE, MI_PROTECT_READ_SEALED); }
  }
  _mi_process_seal_once();
  if (err != 0) {
    if (MI_SEALED_REQUIRED) { _mi_fatal_error("unable to seal the process data (error: %d)\n", err); }
    _mi_error_message(EINVAL, "unable to seal the process data (error: %d)\n", err);
  }
}

// In secure mode, surround the process data with guard pages to protect against overflows of adjacent static data.
// The front guard page is placed just before the sealed pages.
static void mi_process_guard_init(void) {
  #if MI_SECURE && MI_OS_HAS_GUARD_PAGES && !MI_SECURE_NO_STATIC_GUARD
  const size_t front = (size_t)((uint8_t*)mi_process_get_sealed_once() - (uint8_t*)&_mi_process);
  const size_t size  = offsetof(mi_process_t, guard_end) - front;
  _mi_os_secure_guard_pages_set_within(&_mi_process, sizeof(_mi_process), front, size);
  #endif
}


// Allocate the process meta data from the OS (instead of statically) so it is not at a fixed offset
// from the program image, and, in secure mode, surround it with guard pages. It is never freed.
// This is normally called from `mi_process_init_once` right after `_mi_os_init`, but can be called
// lazily if the main subprocess is accessed before process initialization.
mi_decl_noinline mi_process_metadata_t* _mi_process_metadata_init(void) {
  mi_atomic_do_once {
    if (mi_process_get_sealed_once()->os_mem_config.page_size == 0) { _mi_os_init(); }  // accessed before process initialization
    mi_memid_t memid;
    mi_process_metadata_t* metadata = (mi_process_metadata_t*)_mi_os_alloc_with_guards(NULL /* bootstrap stats */, sizeof(mi_process_metadata_t), true /* commit */, false /* allow large */, &memid);
    if (metadata == NULL) {
      _mi_fatal_error("unable to allocate the process meta data (%zu bytes)\n", sizeof(mi_process_metadata_t));
    }
    if (!memid.initially_zero) { mi_memzero_aligned(metadata, sizeof(mi_process_metadata_t)); }
    if (mi_memtag_use_for_meta(!memid.is_pinned)) {  // all non-large OS memory supports tagging
      // tag each field separately (where adjacent fields have different tags)
      #define mi_process_metadata_field_offset(tp,name)  offsetof(mi_process_metadata_t, name),
      static const size_t offsets[] = { mi_process_metadata_fields(mi_process_metadata_field_offset) sizeof(mi_process_metadata_t) };
      #undef mi_process_metadata_field_offset
      const size_t count = sizeof(offsets)/sizeof(offsets[0]) - 1;
      for (size_t i = 0; i < count; i++) {
        mi_assert_internal(offsets[i] % MI_MEMTAG_UNIT_SIZE == 0);
        mi_memtag_meta_init_adjacent((uint8_t*)metadata + offsets[i], offsets[i+1] - offsets[i], i, i > 0 /* has prev */, i + 1 < count /* has next */);
      }
    }
    *((mi_memid_t*)mi_meta_load(&metadata->memid)) = memid;
    mi_process_sealed_once_t* const once = _mi_process_unseal_once();
    once->metadata = metadata;  // stored without memory tag (see `mi_process_metadata`)
    _mi_process_seal_once();
  }
  mi_process_metadata_t* const metadata = mi_process_get_sealed_once()->metadata;
  mi_assert(metadata != NULL);  // fails on a recursive invocation during initialization
  return metadata;
}

mi_page_t* _mi_page_empty_get(void) {
  return (mi_page_t*)&mi_page_empty;
}

mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) {
  return (mi_theap_t*)&_mi_theap_empty;
}

mi_theap_t* _mi_theap_unavailable(void) {
  return mi_process_metadata_field(mi_theap_t, theap_unavailable);
}

bool _mi_is_empty_theap(const mi_theap_t* theap) {
  return (theap == &_mi_theap_empty || _mi_theap_is_unavailable(theap));
}

// A thread is done once `mi_thread_done` set its default theap to the unavailable theap.
// Implicit (lazy) initialization is then no longer done as that would leak if the thread-done hooks already ran
// (for example when thread-local destructors allocate). Only an explicit `mi_thread_init` re-initializes the thread.
bool _mi_thread_is_done(void) {
  return _mi_theap_is_unavailable(_mi_theap_default());
}

/* -----------------------------------------------------------
  Initialization
  Note: on some platforms lock_init or just a thread local access
  can cause allocation and induce recursion during initialization.
----------------------------------------------------------- */

static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc);

// Initialize main heap
static void mi_heap_main_init_once(void) {
  mi_memid_t memid_static = _mi_memid_create(MI_MEM_STATIC);
  mi_theap_t* const theap_unavailable = _mi_theap_unavailable();
  mi_memcpy(theap_unavailable,&_mi_theap_empty,sizeof(*theap_unavailable));

  // initialize the main subprocess
  mi_subproc_t* subproc_main = _mi_subproc_main_init();

  // detached tld for theap_meta
  mi_tld_t* const tld_meta = mi_process_metadata_field(mi_tld_t, tld_meta);
  tld_meta->thread_id = MI_THREAD_ID_DETACHED;
  tld_meta->memid = memid_static;
  mi_tld_init(tld_meta, 0, subproc_main);

  // main process heap
  mi_heap_t* const heap_main = mi_process_metadata_field(mi_heap_t, heap_main);
  heap_main->memid = memid_static;
  mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,heap_main);
  _mi_heap_init(heap_main,mi_thread_local_key_fast,subproc_main,0,false);

  // detached theap for allocating meta-data (we can allocate on this without having an initialized thread)
  mi_theap_t* const theap_meta = mi_process_metadata_field(mi_theap_t, theap_meta);
  theap_meta->memid = memid_static;
  _mi_theap_meta_init(theap_meta,heap_main,tld_meta);
  subproc_main->theap_meta = theap_meta;

  // mi_heap_theap_set(heap_main,theap_main); // set in `mi_thread_init(_theap_default)`
}

static void mi_heap_main_init(void) {
  mi_atomic_do_once {
    mi_heap_main_init_once();
  }
}

mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) {
  mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main);
  if mi_likely(heap!=NULL) {
    return heap;
  }
  else if (_mi_subproc_is_main(subproc)) {
    mi_heap_main_init();
    mi_assert_internal(mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main) != NULL);
    return mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main);
  }
  else {
    mi_assert_internal(false);
    return mi_process_metadata_field(mi_heap_t, heap_main);
  }
}

/* -----------------------------------------------------------
  Thread local data
----------------------------------------------------------- */

mi_theap_t* _mi_theap_main(void) {
  // All subprocess main heaps share this slot; do not follow a possibly deleted default theap.
  mi_process_metadata_t* const metadata = mi_process_get_sealed_once()->metadata;
  if (metadata==NULL) return NULL;
  const mi_thread_local_t key = ((mi_heap_t*)mi_meta_load(&metadata->heap_main))->theap;
  return (key==0 ? NULL : (mi_theap_t*)_mi_thread_local_get(key));
}

static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) {
  tld->subproc = subproc;
  tld->theaps = NULL;
  mi_lock_init(&tld->theaps_lock);
  if (tld->thread_id == MI_THREAD_ID_DETACHED) {
    tld->numa_node = -1;
  }
  else {
    tld->numa_node = _mi_os_numa_node();
    tld->thread_id = _mi_prim_thread_id();
    tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool();
    tld->thread_seq = tseq;
    tld->fork_epoch = _mi_fork_epoch();
    mi_atomic_increment_relaxed(&tld->subproc->thread_count);
  }
  return tld;
}

// Allocate fresh tld
static mi_tld_t* mi_tld_create(mi_subproc_t* subproc) {
  mi_assert_internal(subproc->theap_meta != NULL); // should be initialized on the main thread before other threads allocate
  const size_t tseq = mi_atomic_increment_relaxed(&subproc->thread_total_count);

  mi_memid_t memid;
  mi_tld_t* tld;
  if (_mi_subproc_is_main(subproc) && tseq==0 /* first tld */) {
    tld = mi_process_metadata_field(mi_tld_t, tld_main);
    memid = _mi_memid_create_static(tld,sizeof(*tld));
  }
  else {
    tld  = (mi_tld_t*)_mi_meta_zalloc(subproc, sizeof(mi_tld_t), &memid);
  }
  if (tld==NULL) {
    _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n");
    return NULL;
  }
  tld->memid = memid;
  return mi_tld_init(tld,tseq,subproc);
}

mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) {
  if (tld==NULL) return;
  mi_atomic_decrement_relaxed(&tld->subproc->thread_count);
  tld->thread_id = (mi_thread_id_t)(~0);          // it is best to set an invalid tid for tld_main as sometimes the same thread-id
                                                 // is reused by the OS after a thread has terminated. (see issue #1287)
  mi_lock_done(&tld->theaps_lock);
  _mi_meta_free(tld->subproc, tld, tld->memid);  // note: safe for static tld
}


/* -----------------------------------------------------------
  Thread Init
----------------------------------------------------------- */

#if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD)
static mi_theap_t* mi_heap_check_for_existing_theap(mi_heap_t* heap) {
  const mi_thread_id_t tid = _mi_thread_id();
  mi_theap_t* thread_theap = NULL;
  mi_lock(&heap->theaps_lock) {
    for(mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext ) {
      if (theap->tld->thread_id == tid && theap->tld->fork_epoch == _mi_fork_epoch()) {  // a dead theap in a forked child can have the same id
        thread_theap = theap;
        break;
      }
    }
  }
  return thread_theap;
}
#endif

// Initialize thread
mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main)
{
  // ensure our process has started already
  mi_process_init();

  // if the theap_default is already set we have already initialized
  mi_theap_t* theap = _mi_theap_default();
  if (mi_theap_is_initialized(theap)) return theap;
  _mi_theap_tls_check();  // report if the thread local theap was overwritten
  _mi_prim_thread_init();

  // initialize the default theap
  // note: we cannot access thread-locals yet as that can cause (recursive) allocation
  // (on macOS <= 14 for example where the loader allocates thread-local data on demand).
  if (heap_main==NULL) {
    heap_main = mi_heap_main();
    mi_assert_internal(mi_addr(heap_main) == mi_addr(&mi_process_metadata()->heap_main));
  }
  mi_assert_internal(heap_main!=NULL);

  #if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD)
  theap = mi_heap_check_for_existing_theap(heap_main);  // recursion check
  #if !defined(MI_TLS_RECURSE_GUARD)
  mi_assert_internal(theap==NULL);
  #endif
  #else
  theap = NULL;
  #endif

  if (theap==NULL) {
    // allocated the tld
    mi_tld_t* tld = mi_tld_create(heap_main->subproc);
    if (tld==NULL) return NULL;    // out-of-memory on tld allocation
    // allocate and initialize the theap for the main heap
    if (mi_addr(tld) == mi_addr(&mi_process_metadata()->tld_main)) {
      theap = mi_process_metadata_field(mi_theap_t, theap_main);  // initial theap is pre-allocated
      theap->memid = _mi_memid_create_static(theap,sizeof(*theap));
    }
    else {
      theap = _mi_theap_alloc(heap_main,tld);  // otherwise meta allocate
      if (theap==NULL) { mi_tld_free(tld); return NULL; } // out-of-memory on theap allocation
    }
    const bool attached = _mi_theap_init(theap,heap_main,tld);  // a main heap is never deleted while it is in use
    mi_assert_internal(attached); MI_UNUSED_RELEASE(attached);
  }

  // now initialize the thread
  _mi_theap_default_set(theap);
  // and only then set the heap_theap field as that accesses thread locals
  _mi_heap_theap_set(heap_main, theap);  // todo: can fail!

  mi_assert_internal(mi_theap_is_initialized(theap));
  mi_theap_t* const heap_theap = (heap_main==NULL ? NULL : (mi_theap_t*)_mi_thread_local_get(heap_main->theap));
  mi_assert_internal(heap_main==NULL || heap_theap == theap); MI_UNUSED_RELEASE(heap_theap);

  mi_subproc_stat_increase(mi_theap_subproc(theap), threads, 1);  // or theap stats and wait for merge?
  _mi_fork_child_try_collect();  // in a forked child, abandon dead theaps as they may have the same thread id as this thread
  // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
  return theap;
}

mi_theap_t* _mi_thread_init(void) {
  return _mi_thread_init_with_heap(NULL);
}

void mi_decl_noinline mi_thread_init(void) mi_attr_noexcept {
  _mi_thread_init();
}



/* -----------------------------------------------------------
  Theaps done
----------------------------------------------------------- */

// Free the thread local theaps
static void mi_thread_theaps_done(mi_tld_t* tld)
{
  // abandon the pages of all theaps in this thread
  mi_lock(&tld->theaps_lock) {
    mi_theap_t* theap = tld->theaps;
    while (theap != NULL) {
      mi_theap_t* next = theap->tnext;
      // never destroy theaps; if a dll is linked statically with mimalloc,
      // there may still be delete/free calls after the mi_fls_done is called. Issue #207
      mi_heap_t* const heap = mi_theap_heap(theap);
      mi_lock(&heap->theaps_lock) { // keep page queues stable during a heap walk
        _mi_theap_collect_abandon(theap);
      }
      mi_assert_internal(theap->page_count==0);
      theap = next;
    }
  }

  // reset the thread local theaps and mark the thread as done
  // note: do this after abandon as page->heap may be NULL and mi_heap_main should return the heap
  // belonging to the right subprocess
  _mi_theap_default_set(_mi_theap_unavailable());
  _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty);
  _mi_prim_thread_associate_default_theap(NULL);  // so the thread-done hook does not use the freed theap

  // We might run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically.
  // we first detach our theaps list from any heaps
  _mi_tld_detach_theaps(tld);

  // no heaps point to our theaps anymore, free them
  mi_lock(&tld->theaps_lock) { // paranoia
    mi_theap_t* theap = tld->theaps;
    tld->theaps = NULL;
    while (theap != NULL) {
      mi_theap_t* next = theap->tnext;
      mi_assert_internal(theap->page_count==0);
      mi_assert_internal(mi_theap_heap_peek(theap)==NULL);
      theap->tld = NULL;
      theap->tnext = NULL;
      theap->tprev = NULL;
      mi_assert_internal(mi_atomic_load_relaxed(&theap->refcount) == 1); // as the cached entry is set to empty
      _mi_theap_decref(theap);
      theap = next;
    }
  }

  mi_assert(_mi_thread_is_done()); // careful to not re-initialize the default theap during theap_delete
  mi_assert(!mi_theap_is_initialized(_mi_theap_default()));
}


// --------------------------------------------------------
// Try to run `mi_thread_done()` automatically so any memory
// owned by the thread but not yet released can be abandoned
// and re-owned by another thread.
//
// 1. windows dynamic library:
//     call from DllMain on DLL_THREAD_DETACH
// 2. windows static library:
//     use special linker section to call a destructor when the thread is done
// 3. unix, pthreads:
//     use a pthread key to call a destructor when a pthread is done
//
// In the last two cases we also need to call `mi_process_init`
// to set up the thread local keys.
// --------------------------------------------------------

// Set up hooks so `mi_thread_done` is called automatically
static void mi_process_setup_auto_thread_done(void) {
  mi_atomic_do_once {
    _mi_prim_thread_init_auto_done();
    mi_theap_t* theap = _mi_theap_default();
    mi_assert_internal(mi_theap_is_initialized(theap));
    if (mi_theap_is_initialized(theap)) {
      _mi_theap_default_set(theap);
    }
  }
}

void mi_thread_done(void) mi_attr_noexcept {
  _mi_thread_done(NULL);
}

// `_theap_main` is NULL or the (signed) theap associated with the thread (see `_mi_prim_thread_associate_default_theap`)
void _mi_thread_done(mi_theap_t* _theap_main)
{
  // NULL can be passed on some platforms
  if (_theap_main==NULL) {
    _theap_main = _mi_theap_default();
  }
  else {
    _theap_main = mi_theap_thread_done_auth(_theap_main);
    if (_theap_main==NULL) return;
  }

  // prevent re-entrancy through theap_done/theap_set_default_direct (issue #699)
  if (!mi_theap_is_initialized(_theap_main)) {
    return;
  }

  // get the current tld
  mi_tld_t* const tld = _theap_main->tld;

  // release dynamic thread_local's
  _mi_thread_locals_thread_done();

  // adjust stats
  mi_subproc_stat_decrease(tld->subproc, threads, 1);  // todo: or `_theap_main->heap`?

  // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local theaps...
  if (tld->thread_id != _mi_prim_thread_id()) return;

  // delete the thread local theaps
  mi_thread_theaps_done(tld);

  // free thread local data
  mi_tld_free(tld);
}

void mi_thread_set_in_threadpool(void) mi_attr_noexcept {
  mi_theap_t* theap = mi_theap_get_default();
  theap->tld->is_in_threadpool = true;
}


// --------------------------------------------------------
// Process init and done
// --------------------------------------------------------

// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.
bool mi_decl_noinline _mi_preloading(void) {
  return !_mi_process.libc_is_initialized;
}

// Returns true if mimalloc was redirected
mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {
  return _mi_is_redirected();
}

static void mi_auto_process_init_once(void) {
  _mi_process.libc_is_initialized = true;

  mi_process_init();

  _mi_options_post_init();  // now we can print to stderr
  if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n");

  // show message from the redirector (if present)
  const char* msg = NULL;
  _mi_allocator_init(&msg);
  if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {
    _mi_fputs(NULL,NULL,NULL,msg);
  }

  // reseed random
  mi_theap_t* theap = _mi_theap_default();
  if (theap != NULL) {
    _mi_random_reinit_if_weak(&theap->random);
    mi_subproc_t* subproc = mi_theap_subproc(theap);
    if (subproc->theap_meta != NULL) {
      mi_lock(&subproc->theap_meta_lock) {
        _mi_random_reinit_if_weak(&subproc->theap_meta->random);
      }
    }
  }

  // check if we should start the pprof profiler
  _mi_pprof_profiler_init();
}

// Called by the process loader from `src/prim/prim.c` before `main` is called.
void _mi_auto_process_init(void) {
  mi_atomic_do_once {
    mi_auto_process_init_once();
  }
}


// Initialize the process; called by thread_init, the process loader, or an initial allocation (perhaps by the loader or a system library)
static void mi_process_init_once(void) {
  #if defined(__CYGWIN__)   // we need to kickstart the cygwin runtime
  __mi_thread_id_helper = NULL;
  #endif
  _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());

  _mi_detect_cpu_features();
  _mi_options_init();        // read environment (if possible)
  _mi_stats_init();          // start timer
  _mi_os_init();             // primitive dependent
  _mi_process_metadata_init();  // after os_init so the OS page size is known
  mi_process_guard_init();   // after os_init so the OS page size is known
  mi_heap_main_init();       // before page_map_init so stats are working
  if (!_mi_page_map_init()) { _mi_fatal_error("unable to initialize the allocation page map\n"); }
  mi_process_sealed_init();  // after page_map_init so the page map is stored before sealing (no sealed-once writes are allowed after this)
  mi_thread_init();

  // the following can potentially allocate (on freeBSD for pthread keys)
  _mi_tls_slots_init();      // pthread key create
  _mi_thread_locals_init();  // pthread key create
  _mi_process.process_is_initialized = true;    // before `mi_process_setup_auto_thread_done` so `_mi_theap_default` returns the current theap with `MI_TLS_RECURSE_GUARD`
  mi_process_setup_auto_thread_done();  // after the above mi_thread_init so it can add the current theap
  _mi_fork_init();                      // register fork handlers (can allocate)

  #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS)
  // On windows, when building as a static lib the FLS cleanup happens to early for the main thread.
  // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup
  // will not call _mi_thread_done on the (still executing) main thread. See issue #508.
  _mi_prim_thread_associate_default_theap(NULL);
  #endif

  // mi_stats_reset();  // only call stat reset *after* thread init (or the theap tld == NULL)
  mi_track_init();
  if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
    size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024);
    int reserve_at  = (int)mi_option_get_clamp(mi_option_reserve_huge_os_pages_at, -1, INT_MAX);
    if (reserve_at != -1) {
      mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500);
    } else {
      mi_reserve_huge_os_pages_interleave(pages, 0, pages*500);
    }
  }
  if (mi_option_is_enabled(mi_option_reserve_os_memory)) {
    long ksize = mi_option_get(mi_option_reserve_os_memory);
    if (ksize > 0) {
      mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true);
    }
  }
}

// Initialize the process; called by thread_init or the process loader
void mi_process_init(void) mi_attr_noexcept {
  mi_atomic_do_once {
    mi_process_init_once();
  }
}

// Called once on process exit (when the process/dll is exiting gracefully)
static void mi_process_unwind_once(void)
{
  // stop the profiler
  _mi_pprof_profiler_done();

  // decref any cached theap
  _mi_theap_cached_set(_mi_theap_empty_get());

  // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread
  _mi_prim_thread_done_auto_done();

#ifndef MI_SKIP_COLLECT_ON_EXIT
#if (MI_DEBUG || !defined(MI_SHARED_LIB))
  // free all memory if possible on process exit. This is not needed for a stand-alone process
  // but should be done if mimalloc is statically linked into another shared library which
  // is repeatedly loaded/unloaded, see issue #281.
  mi_theap_collect(_mi_theap_default(), true /* force */);
#endif
#endif

  // done with tracking tools
  mi_track_done();

  // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free
  // since after process_done there might still be other code running that calls `free` (like at_exit routines,
  // or C-runtime termination code.
  mi_subproc_t* subproc_main = _mi_subproc_main();
  if (mi_option_is_enabled(mi_option_destroy_on_exit)) {
    _mi_subprocs_unsafe_destroy_all(); // destroys all mi_subprocs, arenas, thread locals, and the page_map!
  }
  else {
    // free dynamic thread locals (if used at all)
    _mi_thread_locals_thread_done();
    _mi_thread_locals_done();
    if (subproc_main->heap_main != NULL) {
      if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
        mi_theap_collect(subproc_main->theap_meta, false /* force */); // update stats of all pages in theap_meta
        _mi_theap_merge_stats(subproc_main->theap_meta);
        mi_theap_stats_merge_to_heap(_mi_theap_default());  // _mi_thread_locals_done can free; NULL or empty if `exit` is called from a thread that never allocated
        mi_heap_stats_merge_to_subproc(subproc_main->heap_main);
        mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); // note: can try to access (the now freed) thread_locals in mi_heap_theap_peek
      }
    }
  }

  _mi_tls_slots_done();
}

// Called when the process is done
static void mi_process_done_once(void) {
  // only shutdown if we were initialized
  if (!_mi_process.process_is_initialized) return;

  const size_t thread_count = mi_atomic_load_acquire(&_mi_subproc_main()->thread_count);
  if (thread_count == 1 || !_mi_prim_process_is_killed()) {  // can we safely take locks?
    mi_process_unwind_once();
    _mi_verbose_message("process done: 0x%zx\n", mi_process_metadata_field(mi_tld_t, tld_main)->thread_id);
  }

  _mi_subproc_main_done();
  _mi_allocator_done();
  _mi_process.libc_is_initialized = false; // don't call the C runtime anymore
}


// Call when the process is done (cdecl as it is used with `at_exit` on some platforms)
void mi_cdecl mi_process_done(void) mi_attr_noexcept {
  // we don't use `mi_atomic_do_once` here to avoid deadlocks if threads are killed due to a forced exit (issue #1377)
  size_t expected = 0;
  if (mi_atomic_cas_strong_relaxed(&_mi_process.process_done_is_called, &expected, 1)) {
    mi_process_done_once();
  }
}

// Called automatically when the process is done (cdecl as it is used with `at_exit` on some platforms)
void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept {
  if (_mi_option_get_fast(mi_option_destroy_on_exit)>=2) return;  // allow disabling auto process done
  mi_process_done();
}