#include "mimalloc.h"
#include "mimalloc/prim/prim.h"
#include "mimalloc/prim/prim-tls.h"
#include "mimalloc/util/random.h"
#include "mimalloc/util/stats.h"
#include "mimalloc/util/memid.h"
#include "mimalloc/alloc/page.h"
#include "mimalloc/arena/arena-alloc.h"
#include "mimalloc/arena/arena-page.h"
#include "mimalloc/profile/sample-guarded.h"
#include "mimalloc/profile/sample-profile.h"
#include "mimalloc/heap/heap.h"
#include "mimalloc/heap/theap.h"
#if defined(_MSC_VER) && (_MSC_VER < 1920)
#pragma warning(disable:4204)
#endif
typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2);
static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2)
{
if (theap==NULL || theap->page_count==0) return true;
#if MI_DEBUG>1
size_t total = theap->page_count;
size_t count = 0;
#endif
const size_t max_bin = (include_full ? MI_BIN_FULL : MI_BIN_FULL - 1);
for (size_t i = 0; i <= max_bin; i++) {
mi_page_queue_t* pq = &theap->pages[i];
mi_page_t* page = pq->first;
while(page != NULL) {
mi_page_t* next = page->next; mi_assert_internal(mi_page_theap(page) == theap);
#if MI_DEBUG>1
count++;
#endif
if (!fn(theap, pq, page, arg1, arg2)) return false;
page = next; }
}
mi_assert_internal(!include_full || count == total);
return true;
}
#if MI_DEBUG>=3
static bool mi_theap_page_is_valid(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
MI_UNUSED(arg1);
MI_UNUSED(arg2);
MI_UNUSED(pq);
mi_assert_internal(mi_page_theap(page) == theap);
mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap);
mi_assert_internal(page_theap == NULL || theap == page_theap || mi_theap_is_detached(theap));
mi_assert_expensive(_mi_page_is_valid(page));
return true;
}
static bool mi_theap_is_valid(mi_theap_t* theap) {
mi_assert_internal(theap!=NULL);
mi_heap_t* const heap = mi_theap_heap_peek(theap);
mi_assert_internal(heap != NULL);
mi_theap_t* const heap_theap = _mi_heap_theap_peek(heap); mi_assert_internal(heap_theap==NULL || heap_theap == theap || mi_theap_is_detached(theap));
mi_theap_visit_pages(theap, &mi_theap_page_is_valid, true, NULL, NULL);
for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) {
mi_assert_internal(_mi_page_queue_is_valid(theap, &theap->pages[bin]));
}
return true;
}
#endif
typedef enum mi_collect_e {
MI_NORMAL,
MI_FORCE,
MI_ABANDON
} mi_collect_t;
static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {
MI_UNUSED(arg2);
MI_UNUSED(theap);
mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL));
mi_collect_t collect = *((mi_collect_t*)arg_collect);
_mi_page_free_collect(page, collect >= MI_FORCE); _mi_page_update_stats(page);
if (mi_page_all_free(page)) {
if (collect >= MI_FORCE || page->retire_expire == 0) { if (_mi_page_free(page, pq)) return true;
}
}
if (collect == MI_ABANDON) {
_mi_page_abandon(page, pq, true);
}
return true; }
void _mi_theap_merge_stats(mi_theap_t* theap) {
mi_assert_internal(mi_theap_is_initialized(theap));
mi_heap_t* const heap = mi_theap_heap(theap);
_mi_stats_merge_into(&heap->stats, &theap->stats);
}
static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect)
{
if (theap==NULL || !mi_theap_is_initialized(theap) || !mi_theap_handle_is_valid(theap)) return;
mi_assert_expensive(mi_theap_is_valid(theap));
const bool force = (collect >= MI_FORCE);
if (collect != MI_ABANDON) { _mi_deferred_free(theap, force);
}
_mi_theap_collect_retired(theap, force);
mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL);
if (collect != MI_ABANDON) {
_mi_heap_qsbr_collect_delayed(theap->heap); _mi_arenas_collect(collect == MI_FORCE , collect >= MI_FORCE , theap->tld);
}
if (mi_option_is_enabled(mi_option_stats_merge_on_collect)) {
_mi_theap_merge_stats(theap);
}
}
void _mi_theap_collect_abandon(mi_theap_t* theap) {
mi_theap_collect_ex(theap, MI_ABANDON);
}
void _mi_theap_abandon_owned_pages(mi_theap_t* theap) {
mi_assert_internal(theap!=NULL);
mi_assert_internal(theap->tld == NULL); if (theap==NULL) return;
for (size_t bin = 0; bin <= MI_BIN_FULL; bin++) {
mi_page_queue_t* const pq = &theap->pages[bin];
mi_page_t* page = pq->first;
while (page != NULL) {
mi_page_t* const next = page->next; _mi_page_free_collect(page, true); if (!mi_page_all_free(page) || !_mi_page_free(page, pq)) {
_mi_page_abandon(page, pq, true);
}
page = next;
}
}
}
void mi_theap_collect(mi_theap_t* theap, bool force) mi_attr_noexcept {
mi_theap_collect_ex(theap, (force ? MI_FORCE : MI_NORMAL));
}
void mi_collect(bool force) mi_attr_noexcept {
mi_theap_collect(_mi_theap_default(), force);
}
void mi_heap_collect(mi_heap_t* heap, bool force) {
mi_theap_collect(mi_heap_theap(heap), force);
}
mi_theap_t* mi_theap_get_default(void) {
mi_theap_t* theap = _mi_theap_default();
if mi_unlikely(!mi_theap_is_initialized(theap)) {
mi_thread_init();
theap = _mi_theap_default();
mi_assert_internal(mi_theap_is_initialized(theap));
}
return theap;
}
mi_decl_cold bool _mi_theap_invalid(const mi_theap_t* theap) {
_mi_error_message(EFAULT, "invalid theap handle %p (the theap was deleted or the handle was corrupted)\n", theap);
return false;
}
mi_theap_t* mi_theap_set_default(mi_theap_t* theap) {
mi_theap_t* const previous = mi_theap_get_default();
if (mi_theap_is_initialized(theap) && mi_theap_handle_is_valid(theap)) {
_mi_theap_default_set(theap);
}
return previous;
}
static void mi_theap_options_init(mi_theap_t* theap, mi_heap_t* heap) {
theap->allow_page_reclaim = (mi_heap_reclaim_on_free(heap) >= 0);
theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);
theap->is_detached = (theap->tld->thread_id == MI_THREAD_ID_DETACHED);
}
static void mi_theap_detach_from_tld(mi_theap_t* theap);
static void mi_theap_free_mem(mi_theap_t* theap);
static bool mi_theap_attach_to_heap(mi_theap_t* theap, mi_heap_t* heap) {
bool attached = false;
mi_lock(&heap->theaps_lock) {
if (!heap->is_deleted) {
mi_atomic_store_ptr_release(mi_heap_t,&theap->heap,heap);
mi_theap_t* const head = heap->theaps;
theap->hprev = NULL;
theap->hnext = head;
if (head!=NULL) { head->hprev = theap; }
heap->theaps = theap;
attached = true;
}
}
return attached;
}
bool _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld)
{
mi_assert_internal(theap!=NULL);
mi_assert_internal(heap!=NULL);
mi_assert_internal(tld!=NULL);
mi_memid_t memid = theap->memid;
mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t));
theap->memid = memid;
theap->cookie = mi_theap_cookie(theap);
theap->tld = tld; theap->profile_never = false; mi_atomic_store_release(&theap->refcount,1);
mi_atomic_store_ptr_release(mi_subproc_t,&theap->subproc,heap->subproc);
mi_assert_internal(theap->stats.size == sizeof(mi_stats_t));
mi_theap_options_init(theap, heap);
theap->guard_front = false;
theap->guard_end = (MI_SECURE >= 5);
if (theap->tld->is_in_threadpool) {
if (theap->page_full_retain > 0) {
theap->page_full_retain = theap->page_full_retain / 4;
}
}
mi_theap_t* head = NULL;
mi_random_ctx_t head_random;
mi_lock(&theap->tld->theaps_lock) {
head = theap->tld->theaps;
theap->tprev = NULL;
theap->tnext = head;
theap->tld->theaps = theap;
if (head!=NULL) {
head->tprev = theap;
head_random = head->random;
}
}
if (head==NULL) {
}
else {
_mi_random_split(&head_random, &theap->random); }
_mi_theap_guarded_init(theap); if (!theap->is_detached) {
mi_subproc_stat_increase(mi_theap_subproc(theap),theaps,1); }
if (!mi_theap_attach_to_heap(theap, heap)) {
mi_lock(&tld->theaps_lock) { mi_theap_detach_from_tld(theap); }
return false;
}
return true;
}
void _mi_theap_meta_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) {
const bool attached = _mi_theap_init(theap, heap, tld); mi_assert_internal(attached); MI_UNUSED_RELEASE(attached);
theap->allow_page_abandon = false; theap->page_full_retain = 2;
theap->sample_rate = 0; theap->sample_countdown = 0;
theap->guard_front = (MI_SECURE > 0);
theap->guard_end = (MI_SECURE > 0);
mi_theap_profile_never(theap); }
mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld) {
mi_assert_internal(tld!=NULL);
mi_assert_internal(heap!=NULL);
mi_assert_internal(tld->thread_id == MI_THREAD_ID_DETACHED || _mi_thread_id() == tld->thread_id);
mi_memid_t memid;
mi_theap_t* theap;
if (heap->exclusive_arena == NULL) {
theap = (mi_theap_t*)_mi_meta_zalloc(heap->subproc, sizeof(mi_theap_t), &memid);
}
else {
const size_t size = mi_align_up(sizeof(mi_theap_t),MI_ARENA_MIN_OBJ_SIZE);
theap = (mi_theap_t*)_mi_arenas_alloc(heap, size, true, true, heap->exclusive_arena, tld->thread_seq, tld->numa_node, &memid);
}
if (theap==NULL) {
_mi_error_message(ENOMEM, "unable to allocate theap meta-data\n");
return NULL;
}
theap->memid = memid;
return theap;
}
mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) {
mi_theap_t* theap = _mi_theap_alloc(heap,tld);
if (theap == NULL) return NULL;
if (!_mi_theap_init(theap, heap, tld)) { mi_theap_free_mem(theap);
return NULL;
}
return theap;
}
size_t _mi_theap_random_next(mi_theap_t* theap) {
return _mi_random_next(&theap->random);
}
static void mi_theap_free_mem(mi_theap_t* theap) {
if (theap!=NULL) {
#if MI_QSBR
mi_assert_internal(theap->qsbr_delayed == NULL && theap->qsbr_delayed_last == NULL);
#endif
mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc);
if (!theap->is_detached) {
mi_subproc_stat_decrease(subproc,theaps,1);
}
theap->cookie = 0; _mi_meta_free(subproc, theap, theap->memid);
}
}
void _mi_theap_incref(mi_theap_t* theap) {
if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) {
mi_atomic_increment_acq_rel(&theap->refcount);
}
}
void _mi_theap_decref(mi_theap_t* theap) {
if (theap!=NULL && !mi_memid_needs_no_free(theap->memid)) {
if (mi_atomic_decrement_acq_rel(&theap->refcount) == 1) {
mi_theap_free_mem(theap);
}
}
}
static void mi_theap_detach_from_tld(mi_theap_t* theap) {
mi_tld_t* const tld = theap->tld;
mi_list_unlink_check(theap, tprev, tnext, tld->theaps, NULL);
if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; }
if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; }
else { mi_assert_internal(tld->theaps == theap); tld->theaps = theap->tnext; }
theap->tnext = theap->tprev = NULL;
theap->tld = NULL;
}
static void mi_theap_detach_from_heap(mi_theap_t* theap) {
mi_heap_t* const heap = mi_theap_heap(theap);
mi_list_unlink_check(theap, hprev, hnext, heap->theaps, NULL);
if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; }
if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; }
else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; }
theap->hnext = theap->hprev = NULL;
mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL);
}
void _mi_heap_detach_theaps( mi_heap_t* heap ) {
bool all_detached;
do {
all_detached = true;
mi_lock(&heap->theaps_lock) {
mi_theap_t* theap = heap->theaps;
while (theap != NULL) {
mi_theap_t* next = theap->hnext;
mi_tld_t* tld = theap->tld;
if (tld != NULL) {
if (mi_lock_try_acquire(&tld->theaps_lock)) {
mi_theap_detach_from_tld(theap);
mi_lock_release(&tld->theaps_lock);
}
else {
all_detached = false;
}
}
theap = next;
}
}
if (!all_detached) {
mi_subproc_stat_counter_increase(heap->subproc,heaps_delete_wait,1);
_mi_prim_thread_yield();
}
} while (!all_detached);
}
void _mi_tld_detach_theaps( mi_tld_t* tld ) {
bool all_detached;
do {
all_detached = true;
mi_lock(&tld->theaps_lock) {
mi_theap_t* theap = tld->theaps;
while (theap != NULL) {
mi_theap_t* next = theap->tnext;
mi_assert_internal(theap->page_count==0);
mi_heap_t* heap = mi_theap_heap_peek(theap); if (heap != NULL) {
if (mi_lock_try_acquire(&heap->theaps_lock)) {
_mi_stats_merge_into(&heap->stats, &theap->stats);
mi_theap_detach_from_heap(theap);
mi_lock_release(&heap->theaps_lock);
}
else {
all_detached = false;
}
}
theap = next;
}
}
if (!all_detached) {
mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1);
_mi_prim_thread_yield();
}
} while (!all_detached);
}
void _mi_theap_release(mi_theap_t* theap, mi_theap_t* slot_theap) {
if (theap==NULL) return;
mi_theap_t* const main_theap = _mi_theap_main();
if (main_theap==NULL) {
_mi_warning_message("cannot delete a theap on an uninitialized thread\n");
return;
}
mi_tld_t* const tld = main_theap->tld;
if (_mi_theap_default() == theap) {
_mi_theap_default_set(main_theap);
}
bool found = false;
bool release = false;
mi_lock(&tld->theaps_lock) {
mi_theap_t* owned = tld->theaps;
while (owned!=NULL && owned!=theap) { owned = owned->tnext; }
if (owned!=NULL) {
found = true;
mi_heap_t* const heap = mi_theap_heap(owned);
mi_assert_internal(heap!=NULL);
mi_lock(&heap->theaps_lock) {
if (owned == main_theap) {
_mi_theap_collect_abandon(owned); _mi_theap_merge_stats(owned);
}
else {
const bool cleared = _mi_thread_local_set(heap->theap, slot_theap);
mi_assert_internal(cleared); MI_UNUSED_RELEASE(cleared);
mi_theap_detach_from_tld(owned);
_mi_theap_abandon_owned_pages(owned);
_mi_theap_merge_stats(owned);
mi_theap_detach_from_heap(owned);
release = true;
}
}
}
}
if (_mi_theap_cached() == theap) {
_mi_theap_cached_set(_mi_theap_empty_get());
}
if (release) { _mi_theap_decref(theap); }
if (!found) { _mi_verbose_message("theap %p is no longer owned by this thread\n", theap); }
}
void mi_theap_delete(mi_theap_t* theap) mi_attr_noexcept {
_mi_theap_release(theap, NULL);
}
void mi_theap_quiescent_abandon(mi_theap_t* theap) mi_attr_noexcept {
if (theap==NULL || !mi_theap_is_initialized(theap) || !mi_theap_handle_is_valid(theap)) return;
mi_tld_t* const tld = theap->tld;
mi_heap_t* const heap = mi_theap_heap(theap);
mi_lock_maybe(&tld->theaps_lock, tld!=NULL) {
mi_lock(&heap->theaps_lock) {
theap->tld = NULL;
_mi_theap_abandon_owned_pages(theap);
theap->tld = tld;
_mi_theap_merge_stats(theap);
}
}
}
void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {
const size_t bsize = mi_page_block_size(page);
const size_t ubsize = mi_page_usable_block_size(page);
area->reserved = page->reserved * bsize;
area->committed = page->capacity * bsize;
area->blocks = mi_ptr_bounded_data(mi_page_start(page), area->reserved);
area->used = mi_page_used(page); area->block_size = ubsize;
area->full_block_size = bsize;
area->reserved1 = NULL;
}
static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {
mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);
*shift = MI_SIZE_BITS - mi_clz(divisor - 1);
*magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);
}
static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) {
mi_assert_internal(n <= UINT32_MAX);
const uint64_t hi = ((uint64_t)n * magic) >> 32;
return (size_t)((hi + n) >> shift);
}
static inline void* mi_visit_block_ptr(uint8_t* block, size_t ubsize, bool memtag) {
return mi_ptr_bounded_data((memtag ? mi_memtag_load(block) : block), ubsize);
}
bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) {
mi_assert(area != NULL);
if (area==NULL) return true;
mi_assert(page != NULL);
if (page == NULL) return true;
if (mi_page_is_owned(page)) { _mi_page_free_collect(page,true); mi_assert_internal(page->local_free == NULL);
}
if (mi_page_used(page) == 0) return true;
size_t psize;
uint8_t* const pstart = mi_page_area(page, &psize);
mi_heap_t* const heap = mi_page_heap(page);
const size_t bsize = mi_page_block_size(page);
const size_t ubsize = mi_page_usable_block_size(page); const bool memtag = mi_page_is_memtag(page);
if (page->capacity == 1) {
mi_assert_internal(mi_page_used(page) == 1 && page->free == NULL);
return visitor(heap, area, mi_visit_block_ptr(pstart, ubsize, memtag), ubsize, arg);
}
mi_assert(bsize <= UINT32_MAX);
if (mi_page_used(page) == page->capacity) {
uint8_t* block = pstart;
for (size_t i = 0; i < page->capacity; i++) {
if (!visitor(heap, area, mi_visit_block_ptr(block, ubsize, memtag), ubsize, arg)) return false;
block += bsize;
}
return true;
}
#define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*))
size_t free_map[MI_MAX_BLOCKS / MI_SIZE_BITS];
const size_t bmapsize = mi_divide_up(page->capacity, MI_SIZE_BITS);
memset(free_map, 0, bmapsize * sizeof(size_t));
if (page->capacity % MI_SIZE_BITS != 0) {
size_t shift = (page->capacity % MI_SIZE_BITS);
size_t mask = (SIZE_MAX << shift);
free_map[bmapsize - 1] = mask;
}
uint64_t magic;
size_t shift;
mi_get_fast_divisor(bsize, &magic, &shift);
#if MI_DEBUG>1
size_t free_count = 0;
bool free_list_corrupted = false;
#endif
for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page, block)) {
#if MI_DEBUG>1
free_count++;
#endif
mi_assert_internal(mi_addr(block) >= mi_addr(pstart) && mi_addr(block) < mi_addr(pstart + psize));
size_t offset = mi_addr(block) - mi_addr(pstart);
mi_assert_internal(offset % bsize == 0);
mi_assert_internal(offset <= UINT32_MAX);
size_t blockidx = mi_fast_divide(offset, magic, shift);
mi_assert_internal(blockidx == offset / bsize);
mi_assert_internal(blockidx < MI_MAX_BLOCKS);
size_t bitidx = (blockidx / MI_SIZE_BITS);
size_t bit = blockidx - (bitidx * MI_SIZE_BITS);
const size_t mask = (size_t)1 << bit;
if mi_unlikely((free_map[bitidx] & mask) != 0) {
_mi_warning_message("corrupted free list (possibly due to a double free)\n");
#if MI_DEBUG>1
free_list_corrupted = true;
#endif
break; }
free_map[bitidx] |= mask;
}
mi_assert_internal(free_list_corrupted || page->capacity == (free_count + mi_page_used(page)));
#if MI_DEBUG>1
size_t used_count = 0;
#endif
uint8_t* block = pstart;
for (size_t i = 0; i < bmapsize; i++) {
if (free_map[i] == 0) {
for (size_t j = 0; j < MI_SIZE_BITS; j++) {
#if MI_DEBUG>1
used_count++;
#endif
if (!visitor(heap, area, mi_visit_block_ptr(block, ubsize, memtag), ubsize, arg)) return false;
block += bsize;
}
}
else {
size_t m = ~free_map[i];
while (m != 0) {
#if MI_DEBUG>1
used_count++;
#endif
size_t bitidx = mi_ctz(m);
if (!visitor(heap, area, mi_visit_block_ptr(block + (bitidx * bsize), ubsize, memtag), ubsize, arg)) return false;
m &= m - 1; }
block += bsize * MI_SIZE_BITS;
}
}
mi_assert_internal(free_list_corrupted || mi_page_used(page) == used_count);
return true;
}
typedef struct mi_theap_area_ex_s {
mi_heap_area_t area;
mi_page_t* page;
} mi_theap_area_ex_t;
typedef bool (mi_theap_area_visit_fun)(const mi_theap_t* theap, const mi_theap_area_ex_t* area, void* arg);
static bool mi_theap_visit_areas_page(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) {
MI_UNUSED(theap);
MI_UNUSED(pq);
mi_theap_area_visit_fun* fun = (mi_theap_area_visit_fun*)vfun;
mi_theap_area_ex_t xarea;
xarea.page = page;
_mi_page_free_collect(page, true); _mi_heap_area_init(&xarea.area, page);
return fun(theap, &xarea, arg);
}
static bool mi_theap_visit_areas(const mi_theap_t* theap, mi_theap_area_visit_fun* visitor, void* arg) {
if (visitor == NULL) return false;
if (theap==NULL || !mi_theap_is_initialized(theap) || !mi_theap_handle_is_valid(theap)) return false;
return mi_theap_visit_pages((mi_theap_t*)theap, &mi_theap_visit_areas_page, true, (void*)(visitor), arg); }
typedef struct mi_visit_blocks_args_s {
bool visit_blocks;
mi_block_visit_fun* visitor;
void* arg;
} mi_visit_blocks_args_t;
static bool mi_theap_area_visitor(const mi_theap_t* theap, const mi_theap_area_ex_t* xarea, void* arg) {
mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg;
if (!args->visitor(mi_theap_heap(theap), &xarea->area, NULL, xarea->area.block_size, args->arg)) return false;
if (args->visit_blocks) {
return _mi_theap_area_visit_blocks(&xarea->area, xarea->page, args->visitor, args->arg);
}
else {
return true;
}
}
bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
mi_visit_blocks_args_t args = { visit_blocks, visitor, arg };
return mi_theap_visit_areas(theap, &mi_theap_area_visitor, &args);
}