#if !defined(MI_IN_ALLOC_C)
#error "this file should be included from 'alloc.c' (so aliases can work from alloc-override)"
#endif
#include "mimalloc.h"
#include "mimalloc/types.h"
#include "mimalloc/prim/prim-tls.h"
#include "mimalloc/util/stats.h"
#include "mimalloc/arena/arena-page.h"
#include "mimalloc/arena/arena.h"
#include "mimalloc/profile/sample-guarded.h"
#include "mimalloc/alloc/page.h"
#include "mimalloc/alloc/free.h"
#include "mimalloc/heap/heap.h"
mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size);
mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block);
static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
static inline void mi_debug_fill_prim(void* block, int c, size_t size, size_t preserve) {
if (preserve < size) {
mi_memset((uint8_t*)block + preserve, c, size - preserve);
}
}
static inline void mi_debug_fill(mi_page_t* page, void* block, int c, size_t size) {
const size_t preserve = mi_atomic_load_relaxed(&mi_page_heap(page)->debug_fill_preserve);
if mi_unlikely(preserve != mi_atomic_load_relaxed(&page->debug_fill_preserve)) {
mi_atomic_store_relaxed(&page->debug_fill_preserve, preserve);
}
mi_debug_fill_prim(block, c, size, preserve);
}
static inline void mi_debug_fill_mt(const mi_page_t* page, void* block, int c, size_t size) {
mi_debug_fill_prim(block, c, size, mi_atomic_load_relaxed(&page->debug_fill_preserve));
}
#endif
static inline mi_block_t* mi_block_free_clear(const mi_page_t* page, mi_block_t* block, bool was_guarded, bool memtag) {
if (was_guarded) return block;
if (memtag) {
const size_t bsize = mi_page_block_size(page);
const void* prev; const void* next;
const size_t exclude = mi_page_block_memtag_adjacent(page, block, bsize, mi_memtag_class_of(block), &prev, &next);
return (mi_block_t*)mi_memtag_block_free(block, bsize, exclude, prev, next);
}
#if MI_SECURE>=3
const size_t usize = mi_page_usable_block_size(page);
mi_memzero_block(block, (usize <= MI_ZERO_ON_FREE_MAX ? usize : MI_ZERO_ON_FREE_MAX));
#endif
return block;
}
static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool check_full, bool memtag)
{
size_t usable_size;
if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return;
if mi_unlikely(!mi_check_double_free(page,block)) return;
block = mi_block_free_clear(page, block, was_guarded, memtag); mi_track_free_size(block, usable_size);
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size);
mi_debug_fill(page, block, MI_DEBUG_FREED, dbgsize);
#endif
mi_used_t xused = page->xused;
mi_block_t* lfree = page->local_free;
xused.used_alloc--;
mi_block_set_next(page, block, lfree);
page->xused = xused;
page->local_free = block;
const bool is_empty = (mi_xused_used_count(xused) == 0);
mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page));
if mi_unlikely(is_empty) { _mi_page_qsbr_clear(page); if (page->retire_expire==0) { _mi_page_retire(page);
}
}
else if mi_unlikely(check_full && mi_page_is_in_full(page)) {
_mi_page_unfull(page);
}
}
static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept;
#if defined(_MSC_VER)
static mi_decl_noinline
#else
static inline
#endif
void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim, bool memtag) mi_attr_noexcept
{
size_t usable_size;
if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; block = mi_block_free_clear(page, block, was_guarded, memtag); mi_track_free_size(block, usable_size);
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
if (!was_guarded) {
const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size);
mi_debug_fill_mt(page, block, MI_DEBUG_FREED, dbgsize);
}
#endif
mi_thread_free_t tf_new;
mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
do {
mi_block_set_next(page, block, mi_tf_block(tf_old));
tf_new = mi_tf_create(block, true);
} while(!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));
mi_assert_internal(mi_tf_is_owned(tf_new));
const bool is_newly_owned = !mi_tf_is_owned(tf_old); if (is_newly_owned) {
mi_assert_internal(mi_page_is_abandoned(page));
mi_free_try_collect_mt(page, block, allow_reclaim);
}
}
static inline size_t mi_page_block_index(const mi_page_t* page, size_t diff) {
#if MI_FREE_IS_CHECKED
mi_assert_internal(page->block_recip==1 || diff <= UINT32_MAX);
const size_t idx = (size_t)mi_mul_hi64_32((uint32_t)diff, page->block_recip);
mi_assert_internal(idx == diff / mi_page_block_size(page));
return idx;
#else
return diff / mi_page_block_size(page);
#endif
}
static inline mi_block_t* mi_page_ptr_unalign_ex(const mi_page_t* page, const void* p, size_t* poffset) {
mi_assert_internal(page!=NULL && p!=NULL);
const size_t diff = mi_addr(p) - mi_addr(mi_page_start(page));
const size_t block_size = mi_page_block_size(page);
size_t adjust = diff & (block_size - 1);
if mi_unlikely(!mi_is_power_of_two(block_size)) {
#if MI_FREE_IS_CHECKED
adjust = diff - (mi_page_block_index(page, diff) * block_size); #else
adjust = diff % block_size;
#endif
}
if (poffset!=NULL) { *poffset = adjust; }
void* const block = mi_ptr_rederive(mi_page_start(page), p);
return (mi_block_t*)mi_ptr_with_addr(block, mi_addr(p) - adjust);
}
mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
return mi_page_ptr_unalign_ex(page,p,NULL);
}
void* _mi_memtag_canonical(void* p) {
if (p==NULL || mi_ptr_memtag(p)==MI_MEMTAG_META) return p;
mi_page_t* const page = mi_ptr_page(p);
mi_block_t* const block = _mi_page_ptr_unalign(page, p);
mi_memtag_clear(block, mi_page_block_size(page));
mi_page_set_has_interior_pointers(page, true); return mi_ptr_with_memtag(p, MI_MEMTAG_META);
}
static inline mi_block_t* mi_block_from_ptr( const mi_page_t* page, const void* p ) {
mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); return (mi_block_t*)mi_ptr_rederive(mi_page_start(page), p);
}
static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) {
#if MI_SECURE>=2
mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); return _mi_page_ptr_unalign(page,p);
#else
return mi_block_from_ptr(page,p);
#endif
}
static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool* was_guarded) mi_attr_noexcept {
MI_UNUSED(was_guarded);
if mi_likely(!mi_page_has_interior_pointers(page)) {
return mi_validate_block_from_ptr(page,p);
}
else {
size_t offset;
mi_block_t* block = mi_page_ptr_unalign_ex(page,p,&offset);
#if MI_GUARDED || MI_PROFILE
if (offset >= sizeof(mi_block_t)) {
#if MI_PROFILE
if (block->next == MI_BLOCK_TAG_PROFILED) {
_mi_page_profile_on_free(page,block,p);
}
else
#endif
#if MI_GUARDED
if (block->next == MI_BLOCK_TAG_GUARDED) {
_mi_page_block_unguard(page, block, p);
*was_guarded = true;
}
else
#endif
{ }
}
#endif
if mi_unlikely(mi_page_is_memtag(page) && !*was_guarded && mi_ptr_memtag(block)==MI_MEMTAG_META) { const size_t bsize = mi_page_block_size(page);
const size_t idx = mi_page_block_index(page, mi_addr(block) - mi_addr(mi_page_start(page))) + mi_page_memtag_parity(page);
const void* prev; const void* next;
const size_t exclude = mi_page_block_memtag_adjacent(page, block, bsize, idx, &prev, &next);
block = (mi_block_t*)mi_memtag_block_uncanonical(block, bsize, exclude, prev, next);
}
return block;
}
}
static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {
mi_assert_internal(p!=NULL && page != NULL);
bool was_guarded = false;
mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded);
mi_free_block_local(page, block, was_guarded, true , mi_page_is_memtag(page));
}
static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept {
mi_assert_internal(p!=NULL && page != NULL);
bool was_guarded = false;
mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded);
mi_free_block_mt(page, block, was_guarded, allow_reclaim, mi_page_is_memtag(page));
}
void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {
if (is_local) mi_free_generic_local(page,p);
else mi_free_generic_mt(page,p,true);
}
#if MI_DEBUG || MI_CHERI
static mi_decl_noinline mi_decl_cold bool mi_ptr_invalid(const char* msg, const void* p, const char* reason) {
_mi_error_message(EINVAL, "%s: invalid pointer%s: %p\n", msg, reason, p);
return false;
}
#endif
static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const char* msg, bool free_small, bool check_p_for_null, mi_page_t** ppage)
{
MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small);
mi_page_t* page;
#if MI_PAGE_META_SMALL_IS_ALIGNED
if (free_small) {
const mi_addr_t up = mi_addr(p) & ~MI_ADDR(MI_SMALL_PAGE_SIZE-1); if mi_unlikely(up==0 && check_p_for_null) { return false; }
page = mi_page_meta_load((mi_page_t*)mi_ptr_with_addr(p, up));
}
else
#endif
{
#if MI_PAGE_META_IS_ALIGNED
{ page = mi_aligned_ptr_page0(p); }
#else
{ page = mi_ptr_page(p); }
#endif
if mi_unlikely(mi_ptr_page_is_null(p,page) && check_p_for_null) {
#if MI_DEBUG || MI_CHERI
if (p!=NULL) { return mi_ptr_invalid(msg, p, ""); }
#endif
return false;
}
#if MI_PAGE_META_IS_ALIGNED
page = mi_page_meta_load(page);
#endif
}
#if MI_DEBUG
mi_page_t* const cpage = mi_checked_ptr_page(p);
if mi_unlikely(cpage==NULL) { mi_ptr_invalid(msg, p, ""); }
#endif
#if MI_PAGE_META_IS_ALIGNED
#if MI_PAGE_META_SMALL_IS_ALIGNED
if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); }
else
#elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE && !MI_GUARDED
if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); }
else
#endif
{
#if MI_TSAN
page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self);
#else
page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self);
#endif
}
#endif
mi_assert_internal(page!=NULL);
mi_assert(cpage==page );
#if MI_DEBUG && !MI_UNSAFE_GUARDED_PRECISE
if mi_unlikely(!mi_is_aligned(p,MI_SIZE_SIZE) && !mi_page_has_interior_pointers(page)) {
return mi_ptr_invalid(msg, p, " (unaligned)");
}
#endif
#if !MI_GUARDED
if (free_small) { mi_assert_internal(page->block_size <= MI_SMALL_MAX_OBJ_SIZE); }
#endif
#if MI_FREE_IS_CHECKED
if mi_unlikely((size_t)(mi_addr(p) - mi_addr(mi_page_start(page))) >= mi_atomic_load_relaxed(&page->capacity_size)) {
#if MI_DEBUG
return mi_ptr_invalid(msg, p, "");
#else
return false;
#endif
}
#endif
#if MI_CHERI && MI_FREE_IS_CHECKED
const size_t ofs = (size_t)__builtin_cheri_offset_get(p);
if mi_unlikely(ofs >= (size_t)__builtin_cheri_length_get(p) || __builtin_cheri_sealed_get(p) ||
(__builtin_cheri_perms_get(p) & MI_CHERI_PERMS_FREE) != MI_CHERI_PERMS_FREE) {
return mi_ptr_invalid(msg, p, " (sealed, insufficient permissions, or outside its capability bounds)");
}
#endif
*ppage = page;
return true;
}
static mi_decl_forceinline bool mi_ptr_page_is_valid(const void* p, const char* msg, mi_page_t** ppage) {
return mi_ptr_page_is_valid_ex(p,msg,false , true , ppage );
}
static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const char* msg) {
mi_page_t* page;
return (mi_ptr_page_is_valid(p,msg,&page) ? page : NULL);
}
#if MI_SECURE>=2
static inline bool mi_page_is_block_start(const mi_page_t* page, const void* p) {
const uint64_t diff = mi_addr(p) - mi_addr(mi_page_start(page));
return (diff * page->block_recip < page->block_recip);
}
static mi_decl_noinline void mi_free_unaligned(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept {
void* const q = (mi_page_has_interior_pointers(page) ? p : (void*)_mi_page_ptr_unalign(page,p));
if (mi_page_thread_id(page) == _mi_thread_id()) { mi_free_generic_local(page, q); }
else { mi_free_generic_mt(page, q, allow_reclaim); }
}
#endif
static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_reclaim)
{
mi_assert_internal(p!=NULL && page!=NULL);
if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); }
#if MI_SECURE>=2
if mi_unlikely(!mi_page_is_block_start(page,p)) { mi_free_unaligned(page, p, allow_reclaim); return; }
#endif
const mi_thread_id_t ptid = mi_page_xthread_id(page);
const mi_thread_id_t xtid = (_mi_prim_thread_id() ^ ptid);
if mi_likely(xtid == 0) { mi_block_t* const block = mi_block_from_ptr(page,p);
mi_free_block_local(page, block, false , false , false );
}
#if MI_MTE
else if (xtid == MI_PAGE_IS_MEMTAGGED) { mi_block_t* const block = mi_block_from_ptr(page,p);
mi_free_block_local(page, block, false , false , true );
}
#endif
else if (xtid <= MI_PAGE_FLAG_MASK) { mi_free_generic_local(page, p);
}
else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { mi_block_t* const block = mi_block_from_ptr(page,p);
mi_free_block_mt(page,block,false , allow_reclaim, false );
}
#if MI_MTE
else if ((xtid & MI_PAGE_FLAG_MASK) == MI_PAGE_IS_MEMTAGGED) { mi_block_t* const block = mi_block_from_ptr(page,p);
mi_free_block_mt(page,block,false , allow_reclaim, true );
}
#endif
else {
mi_free_generic_mt(page, p, allow_reclaim);
}
}
void mi_free(void* p) mi_attr_noexcept {
mi_page_t* page = NULL;
if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) {
mi_free_nonnull(p, page, NULL, true );
}
}
void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept {
mi_page_t* page = NULL;
if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) {
mi_free_nonnull(p, page, pblock_size, true );
}
else {
if (pblock_size!=NULL) { *pblock_size = 0; }
}
}
void _mi_free_subproc_safe(void* p) mi_attr_noexcept {
mi_page_t* page = NULL;
if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) {
mi_free_nonnull(p, page, NULL, false );
}
}
void mi_free_small(void* p) mi_attr_noexcept {
mi_page_t* page = NULL;
if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true ,true , &page)) {
mi_free_nonnull(p, page, NULL, true );
}
}
void mi_free_small_nonnull(void* p) mi_attr_noexcept {
mi_assert(p!=NULL);
mi_page_t* page = NULL;
if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true ,false , &page)) {
mi_free_nonnull(p, page, NULL, true );
}
}
static mi_decl_forceinline void mi_free_small_local_ex(void* p, bool check_p_for_null) mi_attr_noexcept {
mi_assert(check_p_for_null || p!=NULL);
mi_page_t* page = NULL;
if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_local_ex", true , check_p_for_null , &page)) {
mi_assert_internal(mi_page_thread_id(page) == _mi_thread_id());
const mi_page_flags_t flags = mi_page_flags(page);
if mi_likely(flags == 0 || (MI_PAGE_IS_MEMTAGGED != 0 && flags == MI_PAGE_IS_MEMTAGGED)) {
mi_block_t* const block = mi_validate_block_from_ptr(page,p);
mi_free_block_local(page, block, false , false , flags != 0 );
return;
}
else {
mi_free_generic_local(page, p);
}
}
}
void mi_free_small_local(void* p) mi_attr_noexcept {
mi_free_small_local_ex(p, true );
}
void mi_free_small_local_nonnull(void* p) mi_attr_noexcept {
mi_free_small_local_ex(p, false );
}
void mi_free_size(void* p, size_t size) mi_attr_noexcept {
MI_UNUSED_RELEASE(size);
#if MI_DEBUG
const mi_page_t* const page = mi_ptr_page_validate(p,"mi_free_size");
if (page==NULL) return;
mi_assert(p!=NULL);
const mi_block_t* block = _mi_page_ptr_unalign(page, p);
const size_t usable = _mi_page_usable_size(page,p);
const bool is_guarded = mi_block_ptr_is_guarded(block,p); if mi_unlikely(size > usable) {
if (!is_guarded) {
_mi_error_message(EINVAL, "pointer %p is freed with 'mi_free_size' but the size %zu is greater than the usable size %zu\n", p, size, usable);
mi_free(p);
return;
}
}
else if mi_unlikely(size <= MI_SMALL_SIZE_MAX && !mi_page_is_small(page) && !is_guarded) {
_mi_error_message(EINVAL, "pointer %p is freed with 'mi_free_size' with size %zu (<= MI_SMALL_SIZE_MAX), but it was allocated in a non-small page with block size %zu -- maybe a mismatched delete operator?\n", p, size, mi_page_block_size(page));
mi_free(p);
return;
}
#endif
#if MI_PAGE_META_SMALL_IS_ALIGNED
if mi_likely(size <= MI_SMALL_SIZE_MAX) {
mi_free_small(p);
}
else
#endif
{
mi_free(p);
}
}
void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {
mi_assert((mi_addr(p) % alignment) == 0);
if mi_likely(alignment <= size) {
mi_free_size(p,size);
}
else {
mi_free(p);
}
}
void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
MI_UNUSED_RELEASE(alignment);
mi_assert((mi_addr(p) % alignment) == 0);
mi_free(p);
}
bool mi_cfree(void* p) mi_attr_noexcept {
mi_page_t* const page = mi_checked_ptr_page(p);
if mi_likely(page!=NULL) {
mi_free_nonnull(p, page, NULL, true );
return true;
}
else {
return false;
}
}
static bool mi_abandoned_page_collect_empty(mi_page_t* page)
{
if (!mi_page_all_free(page)) return false;
const bool should_delay = _mi_page_qsbr_should_delay(page);
_mi_arenas_page_unabandon(page,NULL);
if mi_likely(!should_delay) {
_mi_page_qsbr_clear(page);
_mi_arenas_page_free_direct(page,NULL);
}
else {
mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
if (theap != NULL && theap == page->theap) {
_mi_theap_page_reclaim(theap, page);
}
else {
_mi_arenas_page_free(page,NULL);
}
}
return true;
}
static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)
{
if (mi_page_is_mostly_used(page)) return false; if (page->memid.memkind != MI_MEM_ARENA || mi_page_is_abandoned_mapped(page)) return false;
mi_assert(!mi_page_is_full(page));
return _mi_arenas_page_try_reabandon_to_mapped(page);
}
static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* page_thread_free) {
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(!mi_page_all_free(page));
mi_thread_free_t tf_expect = mi_tf_create(page_thread_free,true);
mi_thread_free_t tf_new = mi_tf_set_owned(tf_expect,false);
while mi_unlikely(!mi_atomic_cas_strong_release(&page->xthread_free, &tf_expect, tf_new)) {
mi_assert_internal(mi_tf_is_owned(tf_expect));
while (mi_tf_block(tf_expect) != NULL) {
_mi_page_free_collect(page,false); if (mi_abandoned_page_collect_empty(page)) return;
if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
tf_expect = mi_atomic_load_acquire(&page->xthread_free);
}
mi_subproc_stat_counter_increase(mi_page_subproc(page), pages_unown_cas_retry, 1);
mi_assert_internal(mi_tf_block(tf_expect)==NULL);
tf_new = mi_tf_create(NULL, false);
}
}
static inline bool mi_page_queue_len_is_atmost( mi_theap_t* theap, size_t block_size, long atmost) {
if (atmost < 0) return false;
mi_page_queue_t* const pq = mi_page_queue(theap,block_size);
mi_assert_internal(pq!=NULL);
return (pq->count <= (size_t)atmost);
}
static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long reclaim_on_free) mi_attr_noexcept
{
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(!mi_page_all_free(page));
mi_assert_internal(page->block_size <= MI_MAX_RECLAIM_OBJ_SIZE);
mi_assert_internal(reclaim_on_free >= 0);
if (!_mi_thread_is_initialized()) return false;
mi_theap_t* const theap = _mi_page_associated_theap_peek(page);
if (theap==NULL || theap->tld==NULL || !theap->allow_page_reclaim) return false;
long max_reclaim = 0;
if mi_likely(theap == page->theap) { max_reclaim = _mi_option_get_fast(theap->tld->is_in_threadpool ? mi_option_page_cross_thread_max_reclaim : mi_option_page_max_reclaim);
}
else if (reclaim_on_free == 1 && !theap->tld->is_in_threadpool && !mi_page_is_used_at_frac(page,4) && _mi_arena_memid_is_suitable(page->memid, mi_theap_heap(theap)->exclusive_arena)) { max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim);
}
else {
return false;
}
if (max_reclaim >= 0 && !mi_page_queue_len_is_atmost(theap, page->block_size, max_reclaim)) {
return false;
}
_mi_arenas_page_unabandon(page, theap);
_mi_theap_page_reclaim(theap, page);
mi_theap_stat_counter_increase(theap, pages_reclaim_on_free, 1);
return true;
}
static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept
{
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(mt_free != NULL);
if (page->block_size <= MI_SMALL_SIZE_MAX) {
mi_assert_internal(page->reserved>=16); mt_free = _mi_page_free_collect_partly(page, mt_free);
}
else {
_mi_page_free_collect(page,false );
mt_free = NULL; }
const long reclaim_on_free = mi_heap_reclaim_on_free(mi_page_heap(page));
#if MI_DEBUG > 1
if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }
if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); }
#endif
if (mi_abandoned_page_collect_empty(page)) return;
if (allow_reclaim && page->block_size <= MI_MAX_RECLAIM_OBJ_SIZE && reclaim_on_free >= 0) { if (mi_abandoned_page_try_reclaim(page, reclaim_on_free)) return;
}
if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
mi_abandoned_page_unown_from_free(page, mt_free);
}
static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p, const mi_block_t* block) mi_attr_noexcept {
const bool is_guarded = mi_block_ptr_is_guarded(block,p);
const size_t size = mi_page_usable_size_of(page, block, is_guarded);
mi_assert_internal((void*)p >= (void*)block);
const size_t adjust = (uint8_t*)p - (uint8_t*)block;
mi_assert_internal(adjust <= size);
const size_t aligned_size = (adjust <= size ? size - adjust : 0); return aligned_size;
}
size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept {
if mi_unlikely(page==NULL) return 0;
mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page);
const mi_block_t* block = _mi_page_ptr_unalign(page, p); if mi_likely((const void*)block==p) {
return mi_ptr_bounds_limit(p, mi_page_usable_size_of(page, block, false ));
}
else {
return mi_ptr_bounds_limit(p, mi_page_usable_aligned_size_of(page, p, block));
}
}
mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept {
const mi_page_t* const page = mi_ptr_page_validate(p,"mi_usable_size");
return _mi_page_usable_size(page,p);
}
mi_decl_nodiscard size_t mi_thread_id_of(const void* p) mi_attr_noexcept {
const mi_page_t* const page = mi_ptr_page_validate(p,"mi_thread_id_of");
if (page == NULL) return 0;
#if MI_QSBR
return mi_page_qsbr_thread_id(page);
#else
const mi_thread_id_t tid = mi_page_thread_id(page);
return (tid <= MI_THREAD_ID_DETACHED ? 0 : tid);
#endif
}
#if MI_SECURE>=3 && !MI_PADDING
static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) {
const size_t max_count = page->capacity; size_t count = 0;
while (list != NULL && count <= max_count) { if (elem==list) return true;
list = mi_block_next(page, list);
count++;
}
if mi_unlikely(count > max_count) {
_mi_error_message(EFAULT, "corrupted %s list (possibly due to a double free)\n", list_kind);
}
return false;
}
static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const mi_block_t* block) {
if (mi_list_contains(page, page->free, block, "free") ||
mi_list_contains(page, page->local_free, block, "local free") ||
mi_list_contains(page, mi_page_thread_free(page), block, "thread free"))
{
_mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page));
return false;
}
return true;
}
static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) {
if (!mi_is_aligned(block,MI_SIZE_SIZE)) return false; bool valid;
mi_block_decode(page, block, mi_block_link(block), &valid);
return valid; }
static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) {
if mi_unlikely(mi_block_could_be_double_free(page,block)) {
return mi_check_double_freex(page, block);
}
else return true;
}
#else
static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) {
MI_UNUSED(page);
MI_UNUSED(block);
return true;
}
#endif
#if MI_PADDING
static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize, bool* double_free) {
*bsize = mi_page_usable_block_size(page);
mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize);
mi_track_mem_defined(padding,sizeof(mi_padding_t));
*delta = padding->delta;
const uint32_t canary = padding->canary;
const bool ok = (mi_ptr_encode_canary(page,block) == canary && *delta <= *bsize);
if (double_free!=NULL) {
*double_free = (!ok && mi_ptr_decode_canary_is_freed(canary));
if (ok) { padding->canary = mi_ptr_encode_canary_freed(); } }
mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
return ok;
}
static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
if mi_unlikely(is_guarded) {
const size_t bsize = mi_page_block_size(page);
return (bsize - _mi_os_page_size());
}
else {
size_t bsize;
size_t delta;
bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL);
mi_assert_internal(ok); mi_assert_internal(delta <= bsize);
return (ok ? bsize - delta : 0);
}
}
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
if (mi_block_ptr_is_guarded(block,block + 1)) return;
size_t bsize;
size_t delta;
bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL);
mi_assert_internal(ok);
if (!ok || (bsize - delta) >= min_size) return; mi_assert_internal(bsize >= min_size);
if (bsize < min_size) return; size_t new_delta = (bsize - min_size);
mi_assert_internal(new_delta < bsize);
mi_padding_t* padding = (mi_padding_t*)((uint8_t*)block + bsize);
mi_track_mem_defined(padding,sizeof(mi_padding_t));
padding->delta = (uint32_t)new_delta;
mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
}
#else
static inline size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
MI_UNUSED(block);
if mi_unlikely(is_guarded) {
const size_t bsize = mi_page_block_size(page);
return (bsize - _mi_os_page_size());
}
else {
return mi_page_usable_block_size(page);
}
}
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(min_size);
}
#endif
#if MI_PADDING
static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong, bool* is_double_free) {
size_t bsize;
size_t delta;
bool ok = mi_page_decode_padding(page, block, &delta, &bsize, is_double_free );
*size = *wrong = bsize;
if (!ok) return false;
mi_assert_internal(bsize >= delta);
*size = bsize - delta;
#if MI_PADDING_CHECK_BYTES
if (!mi_page_is_huge(page)) {
uint8_t* fill = (uint8_t*)block + bsize - delta;
const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); mi_track_mem_defined(fill, maxpad);
for (size_t i = 0; i < maxpad; i++) {
if (fill[i] != MI_DEBUG_PADDING) {
*wrong = bsize - delta + i;
ok = false;
break;
}
}
mi_track_mem_noaccess(fill, maxpad);
}
#endif
return ok;
}
mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) {
if mi_unlikely(is_guarded) {
const size_t bsize = mi_page_block_size(page);
*usable_size = (bsize - _mi_os_page_size());
return true;
}
else {
size_t wrong;
bool is_double_free;
if mi_unlikely(!mi_verify_padding(page,block,usable_size,&wrong,&is_double_free)) {
if (is_double_free) {
_mi_error_message(EAGAIN, "double free detected of heap block %p with size %zu\n", block, *usable_size);
}
else {
_mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, *usable_size, wrong );
}
return false;
}
return true;
}
}
#else
mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) {
*usable_size = mi_page_usable_size_of(page,block,is_guarded);
return true;
}
#endif
#if 0#else
void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
MI_UNUSED(page); MI_UNUSED(block);
}
#endif