#include "mimalloc.h"
#include "mimalloc/prim/prim-tls.h"
#include "mimalloc/util/libc.h"
#include "mimalloc/util/options.h"
#include "mimalloc/util/stats.h"
#include "mimalloc/profile/pprof.h"
#include "mimalloc/profile/sample-profile.h"
#include "mimalloc/heap/process.h"
#include "mimalloc/prim/os.h"
#include "pprof.h"
static bool mi_locations_init(mi_heap_t* heap, mi_locations_t* locations);
static void mi_locations_done(mi_heap_t* heap, mi_locations_t* locations);
static mi_location_t* mi_locations_find_or_insert(mi_heap_t* heap, mi_locations_t* locations, mi_thread_id_t thread_id, const mi_callstack_t* callstack, size_t callstack_hash);
static inline mi_pprof_profiler_t* downcast( mi_profiler_t* prof ) {
return (mi_pprof_profiler_t*)prof;
}
#if MI_PROFILE_USE_BUILTIN
void _mi_pprof_profiler_init(void) {
#if MI_PROFILE
char fname[1024];
if (_mi_getenv("mimalloc_profile", fname, sizeof(fname)) != 0 || fname[0] == 0) return; if (MI_SECURE_ENV || MI_SECURE > 0 || _mi_os_is_elevated()) {
_mi_warning_message("MIMALLOC_PROFILE is ignored in %s\n",
(MI_SECURE_ENV ? "secure environment mode" : (MI_SECURE > 0 ? "secure mode" : "an elevated process")));
return;
}
const size_t sample_rate = mi_option_get_size(mi_option_profile_sample_rate); const size_t alloc_interval_size = mi_option_get_size(mi_option_profile_alloc_interval); const size_t inuse_interval_size = mi_option_get_size(mi_option_profile_inuse_interval); const size_t time_interval_secs = mi_option_get_clamp(mi_option_profile_time_interval,0,24*60*60L); mi_profiler_t* profiler = mi_pprof_profiler_new(sample_rate, fname, alloc_interval_size, inuse_interval_size, time_interval_secs);
if (profiler == NULL) return;
mi_process_sealed_t* const sealed = _mi_process_unseal();
sealed->profile_env_profiler = (mi_profiler_t*)mi_ptr_memtag_meta(profiler); _mi_process_seal();
_mi_verbose_message("pprof profiler initialized with base file name: %s\n", fname);
mi_profile(profiler); mi_profiler_start(profiler);
#endif
}
void _mi_pprof_profiler_done(void) {
#if MI_PROFILE
mi_profiler_t* profiler = (mi_profiler_t*)mi_meta_load_maybe(mi_process_get_sealed()->profile_env_profiler);
if (profiler == NULL) return;
mi_process_sealed_t* const sealed = _mi_process_unseal();
sealed->profile_env_profiler = NULL;
_mi_process_seal();
mi_profiler_stop(profiler);
_mi_verbose_message("pprof profiler done\n");
mi_profiler_snapshot(profiler); mi_profile(NULL); #endif
}
#else
void _mi_pprof_profiler_init(void) {
}
void _mi_pprof_profiler_done(void) {
}
#endif
static mi_decl_noinline mi_location_t* mi_location_get(mi_pprof_profiler_t* prof) {
void* frames[MI_MAX_BACKTRACE_DEPTH + 1 + MI_BACKTRACE_SKIP_FRAMES]; size_t callstack_hash = 0;
const size_t depth = _mi_pprof_backtrace(frames, MI_MAX_BACKTRACE_DEPTH, &callstack_hash);
if (depth==0) return NULL;
mi_callstack_t callstack = { depth, frames };
const mi_thread_id_t thread_id = _mi_thread_id();
mi_location_t* loc = mi_locations_find_or_insert(prof->profile_heap, &prof->locations, thread_id, &callstack, callstack_hash);
return loc;
}
static size_t mi_exp_sample(mi_theap_t* theap, size_t scale) {
if (scale == 0) return 0; #if MI_PROFILE==2
if (scale == 1) return 1; #endif
const size_t r = _mi_theap_random_next(theap) | 1;
const size_t lz = mi_clz(r);
const size_t frac_q8 = (lz >= MI_SIZE_BITS - 1 ? 0 : (r << (lz + 1)) >> (MI_SIZE_BITS - 8)); const size_t log2m_q8 = frac_q8 + ((frac_q8 * (256 - frac_q8) * 89) >> 16); const size_t bits_q8 = ((lz + 1) << 8) - log2m_q8; size_t ln_q8 = (bits_q8 * 45426) >> 16; if (ln_q8 > (32 << 8)) { ln_q8 = (32 << 8); } size_t next;
if (mi_mul_overflow(scale >> 8, ln_q8, &next) || next > MI_SAMPLE_RATE_MAX) return MI_SAMPLE_RATE_MAX;
return next + (((scale & 0xFF) * ln_q8) >> 8) + 1; }
static bool mi_interval_update(uint64_t ubytes_since_last_sample, size_t alloc_interval_size, _Atomic(mi_ssize_t)* alloc_interval_countdown, mi_ssize_t* interval_pending) {
mi_ssize_t sampled_bytes = 0;
mi_ssize_t bytes_since_last_sample= (ubytes_since_last_sample > MI_SSIZE_MAX ? MI_SSIZE_MAX : (mi_ssize_t)ubytes_since_last_sample);
if (alloc_interval_size < MI_PROFILE_COUNTDOWN_MIN_INTERVAL || MI_SSIZE_MAX - *interval_pending < bytes_since_last_sample ) {
sampled_bytes = bytes_since_last_sample;
}
else {
*interval_pending += bytes_since_last_sample;
if (*interval_pending >= (mi_ssize_t)MI_PROFILE_COUNTDOWN_THRESHOLD) {
sampled_bytes = *interval_pending;
*interval_pending = 0;
}
}
if (sampled_bytes > 0) {
const mi_ssize_t new_countdown = mi_atomic_addi_relaxed(alloc_interval_countdown, -sampled_bytes) - sampled_bytes;
if (new_countdown <= 0 && new_countdown + sampled_bytes > 0) {
mi_atomic_storess_relaxed(alloc_interval_countdown, (mi_ssize_t)alloc_interval_size);
return true;
}
}
return false;
}
static bool mi_inuse_threshold_update(size_t new_total, size_t inuse_interval_size, _Atomic(size_t)* inuse_interval_threshold) {
size_t threshold = mi_atomic_load_relaxed(inuse_interval_threshold);
while (new_total >= threshold) {
size_t next_threshold = threshold + inuse_interval_size;
while (next_threshold <= new_total) { next_threshold += inuse_interval_size; } if (mi_atomic_cas_weak_relaxed(inuse_interval_threshold, &threshold, next_threshold)) {
return true;
}
}
return false;
}
static bool mi_time_threshold_update(mi_msecs_t now, size_t time_interval_secs, _Atomic(mi_msecs_t)* time_interval_deadline) {
const mi_msecs_t interval_msecs = (mi_msecs_t)time_interval_secs * 1000;
mi_msecs_t deadline = mi_atomic_loadi64_relaxed(time_interval_deadline);
while (now >= deadline) {
mi_msecs_t next_deadline = deadline + interval_msecs;
while (next_deadline <= now) { next_deadline += interval_msecs; } if (mi_atomic_casi64_strong_relaxed(time_interval_deadline, &deadline, next_deadline)) {
return true;
}
}
return false;
}
static size_t mi_cdecl on_alloc(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, size_t requested_size, size_t threshold, uint64_t bytes_since_last_sample, mi_heap_t* heap)
{
MI_UNUSED(threshold); MI_UNUSED(ptr);
mi_pprof_profiler_t* prof = downcast(profiler);
mi_theap_t* theap = _mi_heap_theap(heap);
mi_location_t* loc = mi_location_get(prof); const size_t new_threshold = mi_exp_sample(theap, prof->sample_threshold); const size_t alloc_size = requested_size;
if (data!=NULL) {
mi_assert(data->user_data_size >= 2*sizeof(void*));
data->user_data[0] = loc;
data->user_data[1] = (void*)((uintptr_t)alloc_size);
}
if (loc!=NULL) {
loc->alloc_bytes += (int64_t)alloc_size;
loc->alloc_count += 1;
mi_atomic_add_relaxed(&loc->inuse_bytes, alloc_size);
mi_atomic_increment_relaxed(&loc->inuse_count);
}
if (prof->alloc_interval_size > 0) { mi_ssize_t* pinterval_pending = (mi_ssize_t*)&theap->profiler_reserved1;
if (mi_interval_update(bytes_since_last_sample, prof->alloc_interval_size, &prof->alloc_interval_countdown, pinterval_pending)) {
mi_profiler_snapshot(profiler);
}
}
if (prof->inuse_interval_size > 0) { const size_t total_inuse_bytes = mi_atomic_add_relaxed(&prof->inuse_bytes, alloc_size) + alloc_size;
if (mi_inuse_threshold_update(total_inuse_bytes, prof->inuse_interval_size, &prof->inuse_interval_threshold)) {
mi_profiler_snapshot(profiler);
}
}
if (prof->time_interval_secs > 0) { if (mi_time_threshold_update(_mi_clock_now(), prof->time_interval_secs, &prof->time_interval_deadline)) {
mi_profiler_snapshot(profiler);
}
}
return new_threshold;
}
static void mi_cdecl on_free(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, mi_heap_t* heap)
{
MI_UNUSED(heap); MI_UNUSED(ptr);
mi_pprof_profiler_t* prof = downcast(profiler);
if (data!=NULL) {
mi_assert(data->user_data_size >= 2*sizeof(void*));
mi_location_t* loc = (mi_location_t*)data->user_data[0];
const size_t freed_bytes = (size_t)((uintptr_t)data->user_data[1]);
if (loc!=NULL) {
mi_atomic_sub_relaxed(&loc->inuse_bytes, freed_bytes);
mi_atomic_decrement_relaxed(&loc->inuse_count);
}
if (prof->inuse_interval_size > 0) { mi_atomic_sub_relaxed(&prof->inuse_bytes, freed_bytes);
}
}
}
static void mi_cdecl on_snapshot(mi_profiler_t* profiler) {
mi_pprof_profiler_t* prof = downcast(profiler);
_mi_pprof_profiler_snapshot(prof);
}
mi_profiler_t* mi_pprof_profiler_new(size_t initial_threshold, const char* base_file_name, size_t alloc_interval_size, size_t inuse_interval_size, size_t time_interval_secs) {
mi_heap_t* heap = mi_heap_new();
if (heap == NULL) return NULL;
mi_heap_profile_disable(heap);
mi_pprof_profiler_t* prof = mi_heap_zalloc_tp(mi_pprof_profiler_t,heap);
if (prof == NULL) { mi_heap_delete(heap); return NULL; }
prof->sample_threshold = initial_threshold;
prof->profile_heap = heap;
prof->base_file_name = (base_file_name != NULL ? mi_heap_strndup(heap, base_file_name, 1024) : NULL);
prof->format_text = false; if (prof->base_file_name != NULL) {
const char* start = prof->base_file_name;
const char* sep = _mi_strrchr(start, '/');
if (sep != NULL) start = sep + 1;
sep = _mi_strrchr(start, '\\');
if (sep != NULL) start = sep + 1;
const char* dot = _mi_strrchr(start, '.');
if (dot != NULL) {
if (_mi_streq(dot + 1, "heap") || _mi_streq(dot + 1, "text")) {
prof->format_text = true;
}
prof->base_file_name[dot - prof->base_file_name] = 0; }
}
prof->alloc_interval_size = (alloc_interval_size > (size_t)MI_SSIZE_MAX ? (size_t)MI_SSIZE_MAX : alloc_interval_size); mi_atomic_storess_relaxed(&prof->alloc_interval_countdown, (mi_ssize_t)prof->alloc_interval_size); prof->inuse_interval_size = inuse_interval_size;
mi_atomic_store_relaxed(&prof->inuse_interval_threshold, prof->inuse_interval_size); prof->time_interval_secs = time_interval_secs;
prof->start_time_secs = _mi_pprof_time_secs();
prof->start_clock = _mi_clock_now();
mi_atomic_storei64_relaxed(&prof->time_interval_deadline, prof->start_clock + (mi_msecs_t)prof->time_interval_secs * 1000); if (!mi_locations_init(heap, &prof->locations)) {
mi_heap_destroy(heap);
return NULL;
}
prof->profiler.initial_sample_rate = initial_threshold;
prof->profiler.sample_data_size = 2*sizeof(void*); prof->profiler.on_alloc = &on_alloc;
prof->profiler.on_free = &on_free;
prof->profiler.on_snapshot = &on_snapshot;
return &prof->profiler;
}
void mi_pprof_profiler_delete(mi_profiler_t* profiler) {
if (profiler == NULL) return;
mi_pprof_profiler_t* prof = downcast(profiler);
mi_heap_t* heap = prof->profile_heap;
mi_locations_done(heap, &prof->locations);
mi_free(prof->base_file_name);
mi_free(prof);
mi_heap_delete(heap);
}
#define MI_LOCATIONS_BUCKET_COUNT (16 * 1024 - 3)
static size_t mi_location_hash(mi_thread_id_t thread_id, size_t callstack_hash) {
size_t hash = MI_FNV_OFFSET_BASIS;
const size_t prime = MI_FNV_PRIME;
hash = (hash ^ (size_t)thread_id) * prime;
hash = (hash ^ callstack_hash) * prime;
return hash;
}
static bool mi_locations_init(mi_heap_t* heap, mi_locations_t* locations) {
locations->bucket_count = MI_LOCATIONS_BUCKET_COUNT;
locations->buckets = (_Atomic(mi_location_t*)*)mi_heap_zalloc(heap, locations->bucket_count * sizeof(*locations->buckets));
mi_atomic_store_relaxed(&locations->count, (size_t)0);
return (locations->buckets != NULL);
}
static void mi_locations_done(mi_heap_t* heap, mi_locations_t* locations) {
MI_UNUSED(heap);
if (locations->buckets == NULL) return;
for (size_t i = 0; i < locations->bucket_count; i++) {
mi_location_t* loc = mi_atomic_load_ptr_relaxed(mi_location_t, &locations->buckets[i]);
while (loc != NULL) {
mi_location_t* next = mi_atomic_load_ptr_relaxed(mi_location_t, &loc->next);
if (loc->callstack.frames != NULL) {
mi_free(loc->callstack.frames);
}
mi_free(loc);
loc = next;
}
}
mi_free(locations->buckets);
locations->buckets = NULL;
mi_atomic_store_relaxed(&locations->count, (size_t)0);
}
static bool mi_location_matches(const mi_location_t* loc, size_t hash, mi_thread_id_t thread_id, const mi_callstack_t* callstack) {
return (loc->hash == hash && loc->thread_id == thread_id &&
loc->callstack.count == callstack->count &&
(callstack->count == 0 || mi_memcmp(loc->callstack.frames, callstack->frames, callstack->count * sizeof(void*)) == 0));
}
static mi_location_t* mi_locations_find_or_insert(mi_heap_t* heap, mi_locations_t* locations, mi_thread_id_t thread_id, const mi_callstack_t* callstack, size_t callstack_hash) {
if (locations->buckets == NULL) return NULL;
const size_t hash = mi_location_hash(thread_id, callstack_hash);
const size_t idx = hash % locations->bucket_count;
_Atomic(mi_location_t*)* bucket = &locations->buckets[idx];
for (mi_location_t* loc = mi_atomic_load_ptr_relaxed(mi_location_t, bucket); loc != NULL;
loc = mi_atomic_load_ptr_relaxed(mi_location_t, &loc->next))
{
if (mi_location_matches(loc, hash, thread_id, callstack)) return loc;
}
mi_location_t* loc = mi_heap_zalloc_tp(mi_location_t,heap);
if (loc == NULL) return NULL;
if (callstack->count > 0) {
loc->callstack.frames = (void**)mi_heap_mallocn(heap, callstack->count, sizeof(void*));
if (loc->callstack.frames == NULL) { mi_free(loc); return NULL; } mi_memcpy(loc->callstack.frames, callstack->frames, callstack->count * sizeof(void*));
}
loc->callstack.count = callstack->count;
loc->thread_id = thread_id;
loc->hash = hash;
mi_location_t* head = mi_atomic_load_ptr_relaxed(mi_location_t, bucket);
do {
mi_atomic_store_ptr_relaxed(mi_location_t, &loc->next, head); } while (!mi_atomic_cas_ptr_weak_release(mi_location_t, bucket, &head, loc)); mi_atomic_increment_relaxed(&locations->count); return loc;
}