Skip to main content

kaniop_operator/
metrics.rs

1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicI64, Ordering};
6
7use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
8use opentelemetry::{KeyValue, trace::TraceId};
9use tokio::time::Instant;
10use tracing::debug;
11
12pub const KANIDM_RESOURCE_PERSON: &str = "Person";
13pub const KANIDM_RESOURCE_GROUP: &str = "Group";
14pub const KANIDM_RESOURCE_OAUTH2: &str = "OAuth2Client";
15pub const KANIDM_RESOURCE_SERVICE_ACCOUNT: &str = "ServiceAccount";
16pub const KANIDM_RESOURCE_DOMAIN: &str = "Domain";
17pub const KANIDM_RESOURCE_MAIL_SENDER: &str = "MailSender";
18
19pub const KANIDM_OP_GET: &str = "get";
20pub const KANIDM_OP_GET_CREDENTIAL_STATUS: &str = "get_credential_status";
21pub const KANIDM_OP_CREATE: &str = "create";
22pub const KANIDM_OP_UPDATE: &str = "update";
23pub const KANIDM_OP_DELETE: &str = "delete";
24pub const KANIDM_OP_UNIX_EXTEND: &str = "unix_extend";
25pub const KANIDM_OP_CREDENTIAL_UPDATE_INTENT: &str = "credential_update_intent";
26pub const KANIDM_OP_CREDENTIAL_UPDATE_SEND_INTENT: &str = "credential_update_send_intent";
27pub const KANIDM_OP_SET_MEMBERS: &str = "set_members";
28pub const KANIDM_OP_SET_MAIL: &str = "set_mail";
29pub const KANIDM_OP_PURGE_MAIL: &str = "purge_mail";
30pub const KANIDM_OP_ADD_MAIL: &str = "add_mail";
31pub const KANIDM_OP_REMOVE_MAIL: &str = "remove_mail";
32pub const KANIDM_OP_ACCOUNT_POLICY: &str = "account_policy";
33pub const KANIDM_OP_PURGE_ATTR: &str = "purge_attr";
34pub const KANIDM_OP_ROTATE_SECRET: &str = "rotate_secret";
35pub const KANIDM_OP_GET_SECRET: &str = "get_secret";
36pub const KANIDM_OP_ADD_ORIGIN: &str = "add_origin";
37pub const KANIDM_OP_REMOVE_ORIGIN: &str = "remove_origin";
38pub const KANIDM_OP_SCOPE_MAP: &str = "scope_map";
39pub const KANIDM_OP_DELETE_SCOPE_MAP: &str = "delete_scope_map";
40pub const KANIDM_OP_UPDATE_SCOPE_MAP: &str = "update_scope_map";
41pub const KANIDM_OP_SUP_SCOPE_MAP: &str = "sup_scope_map";
42pub const KANIDM_OP_DELETE_SUP_SCOPE_MAP: &str = "delete_sup_scope_map";
43pub const KANIDM_OP_UPDATE_SUP_SCOPE_MAP: &str = "update_sup_scope_map";
44pub const KANIDM_OP_CLAIM_MAP: &str = "claim_map";
45pub const KANIDM_OP_DELETE_CLAIM_MAP: &str = "delete_claim_map";
46pub const KANIDM_OP_UPDATE_CLAIM_MAP: &str = "update_claim_map";
47pub const KANIDM_OP_CLAIM_MAP_JOIN: &str = "claim_map_join";
48pub const KANIDM_OP_UPDATE_CLAIM_MAP_JOIN: &str = "update_claim_map_join";
49pub const KANIDM_OP_IMAGE: &str = "image";
50pub const KANIDM_OP_DELETE_IMAGE: &str = "delete_image";
51pub const KANIDM_OP_UPDATE_IMAGE: &str = "update_image";
52pub const KANIDM_OP_RESET_SECRET: &str = "reset_secret";
53pub const KANIDM_OP_GENERATE_API_TOKEN: &str = "generate_api_token";
54pub const KANIDM_OP_DESTROY_API_TOKEN: &str = "destroy_api_token";
55pub const KANIDM_OP_LIST_API_TOKENS: &str = "list_api_tokens";
56pub const KANIDM_OP_GENERATE_PASSWORD: &str = "generate_password";
57pub const KANIDM_OP_SET_DISPLAY_NAME: &str = "set_display_name";
58pub const KANIDM_OP_ADD_MEMBERS: &str = "add_members";
59pub const KANIDM_OP_REMOVE_MEMBERS: &str = "remove_members";
60
61pub const KANIDM_OUTCOME_CHANGED: &str = "changed";
62pub const KANIDM_OUTCOME_UNCHANGED: &str = "unchanged";
63pub const KANIDM_OUTCOME_ERROR: &str = "error";
64
65pub async fn record_kanidm_sdk_call<F, T, E>(
66    metrics: &ControllerMetrics,
67    resource: &'static str,
68    operation: &'static str,
69    success_outcome: &'static str,
70    future: F,
71) -> std::result::Result<T, E>
72where
73    F: std::future::Future<Output = std::result::Result<T, E>>,
74{
75    let start = Instant::now();
76    let result = future.await;
77    let outcome = match &result {
78        Ok(_) => success_outcome,
79        Err(_) => KANIDM_OUTCOME_ERROR,
80    };
81    metrics.kanidm_sdk_calls.add(
82        1,
83        &[
84            KeyValue::new("resource", resource),
85            KeyValue::new("operation", operation),
86            KeyValue::new("outcome", outcome),
87        ],
88    );
89    metrics.kanidm_sdk_call_duration.record(
90        start.elapsed().as_secs_f64(),
91        &[
92            KeyValue::new("resource", resource),
93            KeyValue::new("operation", operation),
94        ],
95    );
96    result
97}
98
99#[derive(Clone)]
100pub struct Metrics {
101    pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
102}
103
104impl Metrics {
105    pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
106        debug!(
107            "Initializing operator metrics for controllers: {:?}",
108            controller_names
109        );
110        let controllers = controller_names
111            .iter()
112            .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
113            .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
114
115        debug!("Operator metrics initialized");
116        Self { controllers }
117    }
118}
119
120#[derive(Clone)]
121pub struct ControllerMetrics {
122    controller: String,
123    pub reconcile: ReconcileMetrics,
124    spec_replicas: Gauge<i64>,
125    status_update_errors: Counter<u64>,
126    triggered: Counter<u64>,
127    watch_operations_failed: Counter<u64>,
128    ready: Gauge<i64>,
129    active_reconciles: Gauge<i64>,
130    active_reconciles_count: Arc<AtomicI64>,
131    objects_in_backoff: Gauge<i64>,
132    objects_in_backoff_count: Arc<AtomicI64>,
133    kanidm_sdk_calls: Counter<u64>,
134    kanidm_sdk_call_duration: Histogram<f64>,
135    reconcile_outcome: Counter<u64>,
136}
137
138impl ControllerMetrics {
139    pub fn new(controller: &str, meter: &Meter) -> Self {
140        let reconcile = ReconcileMetrics::new(meter);
141
142        let spec_replicas = meter
143            .i64_gauge("spec_replicas")
144            .with_description("Number of expected replicas for the object")
145            .build();
146
147        let status_update_errors = meter
148            .u64_counter("status_update_errors")
149            .with_description(
150                "Number of errors that occurred during update operations to status subresources",
151            )
152            .build();
153
154        let triggered = meter
155            .u64_counter("triggered")
156            .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
157            .build();
158
159        let watch_operations_failed = meter
160            .u64_counter("watch_operations_failed")
161            .with_description("Total number of watch operations that failed")
162            .build();
163
164        let ready = meter
165            .i64_gauge("ready")
166            .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
167            .build();
168
169        let active_reconciles = meter
170            .i64_gauge("active_reconciles")
171            .with_description("Number of reconcile operations currently in flight")
172            .build();
173
174        let objects_in_backoff = meter
175            .i64_gauge("objects_in_backoff")
176            .with_description("Number of objects currently in error backoff state")
177            .build();
178
179        let kanidm_sdk_calls = meter
180            .u64_counter("kanidm_sdk_calls")
181            .with_description("Total number of Kanidm SDK calls")
182            .build();
183
184        let kanidm_sdk_call_duration = meter
185            .f64_histogram("kanidm_sdk_call_duration_seconds")
186            .with_description("Duration of Kanidm SDK calls in seconds")
187            .with_boundaries(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0])
188            .build();
189
190        let reconcile_outcome = meter
191            .u64_counter("reconcile_outcome")
192            .with_description("Outcome of reconcile operations: changed, unchanged, or error")
193            .build();
194
195        Self {
196            controller: controller.to_string(),
197            reconcile,
198            spec_replicas,
199            status_update_errors,
200            triggered,
201            watch_operations_failed,
202            ready,
203            active_reconciles,
204            active_reconciles_count: Arc::new(AtomicI64::new(0)),
205            objects_in_backoff,
206            objects_in_backoff_count: Arc::new(AtomicI64::new(0)),
207            kanidm_sdk_calls,
208            kanidm_sdk_call_duration,
209            reconcile_outcome,
210        }
211    }
212
213    pub fn reconcile_failure_inc(&self) {
214        self.reconcile
215            .failures
216            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
217    }
218
219    pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
220        self.reconcile
221            .operations
222            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
223        ReconcileMeasurer {
224            start: Instant::now(),
225            controller: self.controller.clone(),
226            metric: self.reconcile.duration.clone(),
227        }
228    }
229
230    pub fn reconcile_deploy_delete_create_inc(&self, resource_kind: &str, reason: &str) {
231        self.reconcile.deploy_delete_create.add(
232            1,
233            &[
234                KeyValue::new("controller", self.controller.clone()),
235                KeyValue::new("resource_kind", resource_kind.to_string()),
236                KeyValue::new("reason", reason.to_string()),
237            ],
238        );
239    }
240
241    pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
242        self.spec_replicas.record(
243            replicas as i64,
244            &[
245                KeyValue::new("controller", self.controller.clone()),
246                KeyValue::new("namespace", namespace.to_string()),
247                KeyValue::new("name", name.to_string()),
248            ],
249        );
250    }
251
252    pub fn status_update_errors_inc(&self) {
253        self.status_update_errors
254            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
255    }
256
257    pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
258        self.triggered.add(
259            1,
260            &[
261                KeyValue::new("controller", self.controller.clone()),
262                KeyValue::new("action", action.as_str()),
263                KeyValue::new("triggered_by", triggered_by.to_string()),
264            ],
265        );
266    }
267
268    pub fn watch_operations_failed_inc(&self) {
269        self.watch_operations_failed
270            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
271    }
272
273    pub fn ready_set(&self, status: i64) {
274        self.ready.record(
275            status,
276            &[KeyValue::new("controller", self.controller.clone())],
277        );
278    }
279
280    pub fn active_reconcile_inc(&self) -> ActiveReconcileGuard {
281        let new = self.active_reconciles_count.fetch_add(1, Ordering::Relaxed) + 1;
282        self.active_reconciles
283            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
284        ActiveReconcileGuard {
285            count: self.active_reconciles_count.clone(),
286            gauge: self.active_reconciles.clone(),
287            controller: self.controller.clone(),
288        }
289    }
290
291    pub fn objects_in_backoff_inc(&self) {
292        let new = self
293            .objects_in_backoff_count
294            .fetch_add(1, Ordering::Relaxed)
295            + 1;
296        self.objects_in_backoff
297            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
298    }
299
300    pub fn objects_in_backoff_dec(&self) {
301        let prev = self
302            .objects_in_backoff_count
303            .fetch_sub(1, Ordering::Relaxed);
304        let new = (prev - 1).max(0);
305        self.objects_in_backoff
306            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
307    }
308
309    pub fn active_reconciles_count(&self) -> i64 {
310        self.active_reconciles_count.load(Ordering::Relaxed)
311    }
312
313    pub fn objects_in_backoff_count(&self) -> i64 {
314        self.objects_in_backoff_count.load(Ordering::Relaxed)
315    }
316
317    pub fn kanidm_sdk_call_inc(
318        &self,
319        resource: &'static str,
320        operation: &'static str,
321        outcome: &'static str,
322    ) {
323        self.kanidm_sdk_calls.add(
324            1,
325            &[
326                KeyValue::new("resource", resource),
327                KeyValue::new("operation", operation),
328                KeyValue::new("outcome", outcome),
329            ],
330        );
331    }
332
333    pub fn record_kanidm_sdk_outcome(
334        &self,
335        resource: &'static str,
336        operation: &'static str,
337        outcome: &'static str,
338        duration: std::time::Duration,
339    ) {
340        self.kanidm_sdk_calls.add(
341            1,
342            &[
343                KeyValue::new("resource", resource),
344                KeyValue::new("operation", operation),
345                KeyValue::new("outcome", outcome),
346            ],
347        );
348        self.kanidm_sdk_call_duration.record(
349            duration.as_secs_f64(),
350            &[
351                KeyValue::new("resource", resource),
352                KeyValue::new("operation", operation),
353            ],
354        );
355    }
356
357    pub fn reconcile_outcome_record(&self, outcome: &'static str) {
358        self.reconcile_outcome.add(
359            1,
360            &[
361                KeyValue::new("controller", self.controller.clone()),
362                KeyValue::new("outcome", outcome),
363            ],
364        );
365    }
366}
367
368#[derive(Clone)]
369pub struct ReconcileMetrics {
370    pub operations: Counter<u64>,
371    pub failures: Counter<u64>,
372    pub duration: Histogram<f64>,
373    pub deploy_delete_create: Counter<u64>,
374}
375
376impl ReconcileMetrics {
377    pub fn new(meter: &Meter) -> Self {
378        debug!("Initializing reconcile metrics");
379        let operations = meter
380            .u64_counter("reconcile_operations")
381            .with_description("Total number of reconcile operations")
382            .build();
383
384        let failures = meter
385            .u64_counter("reconcile_failures")
386            .with_description("Number of errors that occurred during reconcile operations")
387            .build();
388
389        let duration = meter
390            .f64_histogram("reconcile_duration_seconds")
391            .with_description("Histogram of reconcile operations")
392            .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
393            .build();
394
395        let deploy_delete_create = meter
396            .u64_counter("reconcile_deploy_delete_create")
397            .with_description("Number of explicit resource delete/recreate operations (not limited to deployments)")
398            .build();
399
400        debug!("Reconcile metrics initialized");
401        Self {
402            operations,
403            failures,
404            duration,
405            deploy_delete_create,
406        }
407    }
408}
409
410/// Smart function duration measurer
411///
412/// Relies on Drop to calculate duration and register the observation in the histogram
413pub struct ReconcileMeasurer {
414    start: Instant,
415    controller: String,
416    metric: Histogram<f64>,
417}
418
419impl Drop for ReconcileMeasurer {
420    fn drop(&mut self) {
421        let duration = self.start.elapsed().as_secs_f64();
422        self.metric.record(
423            duration,
424            &[KeyValue::new("controller", self.controller.clone())],
425        );
426    }
427}
428
429pub struct ActiveReconcileGuard {
430    count: Arc<AtomicI64>,
431    gauge: Gauge<i64>,
432    controller: String,
433}
434
435impl Drop for ActiveReconcileGuard {
436    fn drop(&mut self) {
437        let prev = self.count.fetch_sub(1, Ordering::Relaxed);
438        self.gauge.record(
439            (prev - 1).max(0),
440            &[KeyValue::new("controller", self.controller.clone())],
441        );
442    }
443}
444
445#[derive(Clone, Debug)]
446pub enum Action {
447    Apply,
448    Delete,
449}
450
451impl Action {
452    pub fn as_str(&self) -> &'static str {
453        match self {
454            Action::Apply => "apply",
455            Action::Delete => "delete",
456        }
457    }
458}
459
460#[cfg(test)]
461mod tests {
462    use super::*;
463    use opentelemetry::metrics::MeterProvider;
464
465    fn test_metrics() -> ControllerMetrics {
466        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
467        let meter = provider.meter("test");
468        ControllerMetrics::new("test-controller", &meter)
469    }
470
471    #[test]
472    fn active_reconcile_guard_increments_on_creation() {
473        let m = test_metrics();
474        assert_eq!(m.active_reconciles_count(), 0);
475        let _guard = m.active_reconcile_inc();
476        assert_eq!(m.active_reconciles_count(), 1);
477    }
478
479    #[test]
480    fn active_reconcile_guard_decrements_on_drop() {
481        let m = test_metrics();
482        let guard = m.active_reconcile_inc();
483        assert_eq!(m.active_reconciles_count(), 1);
484        drop(guard);
485        assert_eq!(m.active_reconciles_count(), 0);
486    }
487
488    #[test]
489    fn active_reconcile_multiple_guards() {
490        let m = test_metrics();
491        let g1 = m.active_reconcile_inc();
492        let g2 = m.active_reconcile_inc();
493        let g3 = m.active_reconcile_inc();
494        assert_eq!(m.active_reconciles_count(), 3);
495        drop(g2);
496        assert_eq!(m.active_reconciles_count(), 2);
497        drop(g1);
498        assert_eq!(m.active_reconciles_count(), 1);
499        drop(g3);
500        assert_eq!(m.active_reconciles_count(), 0);
501    }
502
503    #[test]
504    fn active_reconcile_guard_does_not_underflow() {
505        let m = test_metrics();
506        let guard = m.active_reconcile_inc();
507        drop(guard);
508        assert_eq!(m.active_reconciles_count(), 0);
509    }
510
511    #[test]
512    fn objects_in_backoff_inc_dec() {
513        let m = test_metrics();
514        assert_eq!(m.objects_in_backoff_count(), 0);
515        m.objects_in_backoff_inc();
516        assert_eq!(m.objects_in_backoff_count(), 1);
517        m.objects_in_backoff_inc();
518        assert_eq!(m.objects_in_backoff_count(), 2);
519        m.objects_in_backoff_dec();
520        assert_eq!(m.objects_in_backoff_count(), 1);
521        m.objects_in_backoff_dec();
522        assert_eq!(m.objects_in_backoff_count(), 0);
523    }
524
525    #[test]
526    fn cloned_metrics_share_active_reconcile_counter() {
527        let m = test_metrics();
528        let m2 = m.clone();
529        let _g = m.active_reconcile_inc();
530        assert_eq!(m2.active_reconciles_count(), 1);
531    }
532
533    #[test]
534    fn cloned_metrics_share_backoff_counter() {
535        let m = test_metrics();
536        let m2 = m.clone();
537        m.objects_in_backoff_inc();
538        assert_eq!(m2.objects_in_backoff_count(), 1);
539        m2.objects_in_backoff_dec();
540        assert_eq!(m.objects_in_backoff_count(), 0);
541    }
542
543    #[test]
544    fn kanidm_sdk_call_inc_records_counter() {
545        let m = test_metrics();
546        m.kanidm_sdk_call_inc(
547            KANIDM_RESOURCE_PERSON,
548            KANIDM_OP_CREATE,
549            KANIDM_OUTCOME_CHANGED,
550        );
551        m.kanidm_sdk_call_inc(
552            KANIDM_RESOURCE_PERSON,
553            KANIDM_OP_GET,
554            KANIDM_OUTCOME_UNCHANGED,
555        );
556        m.kanidm_sdk_call_inc(
557            KANIDM_RESOURCE_GROUP,
558            KANIDM_OP_DELETE,
559            KANIDM_OUTCOME_ERROR,
560        );
561    }
562
563    #[test]
564    fn record_kanidm_sdk_outcome_records_count_and_duration() {
565        let m = test_metrics();
566        m.record_kanidm_sdk_outcome(
567            KANIDM_RESOURCE_PERSON,
568            KANIDM_OP_CREATE,
569            KANIDM_OUTCOME_CHANGED,
570            std::time::Duration::from_millis(50),
571        );
572        m.record_kanidm_sdk_outcome(
573            KANIDM_RESOURCE_PERSON,
574            KANIDM_OP_GET,
575            KANIDM_OUTCOME_UNCHANGED,
576            std::time::Duration::from_millis(10),
577        );
578    }
579
580    #[tokio::test]
581    async fn record_kanidm_sdk_call_helper_records_changed_on_success() {
582        use opentelemetry::metrics::MeterProvider;
583        use opentelemetry_sdk::metrics::{
584            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
585        };
586
587        let exporter = InMemoryMetricExporter::default();
588        let reader = PeriodicReader::builder(exporter.clone())
589            .with_interval(std::time::Duration::from_millis(50))
590            .build();
591        let provider = SdkMeterProvider::builder().with_reader(reader).build();
592        let meter = provider.meter("test");
593        let m = ControllerMetrics::new("test", &meter);
594
595        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
596            &m,
597            KANIDM_RESOURCE_PERSON,
598            KANIDM_OP_CREATE,
599            KANIDM_OUTCOME_CHANGED,
600            async { Ok(()) },
601        )
602        .await;
603        assert!(result.is_ok());
604
605        provider.force_flush().unwrap();
606        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
607
608        let metrics = exporter.get_finished_metrics().unwrap();
609        let text = format!("{:?}", metrics);
610        assert!(text.contains("kanidm_sdk_calls"));
611        assert!(text.contains("kanidm_sdk_call_duration_seconds"));
612        assert!(text.contains("changed"));
613    }
614
615    #[tokio::test]
616    async fn record_kanidm_sdk_call_helper_records_error_on_failure() {
617        use opentelemetry::metrics::MeterProvider;
618        use opentelemetry_sdk::metrics::{
619            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
620        };
621
622        let exporter = InMemoryMetricExporter::default();
623        let reader = PeriodicReader::builder(exporter.clone())
624            .with_interval(std::time::Duration::from_millis(50))
625            .build();
626        let provider = SdkMeterProvider::builder().with_reader(reader).build();
627        let meter = provider.meter("test");
628        let m = ControllerMetrics::new("test", &meter);
629
630        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
631            &m,
632            KANIDM_RESOURCE_GROUP,
633            KANIDM_OP_DELETE,
634            KANIDM_OUTCOME_CHANGED,
635            async { Err("something failed") },
636        )
637        .await;
638        assert!(result.is_err());
639
640        provider.force_flush().unwrap();
641        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
642
643        let metrics = exporter.get_finished_metrics().unwrap();
644        let text = format!("{:?}", metrics);
645        assert!(text.contains("error"));
646    }
647
648    #[tokio::test]
649    async fn record_kanidm_sdk_call_with_explicit_outcome() {
650        use opentelemetry::metrics::MeterProvider;
651        use opentelemetry_sdk::metrics::{
652            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
653        };
654
655        let exporter = InMemoryMetricExporter::default();
656        let reader = PeriodicReader::builder(exporter.clone())
657            .with_interval(std::time::Duration::from_millis(50))
658            .build();
659        let provider = SdkMeterProvider::builder().with_reader(reader).build();
660        let meter = provider.meter("test");
661        let m = ControllerMetrics::new("test", &meter);
662
663        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
664            &m,
665            KANIDM_RESOURCE_OAUTH2,
666            KANIDM_OP_GET_SECRET,
667            KANIDM_OUTCOME_UNCHANGED,
668            async { Ok(()) },
669        )
670        .await;
671        assert!(result.is_ok());
672
673        provider.force_flush().unwrap();
674        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
675
676        let metrics = exporter.get_finished_metrics().unwrap();
677        let text = format!("{:?}", metrics);
678        assert!(text.contains(r#""unchanged""#));
679        assert!(!text.contains(r#""changed""#));
680    }
681
682    #[tokio::test]
683    async fn concurrent_active_reconcile_guards_track_correctly() {
684        let m = Arc::new(test_metrics());
685        let mut handles = Vec::new();
686        for _ in 0..100 {
687            let m = m.clone();
688            handles.push(tokio::spawn(async move {
689                let _guard = m.active_reconcile_inc();
690                tokio::task::yield_now().await;
691            }));
692        }
693        for h in handles {
694            h.await.unwrap();
695        }
696        assert_eq!(m.active_reconciles_count(), 0);
697    }
698
699    #[tokio::test]
700    async fn concurrent_backoff_inc_dec_stays_balanced() {
701        let m = Arc::new(test_metrics());
702        let mut handles = Vec::new();
703        for _ in 0..50 {
704            let m = m.clone();
705            handles.push(tokio::spawn(async move {
706                m.objects_in_backoff_inc();
707                tokio::task::yield_now().await;
708                m.objects_in_backoff_dec();
709            }));
710        }
711        for h in handles {
712            h.await.unwrap();
713        }
714        assert_eq!(m.objects_in_backoff_count(), 0);
715    }
716
717    #[tokio::test]
718    async fn entry_api_insert_only_increments_once() {
719        use std::collections::HashMap;
720        use std::collections::hash_map::Entry;
721        use tokio::sync::RwLock;
722
723        let m = Arc::new(test_metrics());
724        let cache: Arc<RwLock<HashMap<String, RwLock<u32>>>> =
725            Arc::new(RwLock::new(HashMap::new()));
726        let mut handles = Vec::new();
727
728        for _ in 0..100 {
729            let m = m.clone();
730            let cache = cache.clone();
731            handles.push(tokio::spawn(async move {
732                let mut guard = cache.write().await;
733                match guard.entry("same-key".to_string()) {
734                    Entry::Vacant(v) => {
735                        v.insert(RwLock::new(0));
736                        m.objects_in_backoff_inc();
737                    }
738                    Entry::Occupied(_) => {}
739                }
740            }));
741        }
742        for h in handles {
743            h.await.unwrap();
744        }
745        assert_eq!(m.objects_in_backoff_count(), 1);
746        assert_eq!(cache.read().await.len(), 1);
747    }
748
749    #[tokio::test]
750    async fn remove_only_decrements_on_actual_removal() {
751        use std::collections::HashMap;
752        use tokio::sync::RwLock;
753
754        let m = Arc::new(test_metrics());
755        let cache: Arc<RwLock<HashMap<String, u32>>> = Arc::new(RwLock::new(HashMap::new()));
756        cache.write().await.insert("key".to_string(), 42);
757        m.objects_in_backoff_inc();
758
759        let mut handles = Vec::new();
760        for _ in 0..100 {
761            let m = m.clone();
762            let cache = cache.clone();
763            handles.push(tokio::spawn(async move {
764                let removed = cache.write().await.remove("key");
765                if removed.is_some() {
766                    m.objects_in_backoff_dec();
767                }
768            }));
769        }
770        for h in handles {
771            h.await.unwrap();
772        }
773        assert_eq!(m.objects_in_backoff_count(), 0);
774        assert!(cache.read().await.is_empty());
775    }
776
777    #[tokio::test]
778    async fn reconcile_outcome_record_records_changed() {
779        use opentelemetry::metrics::MeterProvider;
780        use opentelemetry_sdk::metrics::{
781            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
782        };
783
784        let exporter = InMemoryMetricExporter::default();
785        let reader = PeriodicReader::builder(exporter.clone())
786            .with_interval(std::time::Duration::from_millis(50))
787            .build();
788        let provider = SdkMeterProvider::builder().with_reader(reader).build();
789        let meter = provider.meter("test");
790        let m = ControllerMetrics::new("test-ctrl", &meter);
791
792        m.reconcile_outcome_record(KANIDM_OUTCOME_CHANGED);
793
794        provider.force_flush().unwrap();
795        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
796
797        let metrics = exporter.get_finished_metrics().unwrap();
798        let text = format!("{:?}", metrics);
799        assert!(text.contains("reconcile_outcome"));
800        assert!(text.contains("changed"));
801        assert!(text.contains("test-ctrl"));
802    }
803
804    #[tokio::test]
805    async fn reconcile_outcome_record_records_unchanged() {
806        use opentelemetry::metrics::MeterProvider;
807        use opentelemetry_sdk::metrics::{
808            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
809        };
810
811        let exporter = InMemoryMetricExporter::default();
812        let reader = PeriodicReader::builder(exporter.clone())
813            .with_interval(std::time::Duration::from_millis(50))
814            .build();
815        let provider = SdkMeterProvider::builder().with_reader(reader).build();
816        let meter = provider.meter("test");
817        let m = ControllerMetrics::new("test-ctrl", &meter);
818
819        m.reconcile_outcome_record(KANIDM_OUTCOME_UNCHANGED);
820
821        provider.force_flush().unwrap();
822        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
823
824        let metrics = exporter.get_finished_metrics().unwrap();
825        let text = format!("{:?}", metrics);
826        assert!(text.contains("reconcile_outcome"));
827        assert!(text.contains("unchanged"));
828    }
829
830    #[tokio::test]
831    async fn reconcile_outcome_record_records_error() {
832        use opentelemetry::metrics::MeterProvider;
833        use opentelemetry_sdk::metrics::{
834            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
835        };
836
837        let exporter = InMemoryMetricExporter::default();
838        let reader = PeriodicReader::builder(exporter.clone())
839            .with_interval(std::time::Duration::from_millis(50))
840            .build();
841        let provider = SdkMeterProvider::builder().with_reader(reader).build();
842        let meter = provider.meter("test");
843        let m = ControllerMetrics::new("test-ctrl", &meter);
844
845        m.reconcile_outcome_record(KANIDM_OUTCOME_ERROR);
846
847        provider.force_flush().unwrap();
848        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
849
850        let metrics = exporter.get_finished_metrics().unwrap();
851        let text = format!("{:?}", metrics);
852        assert!(text.contains("reconcile_outcome"));
853        assert!(text.contains("error"));
854    }
855}