Skip to main content

kaniop_operator/
metrics.rs

1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicI64, Ordering};
6
7use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
8use opentelemetry::{KeyValue, trace::TraceId};
9use tokio::time::Instant;
10use tracing::debug;
11
12pub const KANIDM_RESOURCE_PERSON: &str = "Person";
13pub const KANIDM_RESOURCE_GROUP: &str = "Group";
14pub const KANIDM_RESOURCE_OAUTH2: &str = "OAuth2Client";
15pub const KANIDM_RESOURCE_SERVICE_ACCOUNT: &str = "ServiceAccount";
16pub const KANIDM_RESOURCE_DOMAIN: &str = "Domain";
17pub const KANIDM_RESOURCE_MAIL_SENDER: &str = "MailSender";
18
19pub const KANIDM_OP_GET: &str = "get";
20pub const KANIDM_OP_GET_CREDENTIAL_STATUS: &str = "get_credential_status";
21pub const KANIDM_OP_CREATE: &str = "create";
22pub const KANIDM_OP_UPDATE: &str = "update";
23pub const KANIDM_OP_DELETE: &str = "delete";
24pub const KANIDM_OP_UNIX_EXTEND: &str = "unix_extend";
25pub const KANIDM_OP_CREDENTIAL_UPDATE_INTENT: &str = "credential_update_intent";
26pub const KANIDM_OP_SET_MEMBERS: &str = "set_members";
27pub const KANIDM_OP_SET_MAIL: &str = "set_mail";
28pub const KANIDM_OP_PURGE_MAIL: &str = "purge_mail";
29pub const KANIDM_OP_ADD_MAIL: &str = "add_mail";
30pub const KANIDM_OP_REMOVE_MAIL: &str = "remove_mail";
31pub const KANIDM_OP_ACCOUNT_POLICY: &str = "account_policy";
32pub const KANIDM_OP_PURGE_ATTR: &str = "purge_attr";
33pub const KANIDM_OP_ROTATE_SECRET: &str = "rotate_secret";
34pub const KANIDM_OP_GET_SECRET: &str = "get_secret";
35pub const KANIDM_OP_ADD_ORIGIN: &str = "add_origin";
36pub const KANIDM_OP_REMOVE_ORIGIN: &str = "remove_origin";
37pub const KANIDM_OP_SCOPE_MAP: &str = "scope_map";
38pub const KANIDM_OP_DELETE_SCOPE_MAP: &str = "delete_scope_map";
39pub const KANIDM_OP_UPDATE_SCOPE_MAP: &str = "update_scope_map";
40pub const KANIDM_OP_SUP_SCOPE_MAP: &str = "sup_scope_map";
41pub const KANIDM_OP_DELETE_SUP_SCOPE_MAP: &str = "delete_sup_scope_map";
42pub const KANIDM_OP_UPDATE_SUP_SCOPE_MAP: &str = "update_sup_scope_map";
43pub const KANIDM_OP_CLAIM_MAP: &str = "claim_map";
44pub const KANIDM_OP_DELETE_CLAIM_MAP: &str = "delete_claim_map";
45pub const KANIDM_OP_UPDATE_CLAIM_MAP: &str = "update_claim_map";
46pub const KANIDM_OP_CLAIM_MAP_JOIN: &str = "claim_map_join";
47pub const KANIDM_OP_UPDATE_CLAIM_MAP_JOIN: &str = "update_claim_map_join";
48pub const KANIDM_OP_IMAGE: &str = "image";
49pub const KANIDM_OP_DELETE_IMAGE: &str = "delete_image";
50pub const KANIDM_OP_UPDATE_IMAGE: &str = "update_image";
51pub const KANIDM_OP_RESET_SECRET: &str = "reset_secret";
52pub const KANIDM_OP_GENERATE_API_TOKEN: &str = "generate_api_token";
53pub const KANIDM_OP_DESTROY_API_TOKEN: &str = "destroy_api_token";
54pub const KANIDM_OP_LIST_API_TOKENS: &str = "list_api_tokens";
55pub const KANIDM_OP_GENERATE_PASSWORD: &str = "generate_password";
56pub const KANIDM_OP_SET_DISPLAY_NAME: &str = "set_display_name";
57pub const KANIDM_OP_ADD_MEMBERS: &str = "add_members";
58pub const KANIDM_OP_REMOVE_MEMBERS: &str = "remove_members";
59
60pub const KANIDM_OUTCOME_CHANGED: &str = "changed";
61pub const KANIDM_OUTCOME_UNCHANGED: &str = "unchanged";
62pub const KANIDM_OUTCOME_ERROR: &str = "error";
63
64pub async fn record_kanidm_sdk_call<F, T, E>(
65    metrics: &ControllerMetrics,
66    resource: &'static str,
67    operation: &'static str,
68    success_outcome: &'static str,
69    future: F,
70) -> std::result::Result<T, E>
71where
72    F: std::future::Future<Output = std::result::Result<T, E>>,
73{
74    let start = Instant::now();
75    let result = future.await;
76    let outcome = match &result {
77        Ok(_) => success_outcome,
78        Err(_) => KANIDM_OUTCOME_ERROR,
79    };
80    metrics.kanidm_sdk_calls.add(
81        1,
82        &[
83            KeyValue::new("resource", resource),
84            KeyValue::new("operation", operation),
85            KeyValue::new("outcome", outcome),
86        ],
87    );
88    metrics.kanidm_sdk_call_duration.record(
89        start.elapsed().as_secs_f64(),
90        &[
91            KeyValue::new("resource", resource),
92            KeyValue::new("operation", operation),
93        ],
94    );
95    result
96}
97
98#[derive(Clone)]
99pub struct Metrics {
100    pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
101}
102
103impl Metrics {
104    pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
105        debug!(
106            "Initializing operator metrics for controllers: {:?}",
107            controller_names
108        );
109        let controllers = controller_names
110            .iter()
111            .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
112            .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
113
114        debug!("Operator metrics initialized");
115        Self { controllers }
116    }
117}
118
119#[derive(Clone)]
120pub struct ControllerMetrics {
121    controller: String,
122    pub reconcile: ReconcileMetrics,
123    spec_replicas: Gauge<i64>,
124    status_update_errors: Counter<u64>,
125    triggered: Counter<u64>,
126    watch_operations_failed: Counter<u64>,
127    ready: Gauge<i64>,
128    active_reconciles: Gauge<i64>,
129    active_reconciles_count: Arc<AtomicI64>,
130    objects_in_backoff: Gauge<i64>,
131    objects_in_backoff_count: Arc<AtomicI64>,
132    kanidm_sdk_calls: Counter<u64>,
133    kanidm_sdk_call_duration: Histogram<f64>,
134    reconcile_outcome: Counter<u64>,
135}
136
137impl ControllerMetrics {
138    pub fn new(controller: &str, meter: &Meter) -> Self {
139        let reconcile = ReconcileMetrics::new(meter);
140
141        let spec_replicas = meter
142            .i64_gauge("spec_replicas")
143            .with_description("Number of expected replicas for the object")
144            .build();
145
146        let status_update_errors = meter
147            .u64_counter("status_update_errors")
148            .with_description(
149                "Number of errors that occurred during update operations to status subresources",
150            )
151            .build();
152
153        let triggered = meter
154            .u64_counter("triggered")
155            .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
156            .build();
157
158        let watch_operations_failed = meter
159            .u64_counter("watch_operations_failed")
160            .with_description("Total number of watch operations that failed")
161            .build();
162
163        let ready = meter
164            .i64_gauge("ready")
165            .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
166            .build();
167
168        let active_reconciles = meter
169            .i64_gauge("active_reconciles")
170            .with_description("Number of reconcile operations currently in flight")
171            .build();
172
173        let objects_in_backoff = meter
174            .i64_gauge("objects_in_backoff")
175            .with_description("Number of objects currently in error backoff state")
176            .build();
177
178        let kanidm_sdk_calls = meter
179            .u64_counter("kanidm_sdk_calls")
180            .with_description("Total number of Kanidm SDK calls")
181            .build();
182
183        let kanidm_sdk_call_duration = meter
184            .f64_histogram("kanidm_sdk_call_duration_seconds")
185            .with_description("Duration of Kanidm SDK calls in seconds")
186            .with_boundaries(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0])
187            .build();
188
189        let reconcile_outcome = meter
190            .u64_counter("reconcile_outcome")
191            .with_description("Outcome of reconcile operations: changed, unchanged, or error")
192            .build();
193
194        Self {
195            controller: controller.to_string(),
196            reconcile,
197            spec_replicas,
198            status_update_errors,
199            triggered,
200            watch_operations_failed,
201            ready,
202            active_reconciles,
203            active_reconciles_count: Arc::new(AtomicI64::new(0)),
204            objects_in_backoff,
205            objects_in_backoff_count: Arc::new(AtomicI64::new(0)),
206            kanidm_sdk_calls,
207            kanidm_sdk_call_duration,
208            reconcile_outcome,
209        }
210    }
211
212    pub fn reconcile_failure_inc(&self) {
213        self.reconcile
214            .failures
215            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
216    }
217
218    pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
219        self.reconcile
220            .operations
221            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
222        ReconcileMeasurer {
223            start: Instant::now(),
224            controller: self.controller.clone(),
225            metric: self.reconcile.duration.clone(),
226        }
227    }
228
229    pub fn reconcile_deploy_delete_create_inc(&self, resource_kind: &str, reason: &str) {
230        self.reconcile.deploy_delete_create.add(
231            1,
232            &[
233                KeyValue::new("controller", self.controller.clone()),
234                KeyValue::new("resource_kind", resource_kind.to_string()),
235                KeyValue::new("reason", reason.to_string()),
236            ],
237        );
238    }
239
240    pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
241        self.spec_replicas.record(
242            replicas as i64,
243            &[
244                KeyValue::new("controller", self.controller.clone()),
245                KeyValue::new("namespace", namespace.to_string()),
246                KeyValue::new("name", name.to_string()),
247            ],
248        );
249    }
250
251    pub fn status_update_errors_inc(&self) {
252        self.status_update_errors
253            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
254    }
255
256    pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
257        self.triggered.add(
258            1,
259            &[
260                KeyValue::new("controller", self.controller.clone()),
261                KeyValue::new("action", action.as_str()),
262                KeyValue::new("triggered_by", triggered_by.to_string()),
263            ],
264        );
265    }
266
267    pub fn watch_operations_failed_inc(&self) {
268        self.watch_operations_failed
269            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
270    }
271
272    pub fn ready_set(&self, status: i64) {
273        self.ready.record(
274            status,
275            &[KeyValue::new("controller", self.controller.clone())],
276        );
277    }
278
279    pub fn active_reconcile_inc(&self) -> ActiveReconcileGuard {
280        let new = self.active_reconciles_count.fetch_add(1, Ordering::Relaxed) + 1;
281        self.active_reconciles
282            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
283        ActiveReconcileGuard {
284            count: self.active_reconciles_count.clone(),
285            gauge: self.active_reconciles.clone(),
286            controller: self.controller.clone(),
287        }
288    }
289
290    pub fn objects_in_backoff_inc(&self) {
291        let new = self
292            .objects_in_backoff_count
293            .fetch_add(1, Ordering::Relaxed)
294            + 1;
295        self.objects_in_backoff
296            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
297    }
298
299    pub fn objects_in_backoff_dec(&self) {
300        let prev = self
301            .objects_in_backoff_count
302            .fetch_sub(1, Ordering::Relaxed);
303        let new = (prev - 1).max(0);
304        self.objects_in_backoff
305            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
306    }
307
308    pub fn active_reconciles_count(&self) -> i64 {
309        self.active_reconciles_count.load(Ordering::Relaxed)
310    }
311
312    pub fn objects_in_backoff_count(&self) -> i64 {
313        self.objects_in_backoff_count.load(Ordering::Relaxed)
314    }
315
316    pub fn kanidm_sdk_call_inc(
317        &self,
318        resource: &'static str,
319        operation: &'static str,
320        outcome: &'static str,
321    ) {
322        self.kanidm_sdk_calls.add(
323            1,
324            &[
325                KeyValue::new("resource", resource),
326                KeyValue::new("operation", operation),
327                KeyValue::new("outcome", outcome),
328            ],
329        );
330    }
331
332    pub fn record_kanidm_sdk_outcome(
333        &self,
334        resource: &'static str,
335        operation: &'static str,
336        outcome: &'static str,
337        duration: std::time::Duration,
338    ) {
339        self.kanidm_sdk_calls.add(
340            1,
341            &[
342                KeyValue::new("resource", resource),
343                KeyValue::new("operation", operation),
344                KeyValue::new("outcome", outcome),
345            ],
346        );
347        self.kanidm_sdk_call_duration.record(
348            duration.as_secs_f64(),
349            &[
350                KeyValue::new("resource", resource),
351                KeyValue::new("operation", operation),
352            ],
353        );
354    }
355
356    pub fn reconcile_outcome_record(&self, outcome: &'static str) {
357        self.reconcile_outcome.add(
358            1,
359            &[
360                KeyValue::new("controller", self.controller.clone()),
361                KeyValue::new("outcome", outcome),
362            ],
363        );
364    }
365}
366
367#[derive(Clone)]
368pub struct ReconcileMetrics {
369    pub operations: Counter<u64>,
370    pub failures: Counter<u64>,
371    pub duration: Histogram<f64>,
372    pub deploy_delete_create: Counter<u64>,
373}
374
375impl ReconcileMetrics {
376    pub fn new(meter: &Meter) -> Self {
377        debug!("Initializing reconcile metrics");
378        let operations = meter
379            .u64_counter("reconcile_operations")
380            .with_description("Total number of reconcile operations")
381            .build();
382
383        let failures = meter
384            .u64_counter("reconcile_failures")
385            .with_description("Number of errors that occurred during reconcile operations")
386            .build();
387
388        let duration = meter
389            .f64_histogram("reconcile_duration_seconds")
390            .with_description("Histogram of reconcile operations")
391            .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
392            .build();
393
394        let deploy_delete_create = meter
395            .u64_counter("reconcile_deploy_delete_create")
396            .with_description("Number of explicit resource delete/recreate operations (not limited to deployments)")
397            .build();
398
399        debug!("Reconcile metrics initialized");
400        Self {
401            operations,
402            failures,
403            duration,
404            deploy_delete_create,
405        }
406    }
407}
408
409/// Smart function duration measurer
410///
411/// Relies on Drop to calculate duration and register the observation in the histogram
412pub struct ReconcileMeasurer {
413    start: Instant,
414    controller: String,
415    metric: Histogram<f64>,
416}
417
418impl Drop for ReconcileMeasurer {
419    fn drop(&mut self) {
420        let duration = self.start.elapsed().as_secs_f64();
421        self.metric.record(
422            duration,
423            &[KeyValue::new("controller", self.controller.clone())],
424        );
425    }
426}
427
428pub struct ActiveReconcileGuard {
429    count: Arc<AtomicI64>,
430    gauge: Gauge<i64>,
431    controller: String,
432}
433
434impl Drop for ActiveReconcileGuard {
435    fn drop(&mut self) {
436        let prev = self.count.fetch_sub(1, Ordering::Relaxed);
437        self.gauge.record(
438            (prev - 1).max(0),
439            &[KeyValue::new("controller", self.controller.clone())],
440        );
441    }
442}
443
444#[derive(Clone, Debug)]
445pub enum Action {
446    Apply,
447    Delete,
448}
449
450impl Action {
451    pub fn as_str(&self) -> &'static str {
452        match self {
453            Action::Apply => "apply",
454            Action::Delete => "delete",
455        }
456    }
457}
458
459#[cfg(test)]
460mod tests {
461    use super::*;
462    use opentelemetry::metrics::MeterProvider;
463
464    fn test_metrics() -> ControllerMetrics {
465        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
466        let meter = provider.meter("test");
467        ControllerMetrics::new("test-controller", &meter)
468    }
469
470    #[test]
471    fn active_reconcile_guard_increments_on_creation() {
472        let m = test_metrics();
473        assert_eq!(m.active_reconciles_count(), 0);
474        let _guard = m.active_reconcile_inc();
475        assert_eq!(m.active_reconciles_count(), 1);
476    }
477
478    #[test]
479    fn active_reconcile_guard_decrements_on_drop() {
480        let m = test_metrics();
481        let guard = m.active_reconcile_inc();
482        assert_eq!(m.active_reconciles_count(), 1);
483        drop(guard);
484        assert_eq!(m.active_reconciles_count(), 0);
485    }
486
487    #[test]
488    fn active_reconcile_multiple_guards() {
489        let m = test_metrics();
490        let g1 = m.active_reconcile_inc();
491        let g2 = m.active_reconcile_inc();
492        let g3 = m.active_reconcile_inc();
493        assert_eq!(m.active_reconciles_count(), 3);
494        drop(g2);
495        assert_eq!(m.active_reconciles_count(), 2);
496        drop(g1);
497        assert_eq!(m.active_reconciles_count(), 1);
498        drop(g3);
499        assert_eq!(m.active_reconciles_count(), 0);
500    }
501
502    #[test]
503    fn active_reconcile_guard_does_not_underflow() {
504        let m = test_metrics();
505        let guard = m.active_reconcile_inc();
506        drop(guard);
507        assert_eq!(m.active_reconciles_count(), 0);
508    }
509
510    #[test]
511    fn objects_in_backoff_inc_dec() {
512        let m = test_metrics();
513        assert_eq!(m.objects_in_backoff_count(), 0);
514        m.objects_in_backoff_inc();
515        assert_eq!(m.objects_in_backoff_count(), 1);
516        m.objects_in_backoff_inc();
517        assert_eq!(m.objects_in_backoff_count(), 2);
518        m.objects_in_backoff_dec();
519        assert_eq!(m.objects_in_backoff_count(), 1);
520        m.objects_in_backoff_dec();
521        assert_eq!(m.objects_in_backoff_count(), 0);
522    }
523
524    #[test]
525    fn cloned_metrics_share_active_reconcile_counter() {
526        let m = test_metrics();
527        let m2 = m.clone();
528        let _g = m.active_reconcile_inc();
529        assert_eq!(m2.active_reconciles_count(), 1);
530    }
531
532    #[test]
533    fn cloned_metrics_share_backoff_counter() {
534        let m = test_metrics();
535        let m2 = m.clone();
536        m.objects_in_backoff_inc();
537        assert_eq!(m2.objects_in_backoff_count(), 1);
538        m2.objects_in_backoff_dec();
539        assert_eq!(m.objects_in_backoff_count(), 0);
540    }
541
542    #[test]
543    fn kanidm_sdk_call_inc_records_counter() {
544        let m = test_metrics();
545        m.kanidm_sdk_call_inc(
546            KANIDM_RESOURCE_PERSON,
547            KANIDM_OP_CREATE,
548            KANIDM_OUTCOME_CHANGED,
549        );
550        m.kanidm_sdk_call_inc(
551            KANIDM_RESOURCE_PERSON,
552            KANIDM_OP_GET,
553            KANIDM_OUTCOME_UNCHANGED,
554        );
555        m.kanidm_sdk_call_inc(
556            KANIDM_RESOURCE_GROUP,
557            KANIDM_OP_DELETE,
558            KANIDM_OUTCOME_ERROR,
559        );
560    }
561
562    #[test]
563    fn record_kanidm_sdk_outcome_records_count_and_duration() {
564        let m = test_metrics();
565        m.record_kanidm_sdk_outcome(
566            KANIDM_RESOURCE_PERSON,
567            KANIDM_OP_CREATE,
568            KANIDM_OUTCOME_CHANGED,
569            std::time::Duration::from_millis(50),
570        );
571        m.record_kanidm_sdk_outcome(
572            KANIDM_RESOURCE_PERSON,
573            KANIDM_OP_GET,
574            KANIDM_OUTCOME_UNCHANGED,
575            std::time::Duration::from_millis(10),
576        );
577    }
578
579    #[tokio::test]
580    async fn record_kanidm_sdk_call_helper_records_changed_on_success() {
581        use opentelemetry::metrics::MeterProvider;
582        use opentelemetry_sdk::metrics::{
583            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
584        };
585
586        let exporter = InMemoryMetricExporter::default();
587        let reader = PeriodicReader::builder(exporter.clone())
588            .with_interval(std::time::Duration::from_millis(50))
589            .build();
590        let provider = SdkMeterProvider::builder().with_reader(reader).build();
591        let meter = provider.meter("test");
592        let m = ControllerMetrics::new("test", &meter);
593
594        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
595            &m,
596            KANIDM_RESOURCE_PERSON,
597            KANIDM_OP_CREATE,
598            KANIDM_OUTCOME_CHANGED,
599            async { Ok(()) },
600        )
601        .await;
602        assert!(result.is_ok());
603
604        provider.force_flush().unwrap();
605        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
606
607        let metrics = exporter.get_finished_metrics().unwrap();
608        let text = format!("{:?}", metrics);
609        assert!(text.contains("kanidm_sdk_calls"));
610        assert!(text.contains("kanidm_sdk_call_duration_seconds"));
611        assert!(text.contains("changed"));
612    }
613
614    #[tokio::test]
615    async fn record_kanidm_sdk_call_helper_records_error_on_failure() {
616        use opentelemetry::metrics::MeterProvider;
617        use opentelemetry_sdk::metrics::{
618            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
619        };
620
621        let exporter = InMemoryMetricExporter::default();
622        let reader = PeriodicReader::builder(exporter.clone())
623            .with_interval(std::time::Duration::from_millis(50))
624            .build();
625        let provider = SdkMeterProvider::builder().with_reader(reader).build();
626        let meter = provider.meter("test");
627        let m = ControllerMetrics::new("test", &meter);
628
629        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
630            &m,
631            KANIDM_RESOURCE_GROUP,
632            KANIDM_OP_DELETE,
633            KANIDM_OUTCOME_CHANGED,
634            async { Err("something failed") },
635        )
636        .await;
637        assert!(result.is_err());
638
639        provider.force_flush().unwrap();
640        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
641
642        let metrics = exporter.get_finished_metrics().unwrap();
643        let text = format!("{:?}", metrics);
644        assert!(text.contains("error"));
645    }
646
647    #[tokio::test]
648    async fn record_kanidm_sdk_call_with_explicit_outcome() {
649        use opentelemetry::metrics::MeterProvider;
650        use opentelemetry_sdk::metrics::{
651            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
652        };
653
654        let exporter = InMemoryMetricExporter::default();
655        let reader = PeriodicReader::builder(exporter.clone())
656            .with_interval(std::time::Duration::from_millis(50))
657            .build();
658        let provider = SdkMeterProvider::builder().with_reader(reader).build();
659        let meter = provider.meter("test");
660        let m = ControllerMetrics::new("test", &meter);
661
662        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
663            &m,
664            KANIDM_RESOURCE_OAUTH2,
665            KANIDM_OP_GET_SECRET,
666            KANIDM_OUTCOME_UNCHANGED,
667            async { Ok(()) },
668        )
669        .await;
670        assert!(result.is_ok());
671
672        provider.force_flush().unwrap();
673        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
674
675        let metrics = exporter.get_finished_metrics().unwrap();
676        let text = format!("{:?}", metrics);
677        assert!(text.contains(r#""unchanged""#));
678        assert!(!text.contains(r#""changed""#));
679    }
680
681    #[tokio::test]
682    async fn concurrent_active_reconcile_guards_track_correctly() {
683        let m = Arc::new(test_metrics());
684        let mut handles = Vec::new();
685        for _ in 0..100 {
686            let m = m.clone();
687            handles.push(tokio::spawn(async move {
688                let _guard = m.active_reconcile_inc();
689                tokio::task::yield_now().await;
690            }));
691        }
692        for h in handles {
693            h.await.unwrap();
694        }
695        assert_eq!(m.active_reconciles_count(), 0);
696    }
697
698    #[tokio::test]
699    async fn concurrent_backoff_inc_dec_stays_balanced() {
700        let m = Arc::new(test_metrics());
701        let mut handles = Vec::new();
702        for _ in 0..50 {
703            let m = m.clone();
704            handles.push(tokio::spawn(async move {
705                m.objects_in_backoff_inc();
706                tokio::task::yield_now().await;
707                m.objects_in_backoff_dec();
708            }));
709        }
710        for h in handles {
711            h.await.unwrap();
712        }
713        assert_eq!(m.objects_in_backoff_count(), 0);
714    }
715
716    #[tokio::test]
717    async fn entry_api_insert_only_increments_once() {
718        use std::collections::HashMap;
719        use std::collections::hash_map::Entry;
720        use tokio::sync::RwLock;
721
722        let m = Arc::new(test_metrics());
723        let cache: Arc<RwLock<HashMap<String, RwLock<u32>>>> =
724            Arc::new(RwLock::new(HashMap::new()));
725        let mut handles = Vec::new();
726
727        for _ in 0..100 {
728            let m = m.clone();
729            let cache = cache.clone();
730            handles.push(tokio::spawn(async move {
731                let mut guard = cache.write().await;
732                match guard.entry("same-key".to_string()) {
733                    Entry::Vacant(v) => {
734                        v.insert(RwLock::new(0));
735                        m.objects_in_backoff_inc();
736                    }
737                    Entry::Occupied(_) => {}
738                }
739            }));
740        }
741        for h in handles {
742            h.await.unwrap();
743        }
744        assert_eq!(m.objects_in_backoff_count(), 1);
745        assert_eq!(cache.read().await.len(), 1);
746    }
747
748    #[tokio::test]
749    async fn remove_only_decrements_on_actual_removal() {
750        use std::collections::HashMap;
751        use tokio::sync::RwLock;
752
753        let m = Arc::new(test_metrics());
754        let cache: Arc<RwLock<HashMap<String, u32>>> = Arc::new(RwLock::new(HashMap::new()));
755        cache.write().await.insert("key".to_string(), 42);
756        m.objects_in_backoff_inc();
757
758        let mut handles = Vec::new();
759        for _ in 0..100 {
760            let m = m.clone();
761            let cache = cache.clone();
762            handles.push(tokio::spawn(async move {
763                let removed = cache.write().await.remove("key");
764                if removed.is_some() {
765                    m.objects_in_backoff_dec();
766                }
767            }));
768        }
769        for h in handles {
770            h.await.unwrap();
771        }
772        assert_eq!(m.objects_in_backoff_count(), 0);
773        assert!(cache.read().await.is_empty());
774    }
775
776    #[tokio::test]
777    async fn reconcile_outcome_record_records_changed() {
778        use opentelemetry::metrics::MeterProvider;
779        use opentelemetry_sdk::metrics::{
780            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
781        };
782
783        let exporter = InMemoryMetricExporter::default();
784        let reader = PeriodicReader::builder(exporter.clone())
785            .with_interval(std::time::Duration::from_millis(50))
786            .build();
787        let provider = SdkMeterProvider::builder().with_reader(reader).build();
788        let meter = provider.meter("test");
789        let m = ControllerMetrics::new("test-ctrl", &meter);
790
791        m.reconcile_outcome_record(KANIDM_OUTCOME_CHANGED);
792
793        provider.force_flush().unwrap();
794        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
795
796        let metrics = exporter.get_finished_metrics().unwrap();
797        let text = format!("{:?}", metrics);
798        assert!(text.contains("reconcile_outcome"));
799        assert!(text.contains("changed"));
800        assert!(text.contains("test-ctrl"));
801    }
802
803    #[tokio::test]
804    async fn reconcile_outcome_record_records_unchanged() {
805        use opentelemetry::metrics::MeterProvider;
806        use opentelemetry_sdk::metrics::{
807            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
808        };
809
810        let exporter = InMemoryMetricExporter::default();
811        let reader = PeriodicReader::builder(exporter.clone())
812            .with_interval(std::time::Duration::from_millis(50))
813            .build();
814        let provider = SdkMeterProvider::builder().with_reader(reader).build();
815        let meter = provider.meter("test");
816        let m = ControllerMetrics::new("test-ctrl", &meter);
817
818        m.reconcile_outcome_record(KANIDM_OUTCOME_UNCHANGED);
819
820        provider.force_flush().unwrap();
821        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
822
823        let metrics = exporter.get_finished_metrics().unwrap();
824        let text = format!("{:?}", metrics);
825        assert!(text.contains("reconcile_outcome"));
826        assert!(text.contains("unchanged"));
827    }
828
829    #[tokio::test]
830    async fn reconcile_outcome_record_records_error() {
831        use opentelemetry::metrics::MeterProvider;
832        use opentelemetry_sdk::metrics::{
833            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
834        };
835
836        let exporter = InMemoryMetricExporter::default();
837        let reader = PeriodicReader::builder(exporter.clone())
838            .with_interval(std::time::Duration::from_millis(50))
839            .build();
840        let provider = SdkMeterProvider::builder().with_reader(reader).build();
841        let meter = provider.meter("test");
842        let m = ControllerMetrics::new("test-ctrl", &meter);
843
844        m.reconcile_outcome_record(KANIDM_OUTCOME_ERROR);
845
846        provider.force_flush().unwrap();
847        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
848
849        let metrics = exporter.get_finished_metrics().unwrap();
850        let text = format!("{:?}", metrics);
851        assert!(text.contains("reconcile_outcome"));
852        assert!(text.contains("error"));
853    }
854}