Skip to main content

kaniop_operator/
metrics.rs

1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicI64, Ordering};
6
7use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
8use opentelemetry::{KeyValue, trace::TraceId};
9use tokio::time::Instant;
10use tracing::debug;
11
12pub const KANIDM_RESOURCE_PERSON: &str = "Person";
13pub const KANIDM_RESOURCE_GROUP: &str = "Group";
14pub const KANIDM_RESOURCE_OAUTH2: &str = "OAuth2Client";
15pub const KANIDM_RESOURCE_SERVICE_ACCOUNT: &str = "ServiceAccount";
16pub const KANIDM_RESOURCE_DOMAIN: &str = "Domain";
17pub const KANIDM_RESOURCE_MAIL_SENDER: &str = "MailSender";
18pub const KANIDM_RESOURCE_RADIUS_SERVER: &str = "RadiusServer";
19
20pub const KANIDM_OP_GET: &str = "get";
21pub const KANIDM_OP_GET_CREDENTIAL_STATUS: &str = "get_credential_status";
22pub const KANIDM_OP_CREATE: &str = "create";
23pub const KANIDM_OP_UPDATE: &str = "update";
24pub const KANIDM_OP_DELETE: &str = "delete";
25pub const KANIDM_OP_UNIX_EXTEND: &str = "unix_extend";
26pub const KANIDM_OP_CREDENTIAL_UPDATE_INTENT: &str = "credential_update_intent";
27pub const KANIDM_OP_CREDENTIAL_UPDATE_SEND_INTENT: &str = "credential_update_send_intent";
28pub const KANIDM_OP_SET_MEMBERS: &str = "set_members";
29pub const KANIDM_OP_SET_MAIL: &str = "set_mail";
30pub const KANIDM_OP_PURGE_MAIL: &str = "purge_mail";
31pub const KANIDM_OP_ADD_MAIL: &str = "add_mail";
32pub const KANIDM_OP_REMOVE_MAIL: &str = "remove_mail";
33pub const KANIDM_OP_ACCOUNT_POLICY: &str = "account_policy";
34pub const KANIDM_OP_PURGE_ATTR: &str = "purge_attr";
35pub const KANIDM_OP_ROTATE_SECRET: &str = "rotate_secret";
36pub const KANIDM_OP_GET_SECRET: &str = "get_secret";
37pub const KANIDM_OP_ADD_ORIGIN: &str = "add_origin";
38pub const KANIDM_OP_REMOVE_ORIGIN: &str = "remove_origin";
39pub const KANIDM_OP_SCOPE_MAP: &str = "scope_map";
40pub const KANIDM_OP_DELETE_SCOPE_MAP: &str = "delete_scope_map";
41pub const KANIDM_OP_UPDATE_SCOPE_MAP: &str = "update_scope_map";
42pub const KANIDM_OP_SUP_SCOPE_MAP: &str = "sup_scope_map";
43pub const KANIDM_OP_DELETE_SUP_SCOPE_MAP: &str = "delete_sup_scope_map";
44pub const KANIDM_OP_UPDATE_SUP_SCOPE_MAP: &str = "update_sup_scope_map";
45pub const KANIDM_OP_CLAIM_MAP: &str = "claim_map";
46pub const KANIDM_OP_DELETE_CLAIM_MAP: &str = "delete_claim_map";
47pub const KANIDM_OP_UPDATE_CLAIM_MAP: &str = "update_claim_map";
48pub const KANIDM_OP_CLAIM_MAP_JOIN: &str = "claim_map_join";
49pub const KANIDM_OP_UPDATE_CLAIM_MAP_JOIN: &str = "update_claim_map_join";
50pub const KANIDM_OP_IMAGE: &str = "image";
51pub const KANIDM_OP_DELETE_IMAGE: &str = "delete_image";
52pub const KANIDM_OP_UPDATE_IMAGE: &str = "update_image";
53pub const KANIDM_OP_RESET_SECRET: &str = "reset_secret";
54pub const KANIDM_OP_GENERATE_API_TOKEN: &str = "generate_api_token";
55pub const KANIDM_OP_DESTROY_API_TOKEN: &str = "destroy_api_token";
56pub const KANIDM_OP_LIST_API_TOKENS: &str = "list_api_tokens";
57pub const KANIDM_OP_GENERATE_PASSWORD: &str = "generate_password";
58pub const KANIDM_OP_SET_DISPLAY_NAME: &str = "set_display_name";
59pub const KANIDM_OP_ADD_MEMBERS: &str = "add_members";
60pub const KANIDM_OP_REMOVE_MEMBERS: &str = "remove_members";
61pub const KANIDM_OP_WHOAMI: &str = "whoami";
62
63pub const KANIDM_OUTCOME_CHANGED: &str = "changed";
64pub const KANIDM_OUTCOME_UNCHANGED: &str = "unchanged";
65pub const KANIDM_OUTCOME_ERROR: &str = "error";
66
67pub async fn record_kanidm_sdk_call<F, T, E>(
68    metrics: &ControllerMetrics,
69    resource: &'static str,
70    operation: &'static str,
71    success_outcome: &'static str,
72    future: F,
73) -> std::result::Result<T, E>
74where
75    F: std::future::Future<Output = std::result::Result<T, E>>,
76{
77    let start = Instant::now();
78    let result = future.await;
79    let outcome = match &result {
80        Ok(_) => success_outcome,
81        Err(_) => KANIDM_OUTCOME_ERROR,
82    };
83    metrics.kanidm_sdk_calls.add(
84        1,
85        &[
86            KeyValue::new("resource", resource),
87            KeyValue::new("operation", operation),
88            KeyValue::new("outcome", outcome),
89        ],
90    );
91    metrics.kanidm_sdk_call_duration.record(
92        start.elapsed().as_secs_f64(),
93        &[
94            KeyValue::new("resource", resource),
95            KeyValue::new("operation", operation),
96        ],
97    );
98    result
99}
100
101#[derive(Clone)]
102pub struct Metrics {
103    pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
104}
105
106impl Metrics {
107    pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
108        debug!(
109            "Initializing operator metrics for controllers: {:?}",
110            controller_names
111        );
112        let controllers = controller_names
113            .iter()
114            .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
115            .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
116
117        debug!("Operator metrics initialized");
118        Self { controllers }
119    }
120}
121
122#[derive(Clone)]
123pub struct ControllerMetrics {
124    controller: String,
125    pub reconcile: ReconcileMetrics,
126    spec_replicas: Gauge<i64>,
127    status_update_errors: Counter<u64>,
128    triggered: Counter<u64>,
129    watch_operations_failed: Counter<u64>,
130    ready: Gauge<i64>,
131    active_reconciles: Gauge<i64>,
132    active_reconciles_count: Arc<AtomicI64>,
133    objects_in_backoff: Gauge<i64>,
134    objects_in_backoff_count: Arc<AtomicI64>,
135    kanidm_sdk_calls: Counter<u64>,
136    kanidm_sdk_call_duration: Histogram<f64>,
137    reconcile_outcome: Counter<u64>,
138}
139
140impl ControllerMetrics {
141    pub fn new(controller: &str, meter: &Meter) -> Self {
142        let reconcile = ReconcileMetrics::new(meter);
143
144        let spec_replicas = meter
145            .i64_gauge("spec_replicas")
146            .with_description("Number of expected replicas for the object")
147            .build();
148
149        let status_update_errors = meter
150            .u64_counter("status_update_errors")
151            .with_description(
152                "Number of errors that occurred during update operations to status subresources",
153            )
154            .build();
155
156        let triggered = meter
157            .u64_counter("triggered")
158            .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
159            .build();
160
161        let watch_operations_failed = meter
162            .u64_counter("watch_operations_failed")
163            .with_description("Total number of watch operations that failed")
164            .build();
165
166        let ready = meter
167            .i64_gauge("ready")
168            .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
169            .build();
170
171        let active_reconciles = meter
172            .i64_gauge("active_reconciles")
173            .with_description("Number of reconcile operations currently in flight")
174            .build();
175
176        let objects_in_backoff = meter
177            .i64_gauge("objects_in_backoff")
178            .with_description("Number of objects currently in error backoff state")
179            .build();
180
181        let kanidm_sdk_calls = meter
182            .u64_counter("kanidm_sdk_calls")
183            .with_description("Total number of Kanidm SDK calls")
184            .build();
185
186        let kanidm_sdk_call_duration = meter
187            .f64_histogram("kanidm_sdk_call_duration_seconds")
188            .with_description("Duration of Kanidm SDK calls in seconds")
189            .with_boundaries(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0])
190            .build();
191
192        let reconcile_outcome = meter
193            .u64_counter("reconcile_outcome")
194            .with_description("Outcome of reconcile operations: changed, unchanged, or error")
195            .build();
196
197        Self {
198            controller: controller.to_string(),
199            reconcile,
200            spec_replicas,
201            status_update_errors,
202            triggered,
203            watch_operations_failed,
204            ready,
205            active_reconciles,
206            active_reconciles_count: Arc::new(AtomicI64::new(0)),
207            objects_in_backoff,
208            objects_in_backoff_count: Arc::new(AtomicI64::new(0)),
209            kanidm_sdk_calls,
210            kanidm_sdk_call_duration,
211            reconcile_outcome,
212        }
213    }
214
215    pub fn reconcile_failure_inc(&self) {
216        self.reconcile
217            .failures
218            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
219    }
220
221    pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
222        self.reconcile
223            .operations
224            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
225        ReconcileMeasurer {
226            start: Instant::now(),
227            controller: self.controller.clone(),
228            metric: self.reconcile.duration.clone(),
229        }
230    }
231
232    pub fn reconcile_deploy_delete_create_inc(&self, resource_kind: &str, reason: &str) {
233        self.reconcile.deploy_delete_create.add(
234            1,
235            &[
236                KeyValue::new("controller", self.controller.clone()),
237                KeyValue::new("resource_kind", resource_kind.to_string()),
238                KeyValue::new("reason", reason.to_string()),
239            ],
240        );
241    }
242
243    pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
244        self.spec_replicas.record(
245            replicas as i64,
246            &[
247                KeyValue::new("controller", self.controller.clone()),
248                KeyValue::new("namespace", namespace.to_string()),
249                KeyValue::new("name", name.to_string()),
250            ],
251        );
252    }
253
254    pub fn status_update_errors_inc(&self) {
255        self.status_update_errors
256            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
257    }
258
259    pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
260        self.triggered.add(
261            1,
262            &[
263                KeyValue::new("controller", self.controller.clone()),
264                KeyValue::new("action", action.as_str()),
265                KeyValue::new("triggered_by", triggered_by.to_string()),
266            ],
267        );
268    }
269
270    pub fn watch_operations_failed_inc(&self) {
271        self.watch_operations_failed
272            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
273    }
274
275    pub fn ready_set(&self, status: i64) {
276        self.ready.record(
277            status,
278            &[KeyValue::new("controller", self.controller.clone())],
279        );
280    }
281
282    pub fn active_reconcile_inc(&self) -> ActiveReconcileGuard {
283        let new = self.active_reconciles_count.fetch_add(1, Ordering::Relaxed) + 1;
284        self.active_reconciles
285            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
286        ActiveReconcileGuard {
287            count: self.active_reconciles_count.clone(),
288            gauge: self.active_reconciles.clone(),
289            controller: self.controller.clone(),
290        }
291    }
292
293    pub fn objects_in_backoff_inc(&self) {
294        let new = self
295            .objects_in_backoff_count
296            .fetch_add(1, Ordering::Relaxed)
297            + 1;
298        self.objects_in_backoff
299            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
300    }
301
302    pub fn objects_in_backoff_dec(&self) {
303        let prev = self
304            .objects_in_backoff_count
305            .fetch_sub(1, Ordering::Relaxed);
306        let new = (prev - 1).max(0);
307        self.objects_in_backoff
308            .record(new, &[KeyValue::new("controller", self.controller.clone())]);
309    }
310
311    pub fn active_reconciles_count(&self) -> i64 {
312        self.active_reconciles_count.load(Ordering::Relaxed)
313    }
314
315    pub fn objects_in_backoff_count(&self) -> i64 {
316        self.objects_in_backoff_count.load(Ordering::Relaxed)
317    }
318
319    pub fn kanidm_sdk_call_inc(
320        &self,
321        resource: &'static str,
322        operation: &'static str,
323        outcome: &'static str,
324    ) {
325        self.kanidm_sdk_calls.add(
326            1,
327            &[
328                KeyValue::new("resource", resource),
329                KeyValue::new("operation", operation),
330                KeyValue::new("outcome", outcome),
331            ],
332        );
333    }
334
335    pub fn record_kanidm_sdk_outcome(
336        &self,
337        resource: &'static str,
338        operation: &'static str,
339        outcome: &'static str,
340        duration: std::time::Duration,
341    ) {
342        self.kanidm_sdk_calls.add(
343            1,
344            &[
345                KeyValue::new("resource", resource),
346                KeyValue::new("operation", operation),
347                KeyValue::new("outcome", outcome),
348            ],
349        );
350        self.kanidm_sdk_call_duration.record(
351            duration.as_secs_f64(),
352            &[
353                KeyValue::new("resource", resource),
354                KeyValue::new("operation", operation),
355            ],
356        );
357    }
358
359    pub fn reconcile_outcome_record(&self, outcome: &'static str) {
360        self.reconcile_outcome.add(
361            1,
362            &[
363                KeyValue::new("controller", self.controller.clone()),
364                KeyValue::new("outcome", outcome),
365            ],
366        );
367    }
368}
369
370#[derive(Clone)]
371pub struct ReconcileMetrics {
372    pub operations: Counter<u64>,
373    pub failures: Counter<u64>,
374    pub duration: Histogram<f64>,
375    pub deploy_delete_create: Counter<u64>,
376}
377
378impl ReconcileMetrics {
379    pub fn new(meter: &Meter) -> Self {
380        debug!("Initializing reconcile metrics");
381        let operations = meter
382            .u64_counter("reconcile_operations")
383            .with_description("Total number of reconcile operations")
384            .build();
385
386        let failures = meter
387            .u64_counter("reconcile_failures")
388            .with_description("Number of errors that occurred during reconcile operations")
389            .build();
390
391        let duration = meter
392            .f64_histogram("reconcile_duration_seconds")
393            .with_description("Histogram of reconcile operations")
394            .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
395            .build();
396
397        let deploy_delete_create = meter
398            .u64_counter("reconcile_deploy_delete_create")
399            .with_description("Number of explicit resource delete/recreate operations (not limited to deployments)")
400            .build();
401
402        debug!("Reconcile metrics initialized");
403        Self {
404            operations,
405            failures,
406            duration,
407            deploy_delete_create,
408        }
409    }
410}
411
412/// Smart function duration measurer
413///
414/// Relies on Drop to calculate duration and register the observation in the histogram
415pub struct ReconcileMeasurer {
416    start: Instant,
417    controller: String,
418    metric: Histogram<f64>,
419}
420
421impl Drop for ReconcileMeasurer {
422    fn drop(&mut self) {
423        let duration = self.start.elapsed().as_secs_f64();
424        self.metric.record(
425            duration,
426            &[KeyValue::new("controller", self.controller.clone())],
427        );
428    }
429}
430
431pub struct ActiveReconcileGuard {
432    count: Arc<AtomicI64>,
433    gauge: Gauge<i64>,
434    controller: String,
435}
436
437impl Drop for ActiveReconcileGuard {
438    fn drop(&mut self) {
439        let prev = self.count.fetch_sub(1, Ordering::Relaxed);
440        self.gauge.record(
441            (prev - 1).max(0),
442            &[KeyValue::new("controller", self.controller.clone())],
443        );
444    }
445}
446
447#[derive(Clone, Debug)]
448pub enum Action {
449    Apply,
450    Delete,
451}
452
453impl Action {
454    pub fn as_str(&self) -> &'static str {
455        match self {
456            Action::Apply => "apply",
457            Action::Delete => "delete",
458        }
459    }
460}
461
462#[cfg(test)]
463mod tests {
464    use super::*;
465    use opentelemetry::metrics::MeterProvider;
466
467    fn test_metrics() -> ControllerMetrics {
468        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
469        let meter = provider.meter("test");
470        ControllerMetrics::new("test-controller", &meter)
471    }
472
473    #[test]
474    fn active_reconcile_guard_increments_on_creation() {
475        let m = test_metrics();
476        assert_eq!(m.active_reconciles_count(), 0);
477        let _guard = m.active_reconcile_inc();
478        assert_eq!(m.active_reconciles_count(), 1);
479    }
480
481    #[test]
482    fn active_reconcile_guard_decrements_on_drop() {
483        let m = test_metrics();
484        let guard = m.active_reconcile_inc();
485        assert_eq!(m.active_reconciles_count(), 1);
486        drop(guard);
487        assert_eq!(m.active_reconciles_count(), 0);
488    }
489
490    #[test]
491    fn active_reconcile_multiple_guards() {
492        let m = test_metrics();
493        let g1 = m.active_reconcile_inc();
494        let g2 = m.active_reconcile_inc();
495        let g3 = m.active_reconcile_inc();
496        assert_eq!(m.active_reconciles_count(), 3);
497        drop(g2);
498        assert_eq!(m.active_reconciles_count(), 2);
499        drop(g1);
500        assert_eq!(m.active_reconciles_count(), 1);
501        drop(g3);
502        assert_eq!(m.active_reconciles_count(), 0);
503    }
504
505    #[test]
506    fn active_reconcile_guard_does_not_underflow() {
507        let m = test_metrics();
508        let guard = m.active_reconcile_inc();
509        drop(guard);
510        assert_eq!(m.active_reconciles_count(), 0);
511    }
512
513    #[test]
514    fn objects_in_backoff_inc_dec() {
515        let m = test_metrics();
516        assert_eq!(m.objects_in_backoff_count(), 0);
517        m.objects_in_backoff_inc();
518        assert_eq!(m.objects_in_backoff_count(), 1);
519        m.objects_in_backoff_inc();
520        assert_eq!(m.objects_in_backoff_count(), 2);
521        m.objects_in_backoff_dec();
522        assert_eq!(m.objects_in_backoff_count(), 1);
523        m.objects_in_backoff_dec();
524        assert_eq!(m.objects_in_backoff_count(), 0);
525    }
526
527    #[test]
528    fn cloned_metrics_share_active_reconcile_counter() {
529        let m = test_metrics();
530        let m2 = m.clone();
531        let _g = m.active_reconcile_inc();
532        assert_eq!(m2.active_reconciles_count(), 1);
533    }
534
535    #[test]
536    fn cloned_metrics_share_backoff_counter() {
537        let m = test_metrics();
538        let m2 = m.clone();
539        m.objects_in_backoff_inc();
540        assert_eq!(m2.objects_in_backoff_count(), 1);
541        m2.objects_in_backoff_dec();
542        assert_eq!(m.objects_in_backoff_count(), 0);
543    }
544
545    #[test]
546    fn kanidm_sdk_call_inc_records_counter() {
547        let m = test_metrics();
548        m.kanidm_sdk_call_inc(
549            KANIDM_RESOURCE_PERSON,
550            KANIDM_OP_CREATE,
551            KANIDM_OUTCOME_CHANGED,
552        );
553        m.kanidm_sdk_call_inc(
554            KANIDM_RESOURCE_PERSON,
555            KANIDM_OP_GET,
556            KANIDM_OUTCOME_UNCHANGED,
557        );
558        m.kanidm_sdk_call_inc(
559            KANIDM_RESOURCE_GROUP,
560            KANIDM_OP_DELETE,
561            KANIDM_OUTCOME_ERROR,
562        );
563    }
564
565    #[test]
566    fn record_kanidm_sdk_outcome_records_count_and_duration() {
567        let m = test_metrics();
568        m.record_kanidm_sdk_outcome(
569            KANIDM_RESOURCE_PERSON,
570            KANIDM_OP_CREATE,
571            KANIDM_OUTCOME_CHANGED,
572            std::time::Duration::from_millis(50),
573        );
574        m.record_kanidm_sdk_outcome(
575            KANIDM_RESOURCE_PERSON,
576            KANIDM_OP_GET,
577            KANIDM_OUTCOME_UNCHANGED,
578            std::time::Duration::from_millis(10),
579        );
580    }
581
582    #[tokio::test]
583    async fn record_kanidm_sdk_call_helper_records_changed_on_success() {
584        use opentelemetry::metrics::MeterProvider;
585        use opentelemetry_sdk::metrics::{
586            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
587        };
588
589        let exporter = InMemoryMetricExporter::default();
590        let reader = PeriodicReader::builder(exporter.clone())
591            .with_interval(std::time::Duration::from_millis(50))
592            .build();
593        let provider = SdkMeterProvider::builder().with_reader(reader).build();
594        let meter = provider.meter("test");
595        let m = ControllerMetrics::new("test", &meter);
596
597        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
598            &m,
599            KANIDM_RESOURCE_PERSON,
600            KANIDM_OP_CREATE,
601            KANIDM_OUTCOME_CHANGED,
602            async { Ok(()) },
603        )
604        .await;
605        assert!(result.is_ok());
606
607        provider.force_flush().unwrap();
608        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
609
610        let metrics = exporter.get_finished_metrics().unwrap();
611        let text = format!("{:?}", metrics);
612        assert!(text.contains("kanidm_sdk_calls"));
613        assert!(text.contains("kanidm_sdk_call_duration_seconds"));
614        assert!(text.contains("changed"));
615    }
616
617    #[tokio::test]
618    async fn record_kanidm_sdk_call_helper_records_error_on_failure() {
619        use opentelemetry::metrics::MeterProvider;
620        use opentelemetry_sdk::metrics::{
621            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
622        };
623
624        let exporter = InMemoryMetricExporter::default();
625        let reader = PeriodicReader::builder(exporter.clone())
626            .with_interval(std::time::Duration::from_millis(50))
627            .build();
628        let provider = SdkMeterProvider::builder().with_reader(reader).build();
629        let meter = provider.meter("test");
630        let m = ControllerMetrics::new("test", &meter);
631
632        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
633            &m,
634            KANIDM_RESOURCE_GROUP,
635            KANIDM_OP_DELETE,
636            KANIDM_OUTCOME_CHANGED,
637            async { Err("something failed") },
638        )
639        .await;
640        assert!(result.is_err());
641
642        provider.force_flush().unwrap();
643        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
644
645        let metrics = exporter.get_finished_metrics().unwrap();
646        let text = format!("{:?}", metrics);
647        assert!(text.contains("error"));
648    }
649
650    #[tokio::test]
651    async fn record_kanidm_sdk_call_with_explicit_outcome() {
652        use opentelemetry::metrics::MeterProvider;
653        use opentelemetry_sdk::metrics::{
654            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
655        };
656
657        let exporter = InMemoryMetricExporter::default();
658        let reader = PeriodicReader::builder(exporter.clone())
659            .with_interval(std::time::Duration::from_millis(50))
660            .build();
661        let provider = SdkMeterProvider::builder().with_reader(reader).build();
662        let meter = provider.meter("test");
663        let m = ControllerMetrics::new("test", &meter);
664
665        let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
666            &m,
667            KANIDM_RESOURCE_OAUTH2,
668            KANIDM_OP_GET_SECRET,
669            KANIDM_OUTCOME_UNCHANGED,
670            async { Ok(()) },
671        )
672        .await;
673        assert!(result.is_ok());
674
675        provider.force_flush().unwrap();
676        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
677
678        let metrics = exporter.get_finished_metrics().unwrap();
679        let text = format!("{:?}", metrics);
680        assert!(text.contains(r#""unchanged""#));
681        assert!(!text.contains(r#""changed""#));
682    }
683
684    #[tokio::test]
685    async fn concurrent_active_reconcile_guards_track_correctly() {
686        let m = Arc::new(test_metrics());
687        let mut handles = Vec::new();
688        for _ in 0..100 {
689            let m = m.clone();
690            handles.push(tokio::spawn(async move {
691                let _guard = m.active_reconcile_inc();
692                tokio::task::yield_now().await;
693            }));
694        }
695        for h in handles {
696            h.await.unwrap();
697        }
698        assert_eq!(m.active_reconciles_count(), 0);
699    }
700
701    #[tokio::test]
702    async fn concurrent_backoff_inc_dec_stays_balanced() {
703        let m = Arc::new(test_metrics());
704        let mut handles = Vec::new();
705        for _ in 0..50 {
706            let m = m.clone();
707            handles.push(tokio::spawn(async move {
708                m.objects_in_backoff_inc();
709                tokio::task::yield_now().await;
710                m.objects_in_backoff_dec();
711            }));
712        }
713        for h in handles {
714            h.await.unwrap();
715        }
716        assert_eq!(m.objects_in_backoff_count(), 0);
717    }
718
719    #[tokio::test]
720    async fn entry_api_insert_only_increments_once() {
721        use std::collections::HashMap;
722        use std::collections::hash_map::Entry;
723        use tokio::sync::RwLock;
724
725        let m = Arc::new(test_metrics());
726        let cache: Arc<RwLock<HashMap<String, RwLock<u32>>>> =
727            Arc::new(RwLock::new(HashMap::new()));
728        let mut handles = Vec::new();
729
730        for _ in 0..100 {
731            let m = m.clone();
732            let cache = cache.clone();
733            handles.push(tokio::spawn(async move {
734                let mut guard = cache.write().await;
735                match guard.entry("same-key".to_string()) {
736                    Entry::Vacant(v) => {
737                        v.insert(RwLock::new(0));
738                        m.objects_in_backoff_inc();
739                    }
740                    Entry::Occupied(_) => {}
741                }
742            }));
743        }
744        for h in handles {
745            h.await.unwrap();
746        }
747        assert_eq!(m.objects_in_backoff_count(), 1);
748        assert_eq!(cache.read().await.len(), 1);
749    }
750
751    #[tokio::test]
752    async fn remove_only_decrements_on_actual_removal() {
753        use std::collections::HashMap;
754        use tokio::sync::RwLock;
755
756        let m = Arc::new(test_metrics());
757        let cache: Arc<RwLock<HashMap<String, u32>>> = Arc::new(RwLock::new(HashMap::new()));
758        cache.write().await.insert("key".to_string(), 42);
759        m.objects_in_backoff_inc();
760
761        let mut handles = Vec::new();
762        for _ in 0..100 {
763            let m = m.clone();
764            let cache = cache.clone();
765            handles.push(tokio::spawn(async move {
766                let removed = cache.write().await.remove("key");
767                if removed.is_some() {
768                    m.objects_in_backoff_dec();
769                }
770            }));
771        }
772        for h in handles {
773            h.await.unwrap();
774        }
775        assert_eq!(m.objects_in_backoff_count(), 0);
776        assert!(cache.read().await.is_empty());
777    }
778
779    #[tokio::test]
780    async fn reconcile_outcome_record_records_changed() {
781        use opentelemetry::metrics::MeterProvider;
782        use opentelemetry_sdk::metrics::{
783            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
784        };
785
786        let exporter = InMemoryMetricExporter::default();
787        let reader = PeriodicReader::builder(exporter.clone())
788            .with_interval(std::time::Duration::from_millis(50))
789            .build();
790        let provider = SdkMeterProvider::builder().with_reader(reader).build();
791        let meter = provider.meter("test");
792        let m = ControllerMetrics::new("test-ctrl", &meter);
793
794        m.reconcile_outcome_record(KANIDM_OUTCOME_CHANGED);
795
796        provider.force_flush().unwrap();
797        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
798
799        let metrics = exporter.get_finished_metrics().unwrap();
800        let text = format!("{:?}", metrics);
801        assert!(text.contains("reconcile_outcome"));
802        assert!(text.contains("changed"));
803        assert!(text.contains("test-ctrl"));
804    }
805
806    #[tokio::test]
807    async fn reconcile_outcome_record_records_unchanged() {
808        use opentelemetry::metrics::MeterProvider;
809        use opentelemetry_sdk::metrics::{
810            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
811        };
812
813        let exporter = InMemoryMetricExporter::default();
814        let reader = PeriodicReader::builder(exporter.clone())
815            .with_interval(std::time::Duration::from_millis(50))
816            .build();
817        let provider = SdkMeterProvider::builder().with_reader(reader).build();
818        let meter = provider.meter("test");
819        let m = ControllerMetrics::new("test-ctrl", &meter);
820
821        m.reconcile_outcome_record(KANIDM_OUTCOME_UNCHANGED);
822
823        provider.force_flush().unwrap();
824        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
825
826        let metrics = exporter.get_finished_metrics().unwrap();
827        let text = format!("{:?}", metrics);
828        assert!(text.contains("reconcile_outcome"));
829        assert!(text.contains("unchanged"));
830    }
831
832    #[tokio::test]
833    async fn reconcile_outcome_record_records_error() {
834        use opentelemetry::metrics::MeterProvider;
835        use opentelemetry_sdk::metrics::{
836            InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
837        };
838
839        let exporter = InMemoryMetricExporter::default();
840        let reader = PeriodicReader::builder(exporter.clone())
841            .with_interval(std::time::Duration::from_millis(50))
842            .build();
843        let provider = SdkMeterProvider::builder().with_reader(reader).build();
844        let meter = provider.meter("test");
845        let m = ControllerMetrics::new("test-ctrl", &meter);
846
847        m.reconcile_outcome_record(KANIDM_OUTCOME_ERROR);
848
849        provider.force_flush().unwrap();
850        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
851
852        let metrics = exporter.get_finished_metrics().unwrap();
853        let text = format!("{:?}", metrics);
854        assert!(text.contains("reconcile_outcome"));
855        assert!(text.contains("error"));
856    }
857}