1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicI64, Ordering};
6
7use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
8use opentelemetry::{KeyValue, trace::TraceId};
9use tokio::time::Instant;
10use tracing::debug;
11
12pub const KANIDM_RESOURCE_PERSON: &str = "Person";
13pub const KANIDM_RESOURCE_GROUP: &str = "Group";
14pub const KANIDM_RESOURCE_OAUTH2: &str = "OAuth2Client";
15pub const KANIDM_RESOURCE_SERVICE_ACCOUNT: &str = "ServiceAccount";
16pub const KANIDM_RESOURCE_DOMAIN: &str = "Domain";
17pub const KANIDM_RESOURCE_MAIL_SENDER: &str = "MailSender";
18
19pub const KANIDM_OP_GET: &str = "get";
20pub const KANIDM_OP_GET_CREDENTIAL_STATUS: &str = "get_credential_status";
21pub const KANIDM_OP_CREATE: &str = "create";
22pub const KANIDM_OP_UPDATE: &str = "update";
23pub const KANIDM_OP_DELETE: &str = "delete";
24pub const KANIDM_OP_UNIX_EXTEND: &str = "unix_extend";
25pub const KANIDM_OP_CREDENTIAL_UPDATE_INTENT: &str = "credential_update_intent";
26pub const KANIDM_OP_CREDENTIAL_UPDATE_SEND_INTENT: &str = "credential_update_send_intent";
27pub const KANIDM_OP_SET_MEMBERS: &str = "set_members";
28pub const KANIDM_OP_SET_MAIL: &str = "set_mail";
29pub const KANIDM_OP_PURGE_MAIL: &str = "purge_mail";
30pub const KANIDM_OP_ADD_MAIL: &str = "add_mail";
31pub const KANIDM_OP_REMOVE_MAIL: &str = "remove_mail";
32pub const KANIDM_OP_ACCOUNT_POLICY: &str = "account_policy";
33pub const KANIDM_OP_PURGE_ATTR: &str = "purge_attr";
34pub const KANIDM_OP_ROTATE_SECRET: &str = "rotate_secret";
35pub const KANIDM_OP_GET_SECRET: &str = "get_secret";
36pub const KANIDM_OP_ADD_ORIGIN: &str = "add_origin";
37pub const KANIDM_OP_REMOVE_ORIGIN: &str = "remove_origin";
38pub const KANIDM_OP_SCOPE_MAP: &str = "scope_map";
39pub const KANIDM_OP_DELETE_SCOPE_MAP: &str = "delete_scope_map";
40pub const KANIDM_OP_UPDATE_SCOPE_MAP: &str = "update_scope_map";
41pub const KANIDM_OP_SUP_SCOPE_MAP: &str = "sup_scope_map";
42pub const KANIDM_OP_DELETE_SUP_SCOPE_MAP: &str = "delete_sup_scope_map";
43pub const KANIDM_OP_UPDATE_SUP_SCOPE_MAP: &str = "update_sup_scope_map";
44pub const KANIDM_OP_CLAIM_MAP: &str = "claim_map";
45pub const KANIDM_OP_DELETE_CLAIM_MAP: &str = "delete_claim_map";
46pub const KANIDM_OP_UPDATE_CLAIM_MAP: &str = "update_claim_map";
47pub const KANIDM_OP_CLAIM_MAP_JOIN: &str = "claim_map_join";
48pub const KANIDM_OP_UPDATE_CLAIM_MAP_JOIN: &str = "update_claim_map_join";
49pub const KANIDM_OP_IMAGE: &str = "image";
50pub const KANIDM_OP_DELETE_IMAGE: &str = "delete_image";
51pub const KANIDM_OP_UPDATE_IMAGE: &str = "update_image";
52pub const KANIDM_OP_RESET_SECRET: &str = "reset_secret";
53pub const KANIDM_OP_GENERATE_API_TOKEN: &str = "generate_api_token";
54pub const KANIDM_OP_DESTROY_API_TOKEN: &str = "destroy_api_token";
55pub const KANIDM_OP_LIST_API_TOKENS: &str = "list_api_tokens";
56pub const KANIDM_OP_GENERATE_PASSWORD: &str = "generate_password";
57pub const KANIDM_OP_SET_DISPLAY_NAME: &str = "set_display_name";
58pub const KANIDM_OP_ADD_MEMBERS: &str = "add_members";
59pub const KANIDM_OP_REMOVE_MEMBERS: &str = "remove_members";
60
61pub const KANIDM_OUTCOME_CHANGED: &str = "changed";
62pub const KANIDM_OUTCOME_UNCHANGED: &str = "unchanged";
63pub const KANIDM_OUTCOME_ERROR: &str = "error";
64
65pub async fn record_kanidm_sdk_call<F, T, E>(
66 metrics: &ControllerMetrics,
67 resource: &'static str,
68 operation: &'static str,
69 success_outcome: &'static str,
70 future: F,
71) -> std::result::Result<T, E>
72where
73 F: std::future::Future<Output = std::result::Result<T, E>>,
74{
75 let start = Instant::now();
76 let result = future.await;
77 let outcome = match &result {
78 Ok(_) => success_outcome,
79 Err(_) => KANIDM_OUTCOME_ERROR,
80 };
81 metrics.kanidm_sdk_calls.add(
82 1,
83 &[
84 KeyValue::new("resource", resource),
85 KeyValue::new("operation", operation),
86 KeyValue::new("outcome", outcome),
87 ],
88 );
89 metrics.kanidm_sdk_call_duration.record(
90 start.elapsed().as_secs_f64(),
91 &[
92 KeyValue::new("resource", resource),
93 KeyValue::new("operation", operation),
94 ],
95 );
96 result
97}
98
99#[derive(Clone)]
100pub struct Metrics {
101 pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
102}
103
104impl Metrics {
105 pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
106 debug!(
107 "Initializing operator metrics for controllers: {:?}",
108 controller_names
109 );
110 let controllers = controller_names
111 .iter()
112 .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
113 .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
114
115 debug!("Operator metrics initialized");
116 Self { controllers }
117 }
118}
119
120#[derive(Clone)]
121pub struct ControllerMetrics {
122 controller: String,
123 pub reconcile: ReconcileMetrics,
124 spec_replicas: Gauge<i64>,
125 status_update_errors: Counter<u64>,
126 triggered: Counter<u64>,
127 watch_operations_failed: Counter<u64>,
128 ready: Gauge<i64>,
129 active_reconciles: Gauge<i64>,
130 active_reconciles_count: Arc<AtomicI64>,
131 objects_in_backoff: Gauge<i64>,
132 objects_in_backoff_count: Arc<AtomicI64>,
133 kanidm_sdk_calls: Counter<u64>,
134 kanidm_sdk_call_duration: Histogram<f64>,
135 reconcile_outcome: Counter<u64>,
136}
137
138impl ControllerMetrics {
139 pub fn new(controller: &str, meter: &Meter) -> Self {
140 let reconcile = ReconcileMetrics::new(meter);
141
142 let spec_replicas = meter
143 .i64_gauge("spec_replicas")
144 .with_description("Number of expected replicas for the object")
145 .build();
146
147 let status_update_errors = meter
148 .u64_counter("status_update_errors")
149 .with_description(
150 "Number of errors that occurred during update operations to status subresources",
151 )
152 .build();
153
154 let triggered = meter
155 .u64_counter("triggered")
156 .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
157 .build();
158
159 let watch_operations_failed = meter
160 .u64_counter("watch_operations_failed")
161 .with_description("Total number of watch operations that failed")
162 .build();
163
164 let ready = meter
165 .i64_gauge("ready")
166 .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
167 .build();
168
169 let active_reconciles = meter
170 .i64_gauge("active_reconciles")
171 .with_description("Number of reconcile operations currently in flight")
172 .build();
173
174 let objects_in_backoff = meter
175 .i64_gauge("objects_in_backoff")
176 .with_description("Number of objects currently in error backoff state")
177 .build();
178
179 let kanidm_sdk_calls = meter
180 .u64_counter("kanidm_sdk_calls")
181 .with_description("Total number of Kanidm SDK calls")
182 .build();
183
184 let kanidm_sdk_call_duration = meter
185 .f64_histogram("kanidm_sdk_call_duration_seconds")
186 .with_description("Duration of Kanidm SDK calls in seconds")
187 .with_boundaries(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0])
188 .build();
189
190 let reconcile_outcome = meter
191 .u64_counter("reconcile_outcome")
192 .with_description("Outcome of reconcile operations: changed, unchanged, or error")
193 .build();
194
195 Self {
196 controller: controller.to_string(),
197 reconcile,
198 spec_replicas,
199 status_update_errors,
200 triggered,
201 watch_operations_failed,
202 ready,
203 active_reconciles,
204 active_reconciles_count: Arc::new(AtomicI64::new(0)),
205 objects_in_backoff,
206 objects_in_backoff_count: Arc::new(AtomicI64::new(0)),
207 kanidm_sdk_calls,
208 kanidm_sdk_call_duration,
209 reconcile_outcome,
210 }
211 }
212
213 pub fn reconcile_failure_inc(&self) {
214 self.reconcile
215 .failures
216 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
217 }
218
219 pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
220 self.reconcile
221 .operations
222 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
223 ReconcileMeasurer {
224 start: Instant::now(),
225 controller: self.controller.clone(),
226 metric: self.reconcile.duration.clone(),
227 }
228 }
229
230 pub fn reconcile_deploy_delete_create_inc(&self, resource_kind: &str, reason: &str) {
231 self.reconcile.deploy_delete_create.add(
232 1,
233 &[
234 KeyValue::new("controller", self.controller.clone()),
235 KeyValue::new("resource_kind", resource_kind.to_string()),
236 KeyValue::new("reason", reason.to_string()),
237 ],
238 );
239 }
240
241 pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
242 self.spec_replicas.record(
243 replicas as i64,
244 &[
245 KeyValue::new("controller", self.controller.clone()),
246 KeyValue::new("namespace", namespace.to_string()),
247 KeyValue::new("name", name.to_string()),
248 ],
249 );
250 }
251
252 pub fn status_update_errors_inc(&self) {
253 self.status_update_errors
254 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
255 }
256
257 pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
258 self.triggered.add(
259 1,
260 &[
261 KeyValue::new("controller", self.controller.clone()),
262 KeyValue::new("action", action.as_str()),
263 KeyValue::new("triggered_by", triggered_by.to_string()),
264 ],
265 );
266 }
267
268 pub fn watch_operations_failed_inc(&self) {
269 self.watch_operations_failed
270 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
271 }
272
273 pub fn ready_set(&self, status: i64) {
274 self.ready.record(
275 status,
276 &[KeyValue::new("controller", self.controller.clone())],
277 );
278 }
279
280 pub fn active_reconcile_inc(&self) -> ActiveReconcileGuard {
281 let new = self.active_reconciles_count.fetch_add(1, Ordering::Relaxed) + 1;
282 self.active_reconciles
283 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
284 ActiveReconcileGuard {
285 count: self.active_reconciles_count.clone(),
286 gauge: self.active_reconciles.clone(),
287 controller: self.controller.clone(),
288 }
289 }
290
291 pub fn objects_in_backoff_inc(&self) {
292 let new = self
293 .objects_in_backoff_count
294 .fetch_add(1, Ordering::Relaxed)
295 + 1;
296 self.objects_in_backoff
297 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
298 }
299
300 pub fn objects_in_backoff_dec(&self) {
301 let prev = self
302 .objects_in_backoff_count
303 .fetch_sub(1, Ordering::Relaxed);
304 let new = (prev - 1).max(0);
305 self.objects_in_backoff
306 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
307 }
308
309 pub fn active_reconciles_count(&self) -> i64 {
310 self.active_reconciles_count.load(Ordering::Relaxed)
311 }
312
313 pub fn objects_in_backoff_count(&self) -> i64 {
314 self.objects_in_backoff_count.load(Ordering::Relaxed)
315 }
316
317 pub fn kanidm_sdk_call_inc(
318 &self,
319 resource: &'static str,
320 operation: &'static str,
321 outcome: &'static str,
322 ) {
323 self.kanidm_sdk_calls.add(
324 1,
325 &[
326 KeyValue::new("resource", resource),
327 KeyValue::new("operation", operation),
328 KeyValue::new("outcome", outcome),
329 ],
330 );
331 }
332
333 pub fn record_kanidm_sdk_outcome(
334 &self,
335 resource: &'static str,
336 operation: &'static str,
337 outcome: &'static str,
338 duration: std::time::Duration,
339 ) {
340 self.kanidm_sdk_calls.add(
341 1,
342 &[
343 KeyValue::new("resource", resource),
344 KeyValue::new("operation", operation),
345 KeyValue::new("outcome", outcome),
346 ],
347 );
348 self.kanidm_sdk_call_duration.record(
349 duration.as_secs_f64(),
350 &[
351 KeyValue::new("resource", resource),
352 KeyValue::new("operation", operation),
353 ],
354 );
355 }
356
357 pub fn reconcile_outcome_record(&self, outcome: &'static str) {
358 self.reconcile_outcome.add(
359 1,
360 &[
361 KeyValue::new("controller", self.controller.clone()),
362 KeyValue::new("outcome", outcome),
363 ],
364 );
365 }
366}
367
368#[derive(Clone)]
369pub struct ReconcileMetrics {
370 pub operations: Counter<u64>,
371 pub failures: Counter<u64>,
372 pub duration: Histogram<f64>,
373 pub deploy_delete_create: Counter<u64>,
374}
375
376impl ReconcileMetrics {
377 pub fn new(meter: &Meter) -> Self {
378 debug!("Initializing reconcile metrics");
379 let operations = meter
380 .u64_counter("reconcile_operations")
381 .with_description("Total number of reconcile operations")
382 .build();
383
384 let failures = meter
385 .u64_counter("reconcile_failures")
386 .with_description("Number of errors that occurred during reconcile operations")
387 .build();
388
389 let duration = meter
390 .f64_histogram("reconcile_duration_seconds")
391 .with_description("Histogram of reconcile operations")
392 .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
393 .build();
394
395 let deploy_delete_create = meter
396 .u64_counter("reconcile_deploy_delete_create")
397 .with_description("Number of explicit resource delete/recreate operations (not limited to deployments)")
398 .build();
399
400 debug!("Reconcile metrics initialized");
401 Self {
402 operations,
403 failures,
404 duration,
405 deploy_delete_create,
406 }
407 }
408}
409
410pub struct ReconcileMeasurer {
414 start: Instant,
415 controller: String,
416 metric: Histogram<f64>,
417}
418
419impl Drop for ReconcileMeasurer {
420 fn drop(&mut self) {
421 let duration = self.start.elapsed().as_secs_f64();
422 self.metric.record(
423 duration,
424 &[KeyValue::new("controller", self.controller.clone())],
425 );
426 }
427}
428
429pub struct ActiveReconcileGuard {
430 count: Arc<AtomicI64>,
431 gauge: Gauge<i64>,
432 controller: String,
433}
434
435impl Drop for ActiveReconcileGuard {
436 fn drop(&mut self) {
437 let prev = self.count.fetch_sub(1, Ordering::Relaxed);
438 self.gauge.record(
439 (prev - 1).max(0),
440 &[KeyValue::new("controller", self.controller.clone())],
441 );
442 }
443}
444
445#[derive(Clone, Debug)]
446pub enum Action {
447 Apply,
448 Delete,
449}
450
451impl Action {
452 pub fn as_str(&self) -> &'static str {
453 match self {
454 Action::Apply => "apply",
455 Action::Delete => "delete",
456 }
457 }
458}
459
460#[cfg(test)]
461mod tests {
462 use super::*;
463 use opentelemetry::metrics::MeterProvider;
464
465 fn test_metrics() -> ControllerMetrics {
466 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
467 let meter = provider.meter("test");
468 ControllerMetrics::new("test-controller", &meter)
469 }
470
471 #[test]
472 fn active_reconcile_guard_increments_on_creation() {
473 let m = test_metrics();
474 assert_eq!(m.active_reconciles_count(), 0);
475 let _guard = m.active_reconcile_inc();
476 assert_eq!(m.active_reconciles_count(), 1);
477 }
478
479 #[test]
480 fn active_reconcile_guard_decrements_on_drop() {
481 let m = test_metrics();
482 let guard = m.active_reconcile_inc();
483 assert_eq!(m.active_reconciles_count(), 1);
484 drop(guard);
485 assert_eq!(m.active_reconciles_count(), 0);
486 }
487
488 #[test]
489 fn active_reconcile_multiple_guards() {
490 let m = test_metrics();
491 let g1 = m.active_reconcile_inc();
492 let g2 = m.active_reconcile_inc();
493 let g3 = m.active_reconcile_inc();
494 assert_eq!(m.active_reconciles_count(), 3);
495 drop(g2);
496 assert_eq!(m.active_reconciles_count(), 2);
497 drop(g1);
498 assert_eq!(m.active_reconciles_count(), 1);
499 drop(g3);
500 assert_eq!(m.active_reconciles_count(), 0);
501 }
502
503 #[test]
504 fn active_reconcile_guard_does_not_underflow() {
505 let m = test_metrics();
506 let guard = m.active_reconcile_inc();
507 drop(guard);
508 assert_eq!(m.active_reconciles_count(), 0);
509 }
510
511 #[test]
512 fn objects_in_backoff_inc_dec() {
513 let m = test_metrics();
514 assert_eq!(m.objects_in_backoff_count(), 0);
515 m.objects_in_backoff_inc();
516 assert_eq!(m.objects_in_backoff_count(), 1);
517 m.objects_in_backoff_inc();
518 assert_eq!(m.objects_in_backoff_count(), 2);
519 m.objects_in_backoff_dec();
520 assert_eq!(m.objects_in_backoff_count(), 1);
521 m.objects_in_backoff_dec();
522 assert_eq!(m.objects_in_backoff_count(), 0);
523 }
524
525 #[test]
526 fn cloned_metrics_share_active_reconcile_counter() {
527 let m = test_metrics();
528 let m2 = m.clone();
529 let _g = m.active_reconcile_inc();
530 assert_eq!(m2.active_reconciles_count(), 1);
531 }
532
533 #[test]
534 fn cloned_metrics_share_backoff_counter() {
535 let m = test_metrics();
536 let m2 = m.clone();
537 m.objects_in_backoff_inc();
538 assert_eq!(m2.objects_in_backoff_count(), 1);
539 m2.objects_in_backoff_dec();
540 assert_eq!(m.objects_in_backoff_count(), 0);
541 }
542
543 #[test]
544 fn kanidm_sdk_call_inc_records_counter() {
545 let m = test_metrics();
546 m.kanidm_sdk_call_inc(
547 KANIDM_RESOURCE_PERSON,
548 KANIDM_OP_CREATE,
549 KANIDM_OUTCOME_CHANGED,
550 );
551 m.kanidm_sdk_call_inc(
552 KANIDM_RESOURCE_PERSON,
553 KANIDM_OP_GET,
554 KANIDM_OUTCOME_UNCHANGED,
555 );
556 m.kanidm_sdk_call_inc(
557 KANIDM_RESOURCE_GROUP,
558 KANIDM_OP_DELETE,
559 KANIDM_OUTCOME_ERROR,
560 );
561 }
562
563 #[test]
564 fn record_kanidm_sdk_outcome_records_count_and_duration() {
565 let m = test_metrics();
566 m.record_kanidm_sdk_outcome(
567 KANIDM_RESOURCE_PERSON,
568 KANIDM_OP_CREATE,
569 KANIDM_OUTCOME_CHANGED,
570 std::time::Duration::from_millis(50),
571 );
572 m.record_kanidm_sdk_outcome(
573 KANIDM_RESOURCE_PERSON,
574 KANIDM_OP_GET,
575 KANIDM_OUTCOME_UNCHANGED,
576 std::time::Duration::from_millis(10),
577 );
578 }
579
580 #[tokio::test]
581 async fn record_kanidm_sdk_call_helper_records_changed_on_success() {
582 use opentelemetry::metrics::MeterProvider;
583 use opentelemetry_sdk::metrics::{
584 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
585 };
586
587 let exporter = InMemoryMetricExporter::default();
588 let reader = PeriodicReader::builder(exporter.clone())
589 .with_interval(std::time::Duration::from_millis(50))
590 .build();
591 let provider = SdkMeterProvider::builder().with_reader(reader).build();
592 let meter = provider.meter("test");
593 let m = ControllerMetrics::new("test", &meter);
594
595 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
596 &m,
597 KANIDM_RESOURCE_PERSON,
598 KANIDM_OP_CREATE,
599 KANIDM_OUTCOME_CHANGED,
600 async { Ok(()) },
601 )
602 .await;
603 assert!(result.is_ok());
604
605 provider.force_flush().unwrap();
606 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
607
608 let metrics = exporter.get_finished_metrics().unwrap();
609 let text = format!("{:?}", metrics);
610 assert!(text.contains("kanidm_sdk_calls"));
611 assert!(text.contains("kanidm_sdk_call_duration_seconds"));
612 assert!(text.contains("changed"));
613 }
614
615 #[tokio::test]
616 async fn record_kanidm_sdk_call_helper_records_error_on_failure() {
617 use opentelemetry::metrics::MeterProvider;
618 use opentelemetry_sdk::metrics::{
619 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
620 };
621
622 let exporter = InMemoryMetricExporter::default();
623 let reader = PeriodicReader::builder(exporter.clone())
624 .with_interval(std::time::Duration::from_millis(50))
625 .build();
626 let provider = SdkMeterProvider::builder().with_reader(reader).build();
627 let meter = provider.meter("test");
628 let m = ControllerMetrics::new("test", &meter);
629
630 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
631 &m,
632 KANIDM_RESOURCE_GROUP,
633 KANIDM_OP_DELETE,
634 KANIDM_OUTCOME_CHANGED,
635 async { Err("something failed") },
636 )
637 .await;
638 assert!(result.is_err());
639
640 provider.force_flush().unwrap();
641 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
642
643 let metrics = exporter.get_finished_metrics().unwrap();
644 let text = format!("{:?}", metrics);
645 assert!(text.contains("error"));
646 }
647
648 #[tokio::test]
649 async fn record_kanidm_sdk_call_with_explicit_outcome() {
650 use opentelemetry::metrics::MeterProvider;
651 use opentelemetry_sdk::metrics::{
652 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
653 };
654
655 let exporter = InMemoryMetricExporter::default();
656 let reader = PeriodicReader::builder(exporter.clone())
657 .with_interval(std::time::Duration::from_millis(50))
658 .build();
659 let provider = SdkMeterProvider::builder().with_reader(reader).build();
660 let meter = provider.meter("test");
661 let m = ControllerMetrics::new("test", &meter);
662
663 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
664 &m,
665 KANIDM_RESOURCE_OAUTH2,
666 KANIDM_OP_GET_SECRET,
667 KANIDM_OUTCOME_UNCHANGED,
668 async { Ok(()) },
669 )
670 .await;
671 assert!(result.is_ok());
672
673 provider.force_flush().unwrap();
674 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
675
676 let metrics = exporter.get_finished_metrics().unwrap();
677 let text = format!("{:?}", metrics);
678 assert!(text.contains(r#""unchanged""#));
679 assert!(!text.contains(r#""changed""#));
680 }
681
682 #[tokio::test]
683 async fn concurrent_active_reconcile_guards_track_correctly() {
684 let m = Arc::new(test_metrics());
685 let mut handles = Vec::new();
686 for _ in 0..100 {
687 let m = m.clone();
688 handles.push(tokio::spawn(async move {
689 let _guard = m.active_reconcile_inc();
690 tokio::task::yield_now().await;
691 }));
692 }
693 for h in handles {
694 h.await.unwrap();
695 }
696 assert_eq!(m.active_reconciles_count(), 0);
697 }
698
699 #[tokio::test]
700 async fn concurrent_backoff_inc_dec_stays_balanced() {
701 let m = Arc::new(test_metrics());
702 let mut handles = Vec::new();
703 for _ in 0..50 {
704 let m = m.clone();
705 handles.push(tokio::spawn(async move {
706 m.objects_in_backoff_inc();
707 tokio::task::yield_now().await;
708 m.objects_in_backoff_dec();
709 }));
710 }
711 for h in handles {
712 h.await.unwrap();
713 }
714 assert_eq!(m.objects_in_backoff_count(), 0);
715 }
716
717 #[tokio::test]
718 async fn entry_api_insert_only_increments_once() {
719 use std::collections::HashMap;
720 use std::collections::hash_map::Entry;
721 use tokio::sync::RwLock;
722
723 let m = Arc::new(test_metrics());
724 let cache: Arc<RwLock<HashMap<String, RwLock<u32>>>> =
725 Arc::new(RwLock::new(HashMap::new()));
726 let mut handles = Vec::new();
727
728 for _ in 0..100 {
729 let m = m.clone();
730 let cache = cache.clone();
731 handles.push(tokio::spawn(async move {
732 let mut guard = cache.write().await;
733 match guard.entry("same-key".to_string()) {
734 Entry::Vacant(v) => {
735 v.insert(RwLock::new(0));
736 m.objects_in_backoff_inc();
737 }
738 Entry::Occupied(_) => {}
739 }
740 }));
741 }
742 for h in handles {
743 h.await.unwrap();
744 }
745 assert_eq!(m.objects_in_backoff_count(), 1);
746 assert_eq!(cache.read().await.len(), 1);
747 }
748
749 #[tokio::test]
750 async fn remove_only_decrements_on_actual_removal() {
751 use std::collections::HashMap;
752 use tokio::sync::RwLock;
753
754 let m = Arc::new(test_metrics());
755 let cache: Arc<RwLock<HashMap<String, u32>>> = Arc::new(RwLock::new(HashMap::new()));
756 cache.write().await.insert("key".to_string(), 42);
757 m.objects_in_backoff_inc();
758
759 let mut handles = Vec::new();
760 for _ in 0..100 {
761 let m = m.clone();
762 let cache = cache.clone();
763 handles.push(tokio::spawn(async move {
764 let removed = cache.write().await.remove("key");
765 if removed.is_some() {
766 m.objects_in_backoff_dec();
767 }
768 }));
769 }
770 for h in handles {
771 h.await.unwrap();
772 }
773 assert_eq!(m.objects_in_backoff_count(), 0);
774 assert!(cache.read().await.is_empty());
775 }
776
777 #[tokio::test]
778 async fn reconcile_outcome_record_records_changed() {
779 use opentelemetry::metrics::MeterProvider;
780 use opentelemetry_sdk::metrics::{
781 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
782 };
783
784 let exporter = InMemoryMetricExporter::default();
785 let reader = PeriodicReader::builder(exporter.clone())
786 .with_interval(std::time::Duration::from_millis(50))
787 .build();
788 let provider = SdkMeterProvider::builder().with_reader(reader).build();
789 let meter = provider.meter("test");
790 let m = ControllerMetrics::new("test-ctrl", &meter);
791
792 m.reconcile_outcome_record(KANIDM_OUTCOME_CHANGED);
793
794 provider.force_flush().unwrap();
795 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
796
797 let metrics = exporter.get_finished_metrics().unwrap();
798 let text = format!("{:?}", metrics);
799 assert!(text.contains("reconcile_outcome"));
800 assert!(text.contains("changed"));
801 assert!(text.contains("test-ctrl"));
802 }
803
804 #[tokio::test]
805 async fn reconcile_outcome_record_records_unchanged() {
806 use opentelemetry::metrics::MeterProvider;
807 use opentelemetry_sdk::metrics::{
808 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
809 };
810
811 let exporter = InMemoryMetricExporter::default();
812 let reader = PeriodicReader::builder(exporter.clone())
813 .with_interval(std::time::Duration::from_millis(50))
814 .build();
815 let provider = SdkMeterProvider::builder().with_reader(reader).build();
816 let meter = provider.meter("test");
817 let m = ControllerMetrics::new("test-ctrl", &meter);
818
819 m.reconcile_outcome_record(KANIDM_OUTCOME_UNCHANGED);
820
821 provider.force_flush().unwrap();
822 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
823
824 let metrics = exporter.get_finished_metrics().unwrap();
825 let text = format!("{:?}", metrics);
826 assert!(text.contains("reconcile_outcome"));
827 assert!(text.contains("unchanged"));
828 }
829
830 #[tokio::test]
831 async fn reconcile_outcome_record_records_error() {
832 use opentelemetry::metrics::MeterProvider;
833 use opentelemetry_sdk::metrics::{
834 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
835 };
836
837 let exporter = InMemoryMetricExporter::default();
838 let reader = PeriodicReader::builder(exporter.clone())
839 .with_interval(std::time::Duration::from_millis(50))
840 .build();
841 let provider = SdkMeterProvider::builder().with_reader(reader).build();
842 let meter = provider.meter("test");
843 let m = ControllerMetrics::new("test-ctrl", &meter);
844
845 m.reconcile_outcome_record(KANIDM_OUTCOME_ERROR);
846
847 provider.force_flush().unwrap();
848 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
849
850 let metrics = exporter.get_finished_metrics().unwrap();
851 let text = format!("{:?}", metrics);
852 assert!(text.contains("reconcile_outcome"));
853 assert!(text.contains("error"));
854 }
855}