1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicI64, Ordering};
6
7use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
8use opentelemetry::{KeyValue, trace::TraceId};
9use tokio::time::Instant;
10use tracing::debug;
11
12pub const KANIDM_RESOURCE_PERSON: &str = "Person";
13pub const KANIDM_RESOURCE_GROUP: &str = "Group";
14pub const KANIDM_RESOURCE_OAUTH2: &str = "OAuth2Client";
15pub const KANIDM_RESOURCE_SERVICE_ACCOUNT: &str = "ServiceAccount";
16pub const KANIDM_RESOURCE_DOMAIN: &str = "Domain";
17pub const KANIDM_RESOURCE_MAIL_SENDER: &str = "MailSender";
18
19pub const KANIDM_OP_GET: &str = "get";
20pub const KANIDM_OP_GET_CREDENTIAL_STATUS: &str = "get_credential_status";
21pub const KANIDM_OP_CREATE: &str = "create";
22pub const KANIDM_OP_UPDATE: &str = "update";
23pub const KANIDM_OP_DELETE: &str = "delete";
24pub const KANIDM_OP_UNIX_EXTEND: &str = "unix_extend";
25pub const KANIDM_OP_CREDENTIAL_UPDATE_INTENT: &str = "credential_update_intent";
26pub const KANIDM_OP_SET_MEMBERS: &str = "set_members";
27pub const KANIDM_OP_SET_MAIL: &str = "set_mail";
28pub const KANIDM_OP_PURGE_MAIL: &str = "purge_mail";
29pub const KANIDM_OP_ADD_MAIL: &str = "add_mail";
30pub const KANIDM_OP_REMOVE_MAIL: &str = "remove_mail";
31pub const KANIDM_OP_ACCOUNT_POLICY: &str = "account_policy";
32pub const KANIDM_OP_PURGE_ATTR: &str = "purge_attr";
33pub const KANIDM_OP_ROTATE_SECRET: &str = "rotate_secret";
34pub const KANIDM_OP_GET_SECRET: &str = "get_secret";
35pub const KANIDM_OP_ADD_ORIGIN: &str = "add_origin";
36pub const KANIDM_OP_REMOVE_ORIGIN: &str = "remove_origin";
37pub const KANIDM_OP_SCOPE_MAP: &str = "scope_map";
38pub const KANIDM_OP_DELETE_SCOPE_MAP: &str = "delete_scope_map";
39pub const KANIDM_OP_UPDATE_SCOPE_MAP: &str = "update_scope_map";
40pub const KANIDM_OP_SUP_SCOPE_MAP: &str = "sup_scope_map";
41pub const KANIDM_OP_DELETE_SUP_SCOPE_MAP: &str = "delete_sup_scope_map";
42pub const KANIDM_OP_UPDATE_SUP_SCOPE_MAP: &str = "update_sup_scope_map";
43pub const KANIDM_OP_CLAIM_MAP: &str = "claim_map";
44pub const KANIDM_OP_DELETE_CLAIM_MAP: &str = "delete_claim_map";
45pub const KANIDM_OP_UPDATE_CLAIM_MAP: &str = "update_claim_map";
46pub const KANIDM_OP_CLAIM_MAP_JOIN: &str = "claim_map_join";
47pub const KANIDM_OP_UPDATE_CLAIM_MAP_JOIN: &str = "update_claim_map_join";
48pub const KANIDM_OP_IMAGE: &str = "image";
49pub const KANIDM_OP_DELETE_IMAGE: &str = "delete_image";
50pub const KANIDM_OP_UPDATE_IMAGE: &str = "update_image";
51pub const KANIDM_OP_RESET_SECRET: &str = "reset_secret";
52pub const KANIDM_OP_GENERATE_API_TOKEN: &str = "generate_api_token";
53pub const KANIDM_OP_DESTROY_API_TOKEN: &str = "destroy_api_token";
54pub const KANIDM_OP_LIST_API_TOKENS: &str = "list_api_tokens";
55pub const KANIDM_OP_GENERATE_PASSWORD: &str = "generate_password";
56pub const KANIDM_OP_SET_DISPLAY_NAME: &str = "set_display_name";
57pub const KANIDM_OP_ADD_MEMBERS: &str = "add_members";
58pub const KANIDM_OP_REMOVE_MEMBERS: &str = "remove_members";
59
60pub const KANIDM_OUTCOME_CHANGED: &str = "changed";
61pub const KANIDM_OUTCOME_UNCHANGED: &str = "unchanged";
62pub const KANIDM_OUTCOME_ERROR: &str = "error";
63
64pub async fn record_kanidm_sdk_call<F, T, E>(
65 metrics: &ControllerMetrics,
66 resource: &'static str,
67 operation: &'static str,
68 success_outcome: &'static str,
69 future: F,
70) -> std::result::Result<T, E>
71where
72 F: std::future::Future<Output = std::result::Result<T, E>>,
73{
74 let start = Instant::now();
75 let result = future.await;
76 let outcome = match &result {
77 Ok(_) => success_outcome,
78 Err(_) => KANIDM_OUTCOME_ERROR,
79 };
80 metrics.kanidm_sdk_calls.add(
81 1,
82 &[
83 KeyValue::new("resource", resource),
84 KeyValue::new("operation", operation),
85 KeyValue::new("outcome", outcome),
86 ],
87 );
88 metrics.kanidm_sdk_call_duration.record(
89 start.elapsed().as_secs_f64(),
90 &[
91 KeyValue::new("resource", resource),
92 KeyValue::new("operation", operation),
93 ],
94 );
95 result
96}
97
98#[derive(Clone)]
99pub struct Metrics {
100 pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
101}
102
103impl Metrics {
104 pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
105 debug!(
106 "Initializing operator metrics for controllers: {:?}",
107 controller_names
108 );
109 let controllers = controller_names
110 .iter()
111 .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
112 .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
113
114 debug!("Operator metrics initialized");
115 Self { controllers }
116 }
117}
118
119#[derive(Clone)]
120pub struct ControllerMetrics {
121 controller: String,
122 pub reconcile: ReconcileMetrics,
123 spec_replicas: Gauge<i64>,
124 status_update_errors: Counter<u64>,
125 triggered: Counter<u64>,
126 watch_operations_failed: Counter<u64>,
127 ready: Gauge<i64>,
128 active_reconciles: Gauge<i64>,
129 active_reconciles_count: Arc<AtomicI64>,
130 objects_in_backoff: Gauge<i64>,
131 objects_in_backoff_count: Arc<AtomicI64>,
132 kanidm_sdk_calls: Counter<u64>,
133 kanidm_sdk_call_duration: Histogram<f64>,
134 reconcile_outcome: Counter<u64>,
135}
136
137impl ControllerMetrics {
138 pub fn new(controller: &str, meter: &Meter) -> Self {
139 let reconcile = ReconcileMetrics::new(meter);
140
141 let spec_replicas = meter
142 .i64_gauge("spec_replicas")
143 .with_description("Number of expected replicas for the object")
144 .build();
145
146 let status_update_errors = meter
147 .u64_counter("status_update_errors")
148 .with_description(
149 "Number of errors that occurred during update operations to status subresources",
150 )
151 .build();
152
153 let triggered = meter
154 .u64_counter("triggered")
155 .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
156 .build();
157
158 let watch_operations_failed = meter
159 .u64_counter("watch_operations_failed")
160 .with_description("Total number of watch operations that failed")
161 .build();
162
163 let ready = meter
164 .i64_gauge("ready")
165 .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
166 .build();
167
168 let active_reconciles = meter
169 .i64_gauge("active_reconciles")
170 .with_description("Number of reconcile operations currently in flight")
171 .build();
172
173 let objects_in_backoff = meter
174 .i64_gauge("objects_in_backoff")
175 .with_description("Number of objects currently in error backoff state")
176 .build();
177
178 let kanidm_sdk_calls = meter
179 .u64_counter("kanidm_sdk_calls")
180 .with_description("Total number of Kanidm SDK calls")
181 .build();
182
183 let kanidm_sdk_call_duration = meter
184 .f64_histogram("kanidm_sdk_call_duration_seconds")
185 .with_description("Duration of Kanidm SDK calls in seconds")
186 .with_boundaries(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0])
187 .build();
188
189 let reconcile_outcome = meter
190 .u64_counter("reconcile_outcome")
191 .with_description("Outcome of reconcile operations: changed, unchanged, or error")
192 .build();
193
194 Self {
195 controller: controller.to_string(),
196 reconcile,
197 spec_replicas,
198 status_update_errors,
199 triggered,
200 watch_operations_failed,
201 ready,
202 active_reconciles,
203 active_reconciles_count: Arc::new(AtomicI64::new(0)),
204 objects_in_backoff,
205 objects_in_backoff_count: Arc::new(AtomicI64::new(0)),
206 kanidm_sdk_calls,
207 kanidm_sdk_call_duration,
208 reconcile_outcome,
209 }
210 }
211
212 pub fn reconcile_failure_inc(&self) {
213 self.reconcile
214 .failures
215 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
216 }
217
218 pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
219 self.reconcile
220 .operations
221 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
222 ReconcileMeasurer {
223 start: Instant::now(),
224 controller: self.controller.clone(),
225 metric: self.reconcile.duration.clone(),
226 }
227 }
228
229 pub fn reconcile_deploy_delete_create_inc(&self, resource_kind: &str, reason: &str) {
230 self.reconcile.deploy_delete_create.add(
231 1,
232 &[
233 KeyValue::new("controller", self.controller.clone()),
234 KeyValue::new("resource_kind", resource_kind.to_string()),
235 KeyValue::new("reason", reason.to_string()),
236 ],
237 );
238 }
239
240 pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
241 self.spec_replicas.record(
242 replicas as i64,
243 &[
244 KeyValue::new("controller", self.controller.clone()),
245 KeyValue::new("namespace", namespace.to_string()),
246 KeyValue::new("name", name.to_string()),
247 ],
248 );
249 }
250
251 pub fn status_update_errors_inc(&self) {
252 self.status_update_errors
253 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
254 }
255
256 pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
257 self.triggered.add(
258 1,
259 &[
260 KeyValue::new("controller", self.controller.clone()),
261 KeyValue::new("action", action.as_str()),
262 KeyValue::new("triggered_by", triggered_by.to_string()),
263 ],
264 );
265 }
266
267 pub fn watch_operations_failed_inc(&self) {
268 self.watch_operations_failed
269 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
270 }
271
272 pub fn ready_set(&self, status: i64) {
273 self.ready.record(
274 status,
275 &[KeyValue::new("controller", self.controller.clone())],
276 );
277 }
278
279 pub fn active_reconcile_inc(&self) -> ActiveReconcileGuard {
280 let new = self.active_reconciles_count.fetch_add(1, Ordering::Relaxed) + 1;
281 self.active_reconciles
282 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
283 ActiveReconcileGuard {
284 count: self.active_reconciles_count.clone(),
285 gauge: self.active_reconciles.clone(),
286 controller: self.controller.clone(),
287 }
288 }
289
290 pub fn objects_in_backoff_inc(&self) {
291 let new = self
292 .objects_in_backoff_count
293 .fetch_add(1, Ordering::Relaxed)
294 + 1;
295 self.objects_in_backoff
296 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
297 }
298
299 pub fn objects_in_backoff_dec(&self) {
300 let prev = self
301 .objects_in_backoff_count
302 .fetch_sub(1, Ordering::Relaxed);
303 let new = (prev - 1).max(0);
304 self.objects_in_backoff
305 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
306 }
307
308 pub fn active_reconciles_count(&self) -> i64 {
309 self.active_reconciles_count.load(Ordering::Relaxed)
310 }
311
312 pub fn objects_in_backoff_count(&self) -> i64 {
313 self.objects_in_backoff_count.load(Ordering::Relaxed)
314 }
315
316 pub fn kanidm_sdk_call_inc(
317 &self,
318 resource: &'static str,
319 operation: &'static str,
320 outcome: &'static str,
321 ) {
322 self.kanidm_sdk_calls.add(
323 1,
324 &[
325 KeyValue::new("resource", resource),
326 KeyValue::new("operation", operation),
327 KeyValue::new("outcome", outcome),
328 ],
329 );
330 }
331
332 pub fn record_kanidm_sdk_outcome(
333 &self,
334 resource: &'static str,
335 operation: &'static str,
336 outcome: &'static str,
337 duration: std::time::Duration,
338 ) {
339 self.kanidm_sdk_calls.add(
340 1,
341 &[
342 KeyValue::new("resource", resource),
343 KeyValue::new("operation", operation),
344 KeyValue::new("outcome", outcome),
345 ],
346 );
347 self.kanidm_sdk_call_duration.record(
348 duration.as_secs_f64(),
349 &[
350 KeyValue::new("resource", resource),
351 KeyValue::new("operation", operation),
352 ],
353 );
354 }
355
356 pub fn reconcile_outcome_record(&self, outcome: &'static str) {
357 self.reconcile_outcome.add(
358 1,
359 &[
360 KeyValue::new("controller", self.controller.clone()),
361 KeyValue::new("outcome", outcome),
362 ],
363 );
364 }
365}
366
367#[derive(Clone)]
368pub struct ReconcileMetrics {
369 pub operations: Counter<u64>,
370 pub failures: Counter<u64>,
371 pub duration: Histogram<f64>,
372 pub deploy_delete_create: Counter<u64>,
373}
374
375impl ReconcileMetrics {
376 pub fn new(meter: &Meter) -> Self {
377 debug!("Initializing reconcile metrics");
378 let operations = meter
379 .u64_counter("reconcile_operations")
380 .with_description("Total number of reconcile operations")
381 .build();
382
383 let failures = meter
384 .u64_counter("reconcile_failures")
385 .with_description("Number of errors that occurred during reconcile operations")
386 .build();
387
388 let duration = meter
389 .f64_histogram("reconcile_duration_seconds")
390 .with_description("Histogram of reconcile operations")
391 .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
392 .build();
393
394 let deploy_delete_create = meter
395 .u64_counter("reconcile_deploy_delete_create")
396 .with_description("Number of explicit resource delete/recreate operations (not limited to deployments)")
397 .build();
398
399 debug!("Reconcile metrics initialized");
400 Self {
401 operations,
402 failures,
403 duration,
404 deploy_delete_create,
405 }
406 }
407}
408
409pub struct ReconcileMeasurer {
413 start: Instant,
414 controller: String,
415 metric: Histogram<f64>,
416}
417
418impl Drop for ReconcileMeasurer {
419 fn drop(&mut self) {
420 let duration = self.start.elapsed().as_secs_f64();
421 self.metric.record(
422 duration,
423 &[KeyValue::new("controller", self.controller.clone())],
424 );
425 }
426}
427
428pub struct ActiveReconcileGuard {
429 count: Arc<AtomicI64>,
430 gauge: Gauge<i64>,
431 controller: String,
432}
433
434impl Drop for ActiveReconcileGuard {
435 fn drop(&mut self) {
436 let prev = self.count.fetch_sub(1, Ordering::Relaxed);
437 self.gauge.record(
438 (prev - 1).max(0),
439 &[KeyValue::new("controller", self.controller.clone())],
440 );
441 }
442}
443
444#[derive(Clone, Debug)]
445pub enum Action {
446 Apply,
447 Delete,
448}
449
450impl Action {
451 pub fn as_str(&self) -> &'static str {
452 match self {
453 Action::Apply => "apply",
454 Action::Delete => "delete",
455 }
456 }
457}
458
459#[cfg(test)]
460mod tests {
461 use super::*;
462 use opentelemetry::metrics::MeterProvider;
463
464 fn test_metrics() -> ControllerMetrics {
465 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
466 let meter = provider.meter("test");
467 ControllerMetrics::new("test-controller", &meter)
468 }
469
470 #[test]
471 fn active_reconcile_guard_increments_on_creation() {
472 let m = test_metrics();
473 assert_eq!(m.active_reconciles_count(), 0);
474 let _guard = m.active_reconcile_inc();
475 assert_eq!(m.active_reconciles_count(), 1);
476 }
477
478 #[test]
479 fn active_reconcile_guard_decrements_on_drop() {
480 let m = test_metrics();
481 let guard = m.active_reconcile_inc();
482 assert_eq!(m.active_reconciles_count(), 1);
483 drop(guard);
484 assert_eq!(m.active_reconciles_count(), 0);
485 }
486
487 #[test]
488 fn active_reconcile_multiple_guards() {
489 let m = test_metrics();
490 let g1 = m.active_reconcile_inc();
491 let g2 = m.active_reconcile_inc();
492 let g3 = m.active_reconcile_inc();
493 assert_eq!(m.active_reconciles_count(), 3);
494 drop(g2);
495 assert_eq!(m.active_reconciles_count(), 2);
496 drop(g1);
497 assert_eq!(m.active_reconciles_count(), 1);
498 drop(g3);
499 assert_eq!(m.active_reconciles_count(), 0);
500 }
501
502 #[test]
503 fn active_reconcile_guard_does_not_underflow() {
504 let m = test_metrics();
505 let guard = m.active_reconcile_inc();
506 drop(guard);
507 assert_eq!(m.active_reconciles_count(), 0);
508 }
509
510 #[test]
511 fn objects_in_backoff_inc_dec() {
512 let m = test_metrics();
513 assert_eq!(m.objects_in_backoff_count(), 0);
514 m.objects_in_backoff_inc();
515 assert_eq!(m.objects_in_backoff_count(), 1);
516 m.objects_in_backoff_inc();
517 assert_eq!(m.objects_in_backoff_count(), 2);
518 m.objects_in_backoff_dec();
519 assert_eq!(m.objects_in_backoff_count(), 1);
520 m.objects_in_backoff_dec();
521 assert_eq!(m.objects_in_backoff_count(), 0);
522 }
523
524 #[test]
525 fn cloned_metrics_share_active_reconcile_counter() {
526 let m = test_metrics();
527 let m2 = m.clone();
528 let _g = m.active_reconcile_inc();
529 assert_eq!(m2.active_reconciles_count(), 1);
530 }
531
532 #[test]
533 fn cloned_metrics_share_backoff_counter() {
534 let m = test_metrics();
535 let m2 = m.clone();
536 m.objects_in_backoff_inc();
537 assert_eq!(m2.objects_in_backoff_count(), 1);
538 m2.objects_in_backoff_dec();
539 assert_eq!(m.objects_in_backoff_count(), 0);
540 }
541
542 #[test]
543 fn kanidm_sdk_call_inc_records_counter() {
544 let m = test_metrics();
545 m.kanidm_sdk_call_inc(
546 KANIDM_RESOURCE_PERSON,
547 KANIDM_OP_CREATE,
548 KANIDM_OUTCOME_CHANGED,
549 );
550 m.kanidm_sdk_call_inc(
551 KANIDM_RESOURCE_PERSON,
552 KANIDM_OP_GET,
553 KANIDM_OUTCOME_UNCHANGED,
554 );
555 m.kanidm_sdk_call_inc(
556 KANIDM_RESOURCE_GROUP,
557 KANIDM_OP_DELETE,
558 KANIDM_OUTCOME_ERROR,
559 );
560 }
561
562 #[test]
563 fn record_kanidm_sdk_outcome_records_count_and_duration() {
564 let m = test_metrics();
565 m.record_kanidm_sdk_outcome(
566 KANIDM_RESOURCE_PERSON,
567 KANIDM_OP_CREATE,
568 KANIDM_OUTCOME_CHANGED,
569 std::time::Duration::from_millis(50),
570 );
571 m.record_kanidm_sdk_outcome(
572 KANIDM_RESOURCE_PERSON,
573 KANIDM_OP_GET,
574 KANIDM_OUTCOME_UNCHANGED,
575 std::time::Duration::from_millis(10),
576 );
577 }
578
579 #[tokio::test]
580 async fn record_kanidm_sdk_call_helper_records_changed_on_success() {
581 use opentelemetry::metrics::MeterProvider;
582 use opentelemetry_sdk::metrics::{
583 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
584 };
585
586 let exporter = InMemoryMetricExporter::default();
587 let reader = PeriodicReader::builder(exporter.clone())
588 .with_interval(std::time::Duration::from_millis(50))
589 .build();
590 let provider = SdkMeterProvider::builder().with_reader(reader).build();
591 let meter = provider.meter("test");
592 let m = ControllerMetrics::new("test", &meter);
593
594 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
595 &m,
596 KANIDM_RESOURCE_PERSON,
597 KANIDM_OP_CREATE,
598 KANIDM_OUTCOME_CHANGED,
599 async { Ok(()) },
600 )
601 .await;
602 assert!(result.is_ok());
603
604 provider.force_flush().unwrap();
605 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
606
607 let metrics = exporter.get_finished_metrics().unwrap();
608 let text = format!("{:?}", metrics);
609 assert!(text.contains("kanidm_sdk_calls"));
610 assert!(text.contains("kanidm_sdk_call_duration_seconds"));
611 assert!(text.contains("changed"));
612 }
613
614 #[tokio::test]
615 async fn record_kanidm_sdk_call_helper_records_error_on_failure() {
616 use opentelemetry::metrics::MeterProvider;
617 use opentelemetry_sdk::metrics::{
618 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
619 };
620
621 let exporter = InMemoryMetricExporter::default();
622 let reader = PeriodicReader::builder(exporter.clone())
623 .with_interval(std::time::Duration::from_millis(50))
624 .build();
625 let provider = SdkMeterProvider::builder().with_reader(reader).build();
626 let meter = provider.meter("test");
627 let m = ControllerMetrics::new("test", &meter);
628
629 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
630 &m,
631 KANIDM_RESOURCE_GROUP,
632 KANIDM_OP_DELETE,
633 KANIDM_OUTCOME_CHANGED,
634 async { Err("something failed") },
635 )
636 .await;
637 assert!(result.is_err());
638
639 provider.force_flush().unwrap();
640 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
641
642 let metrics = exporter.get_finished_metrics().unwrap();
643 let text = format!("{:?}", metrics);
644 assert!(text.contains("error"));
645 }
646
647 #[tokio::test]
648 async fn record_kanidm_sdk_call_with_explicit_outcome() {
649 use opentelemetry::metrics::MeterProvider;
650 use opentelemetry_sdk::metrics::{
651 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
652 };
653
654 let exporter = InMemoryMetricExporter::default();
655 let reader = PeriodicReader::builder(exporter.clone())
656 .with_interval(std::time::Duration::from_millis(50))
657 .build();
658 let provider = SdkMeterProvider::builder().with_reader(reader).build();
659 let meter = provider.meter("test");
660 let m = ControllerMetrics::new("test", &meter);
661
662 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
663 &m,
664 KANIDM_RESOURCE_OAUTH2,
665 KANIDM_OP_GET_SECRET,
666 KANIDM_OUTCOME_UNCHANGED,
667 async { Ok(()) },
668 )
669 .await;
670 assert!(result.is_ok());
671
672 provider.force_flush().unwrap();
673 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
674
675 let metrics = exporter.get_finished_metrics().unwrap();
676 let text = format!("{:?}", metrics);
677 assert!(text.contains(r#""unchanged""#));
678 assert!(!text.contains(r#""changed""#));
679 }
680
681 #[tokio::test]
682 async fn concurrent_active_reconcile_guards_track_correctly() {
683 let m = Arc::new(test_metrics());
684 let mut handles = Vec::new();
685 for _ in 0..100 {
686 let m = m.clone();
687 handles.push(tokio::spawn(async move {
688 let _guard = m.active_reconcile_inc();
689 tokio::task::yield_now().await;
690 }));
691 }
692 for h in handles {
693 h.await.unwrap();
694 }
695 assert_eq!(m.active_reconciles_count(), 0);
696 }
697
698 #[tokio::test]
699 async fn concurrent_backoff_inc_dec_stays_balanced() {
700 let m = Arc::new(test_metrics());
701 let mut handles = Vec::new();
702 for _ in 0..50 {
703 let m = m.clone();
704 handles.push(tokio::spawn(async move {
705 m.objects_in_backoff_inc();
706 tokio::task::yield_now().await;
707 m.objects_in_backoff_dec();
708 }));
709 }
710 for h in handles {
711 h.await.unwrap();
712 }
713 assert_eq!(m.objects_in_backoff_count(), 0);
714 }
715
716 #[tokio::test]
717 async fn entry_api_insert_only_increments_once() {
718 use std::collections::HashMap;
719 use std::collections::hash_map::Entry;
720 use tokio::sync::RwLock;
721
722 let m = Arc::new(test_metrics());
723 let cache: Arc<RwLock<HashMap<String, RwLock<u32>>>> =
724 Arc::new(RwLock::new(HashMap::new()));
725 let mut handles = Vec::new();
726
727 for _ in 0..100 {
728 let m = m.clone();
729 let cache = cache.clone();
730 handles.push(tokio::spawn(async move {
731 let mut guard = cache.write().await;
732 match guard.entry("same-key".to_string()) {
733 Entry::Vacant(v) => {
734 v.insert(RwLock::new(0));
735 m.objects_in_backoff_inc();
736 }
737 Entry::Occupied(_) => {}
738 }
739 }));
740 }
741 for h in handles {
742 h.await.unwrap();
743 }
744 assert_eq!(m.objects_in_backoff_count(), 1);
745 assert_eq!(cache.read().await.len(), 1);
746 }
747
748 #[tokio::test]
749 async fn remove_only_decrements_on_actual_removal() {
750 use std::collections::HashMap;
751 use tokio::sync::RwLock;
752
753 let m = Arc::new(test_metrics());
754 let cache: Arc<RwLock<HashMap<String, u32>>> = Arc::new(RwLock::new(HashMap::new()));
755 cache.write().await.insert("key".to_string(), 42);
756 m.objects_in_backoff_inc();
757
758 let mut handles = Vec::new();
759 for _ in 0..100 {
760 let m = m.clone();
761 let cache = cache.clone();
762 handles.push(tokio::spawn(async move {
763 let removed = cache.write().await.remove("key");
764 if removed.is_some() {
765 m.objects_in_backoff_dec();
766 }
767 }));
768 }
769 for h in handles {
770 h.await.unwrap();
771 }
772 assert_eq!(m.objects_in_backoff_count(), 0);
773 assert!(cache.read().await.is_empty());
774 }
775
776 #[tokio::test]
777 async fn reconcile_outcome_record_records_changed() {
778 use opentelemetry::metrics::MeterProvider;
779 use opentelemetry_sdk::metrics::{
780 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
781 };
782
783 let exporter = InMemoryMetricExporter::default();
784 let reader = PeriodicReader::builder(exporter.clone())
785 .with_interval(std::time::Duration::from_millis(50))
786 .build();
787 let provider = SdkMeterProvider::builder().with_reader(reader).build();
788 let meter = provider.meter("test");
789 let m = ControllerMetrics::new("test-ctrl", &meter);
790
791 m.reconcile_outcome_record(KANIDM_OUTCOME_CHANGED);
792
793 provider.force_flush().unwrap();
794 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
795
796 let metrics = exporter.get_finished_metrics().unwrap();
797 let text = format!("{:?}", metrics);
798 assert!(text.contains("reconcile_outcome"));
799 assert!(text.contains("changed"));
800 assert!(text.contains("test-ctrl"));
801 }
802
803 #[tokio::test]
804 async fn reconcile_outcome_record_records_unchanged() {
805 use opentelemetry::metrics::MeterProvider;
806 use opentelemetry_sdk::metrics::{
807 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
808 };
809
810 let exporter = InMemoryMetricExporter::default();
811 let reader = PeriodicReader::builder(exporter.clone())
812 .with_interval(std::time::Duration::from_millis(50))
813 .build();
814 let provider = SdkMeterProvider::builder().with_reader(reader).build();
815 let meter = provider.meter("test");
816 let m = ControllerMetrics::new("test-ctrl", &meter);
817
818 m.reconcile_outcome_record(KANIDM_OUTCOME_UNCHANGED);
819
820 provider.force_flush().unwrap();
821 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
822
823 let metrics = exporter.get_finished_metrics().unwrap();
824 let text = format!("{:?}", metrics);
825 assert!(text.contains("reconcile_outcome"));
826 assert!(text.contains("unchanged"));
827 }
828
829 #[tokio::test]
830 async fn reconcile_outcome_record_records_error() {
831 use opentelemetry::metrics::MeterProvider;
832 use opentelemetry_sdk::metrics::{
833 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
834 };
835
836 let exporter = InMemoryMetricExporter::default();
837 let reader = PeriodicReader::builder(exporter.clone())
838 .with_interval(std::time::Duration::from_millis(50))
839 .build();
840 let provider = SdkMeterProvider::builder().with_reader(reader).build();
841 let meter = provider.meter("test");
842 let m = ControllerMetrics::new("test-ctrl", &meter);
843
844 m.reconcile_outcome_record(KANIDM_OUTCOME_ERROR);
845
846 provider.force_flush().unwrap();
847 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
848
849 let metrics = exporter.get_finished_metrics().unwrap();
850 let text = format!("{:?}", metrics);
851 assert!(text.contains("reconcile_outcome"));
852 assert!(text.contains("error"));
853 }
854}