1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicI64, Ordering};
6
7use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
8use opentelemetry::{KeyValue, trace::TraceId};
9use tokio::time::Instant;
10use tracing::debug;
11
12pub const KANIDM_RESOURCE_PERSON: &str = "Person";
13pub const KANIDM_RESOURCE_GROUP: &str = "Group";
14pub const KANIDM_RESOURCE_OAUTH2: &str = "OAuth2Client";
15pub const KANIDM_RESOURCE_SERVICE_ACCOUNT: &str = "ServiceAccount";
16pub const KANIDM_RESOURCE_DOMAIN: &str = "Domain";
17pub const KANIDM_RESOURCE_MAIL_SENDER: &str = "MailSender";
18pub const KANIDM_RESOURCE_RADIUS_SERVER: &str = "RadiusServer";
19
20pub const KANIDM_OP_GET: &str = "get";
21pub const KANIDM_OP_GET_CREDENTIAL_STATUS: &str = "get_credential_status";
22pub const KANIDM_OP_CREATE: &str = "create";
23pub const KANIDM_OP_UPDATE: &str = "update";
24pub const KANIDM_OP_DELETE: &str = "delete";
25pub const KANIDM_OP_UNIX_EXTEND: &str = "unix_extend";
26pub const KANIDM_OP_CREDENTIAL_UPDATE_INTENT: &str = "credential_update_intent";
27pub const KANIDM_OP_CREDENTIAL_UPDATE_SEND_INTENT: &str = "credential_update_send_intent";
28pub const KANIDM_OP_SET_MEMBERS: &str = "set_members";
29pub const KANIDM_OP_SET_MAIL: &str = "set_mail";
30pub const KANIDM_OP_PURGE_MAIL: &str = "purge_mail";
31pub const KANIDM_OP_ADD_MAIL: &str = "add_mail";
32pub const KANIDM_OP_REMOVE_MAIL: &str = "remove_mail";
33pub const KANIDM_OP_ACCOUNT_POLICY: &str = "account_policy";
34pub const KANIDM_OP_PURGE_ATTR: &str = "purge_attr";
35pub const KANIDM_OP_ROTATE_SECRET: &str = "rotate_secret";
36pub const KANIDM_OP_GET_SECRET: &str = "get_secret";
37pub const KANIDM_OP_ADD_ORIGIN: &str = "add_origin";
38pub const KANIDM_OP_REMOVE_ORIGIN: &str = "remove_origin";
39pub const KANIDM_OP_SCOPE_MAP: &str = "scope_map";
40pub const KANIDM_OP_DELETE_SCOPE_MAP: &str = "delete_scope_map";
41pub const KANIDM_OP_UPDATE_SCOPE_MAP: &str = "update_scope_map";
42pub const KANIDM_OP_SUP_SCOPE_MAP: &str = "sup_scope_map";
43pub const KANIDM_OP_DELETE_SUP_SCOPE_MAP: &str = "delete_sup_scope_map";
44pub const KANIDM_OP_UPDATE_SUP_SCOPE_MAP: &str = "update_sup_scope_map";
45pub const KANIDM_OP_CLAIM_MAP: &str = "claim_map";
46pub const KANIDM_OP_DELETE_CLAIM_MAP: &str = "delete_claim_map";
47pub const KANIDM_OP_UPDATE_CLAIM_MAP: &str = "update_claim_map";
48pub const KANIDM_OP_CLAIM_MAP_JOIN: &str = "claim_map_join";
49pub const KANIDM_OP_UPDATE_CLAIM_MAP_JOIN: &str = "update_claim_map_join";
50pub const KANIDM_OP_IMAGE: &str = "image";
51pub const KANIDM_OP_DELETE_IMAGE: &str = "delete_image";
52pub const KANIDM_OP_UPDATE_IMAGE: &str = "update_image";
53pub const KANIDM_OP_RESET_SECRET: &str = "reset_secret";
54pub const KANIDM_OP_GENERATE_API_TOKEN: &str = "generate_api_token";
55pub const KANIDM_OP_DESTROY_API_TOKEN: &str = "destroy_api_token";
56pub const KANIDM_OP_LIST_API_TOKENS: &str = "list_api_tokens";
57pub const KANIDM_OP_GENERATE_PASSWORD: &str = "generate_password";
58pub const KANIDM_OP_SET_DISPLAY_NAME: &str = "set_display_name";
59pub const KANIDM_OP_ADD_MEMBERS: &str = "add_members";
60pub const KANIDM_OP_REMOVE_MEMBERS: &str = "remove_members";
61pub const KANIDM_OP_WHOAMI: &str = "whoami";
62
63pub const KANIDM_OUTCOME_CHANGED: &str = "changed";
64pub const KANIDM_OUTCOME_UNCHANGED: &str = "unchanged";
65pub const KANIDM_OUTCOME_ERROR: &str = "error";
66
67pub async fn record_kanidm_sdk_call<F, T, E>(
68 metrics: &ControllerMetrics,
69 resource: &'static str,
70 operation: &'static str,
71 success_outcome: &'static str,
72 future: F,
73) -> std::result::Result<T, E>
74where
75 F: std::future::Future<Output = std::result::Result<T, E>>,
76{
77 let start = Instant::now();
78 let result = future.await;
79 let outcome = match &result {
80 Ok(_) => success_outcome,
81 Err(_) => KANIDM_OUTCOME_ERROR,
82 };
83 metrics.kanidm_sdk_calls.add(
84 1,
85 &[
86 KeyValue::new("resource", resource),
87 KeyValue::new("operation", operation),
88 KeyValue::new("outcome", outcome),
89 ],
90 );
91 metrics.kanidm_sdk_call_duration.record(
92 start.elapsed().as_secs_f64(),
93 &[
94 KeyValue::new("resource", resource),
95 KeyValue::new("operation", operation),
96 ],
97 );
98 result
99}
100
101#[derive(Clone)]
102pub struct Metrics {
103 pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
104}
105
106impl Metrics {
107 pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
108 debug!(
109 "Initializing operator metrics for controllers: {:?}",
110 controller_names
111 );
112 let controllers = controller_names
113 .iter()
114 .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
115 .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
116
117 debug!("Operator metrics initialized");
118 Self { controllers }
119 }
120}
121
122#[derive(Clone)]
123pub struct ControllerMetrics {
124 controller: String,
125 pub reconcile: ReconcileMetrics,
126 spec_replicas: Gauge<i64>,
127 status_update_errors: Counter<u64>,
128 triggered: Counter<u64>,
129 watch_operations_failed: Counter<u64>,
130 ready: Gauge<i64>,
131 active_reconciles: Gauge<i64>,
132 active_reconciles_count: Arc<AtomicI64>,
133 objects_in_backoff: Gauge<i64>,
134 objects_in_backoff_count: Arc<AtomicI64>,
135 kanidm_sdk_calls: Counter<u64>,
136 kanidm_sdk_call_duration: Histogram<f64>,
137 reconcile_outcome: Counter<u64>,
138}
139
140impl ControllerMetrics {
141 pub fn new(controller: &str, meter: &Meter) -> Self {
142 let reconcile = ReconcileMetrics::new(meter);
143
144 let spec_replicas = meter
145 .i64_gauge("spec_replicas")
146 .with_description("Number of expected replicas for the object")
147 .build();
148
149 let status_update_errors = meter
150 .u64_counter("status_update_errors")
151 .with_description(
152 "Number of errors that occurred during update operations to status subresources",
153 )
154 .build();
155
156 let triggered = meter
157 .u64_counter("triggered")
158 .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
159 .build();
160
161 let watch_operations_failed = meter
162 .u64_counter("watch_operations_failed")
163 .with_description("Total number of watch operations that failed")
164 .build();
165
166 let ready = meter
167 .i64_gauge("ready")
168 .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
169 .build();
170
171 let active_reconciles = meter
172 .i64_gauge("active_reconciles")
173 .with_description("Number of reconcile operations currently in flight")
174 .build();
175
176 let objects_in_backoff = meter
177 .i64_gauge("objects_in_backoff")
178 .with_description("Number of objects currently in error backoff state")
179 .build();
180
181 let kanidm_sdk_calls = meter
182 .u64_counter("kanidm_sdk_calls")
183 .with_description("Total number of Kanidm SDK calls")
184 .build();
185
186 let kanidm_sdk_call_duration = meter
187 .f64_histogram("kanidm_sdk_call_duration_seconds")
188 .with_description("Duration of Kanidm SDK calls in seconds")
189 .with_boundaries(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0])
190 .build();
191
192 let reconcile_outcome = meter
193 .u64_counter("reconcile_outcome")
194 .with_description("Outcome of reconcile operations: changed, unchanged, or error")
195 .build();
196
197 Self {
198 controller: controller.to_string(),
199 reconcile,
200 spec_replicas,
201 status_update_errors,
202 triggered,
203 watch_operations_failed,
204 ready,
205 active_reconciles,
206 active_reconciles_count: Arc::new(AtomicI64::new(0)),
207 objects_in_backoff,
208 objects_in_backoff_count: Arc::new(AtomicI64::new(0)),
209 kanidm_sdk_calls,
210 kanidm_sdk_call_duration,
211 reconcile_outcome,
212 }
213 }
214
215 pub fn reconcile_failure_inc(&self) {
216 self.reconcile
217 .failures
218 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
219 }
220
221 pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
222 self.reconcile
223 .operations
224 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
225 ReconcileMeasurer {
226 start: Instant::now(),
227 controller: self.controller.clone(),
228 metric: self.reconcile.duration.clone(),
229 }
230 }
231
232 pub fn reconcile_deploy_delete_create_inc(&self, resource_kind: &str, reason: &str) {
233 self.reconcile.deploy_delete_create.add(
234 1,
235 &[
236 KeyValue::new("controller", self.controller.clone()),
237 KeyValue::new("resource_kind", resource_kind.to_string()),
238 KeyValue::new("reason", reason.to_string()),
239 ],
240 );
241 }
242
243 pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
244 self.spec_replicas.record(
245 replicas as i64,
246 &[
247 KeyValue::new("controller", self.controller.clone()),
248 KeyValue::new("namespace", namespace.to_string()),
249 KeyValue::new("name", name.to_string()),
250 ],
251 );
252 }
253
254 pub fn status_update_errors_inc(&self) {
255 self.status_update_errors
256 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
257 }
258
259 pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
260 self.triggered.add(
261 1,
262 &[
263 KeyValue::new("controller", self.controller.clone()),
264 KeyValue::new("action", action.as_str()),
265 KeyValue::new("triggered_by", triggered_by.to_string()),
266 ],
267 );
268 }
269
270 pub fn watch_operations_failed_inc(&self) {
271 self.watch_operations_failed
272 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
273 }
274
275 pub fn ready_set(&self, status: i64) {
276 self.ready.record(
277 status,
278 &[KeyValue::new("controller", self.controller.clone())],
279 );
280 }
281
282 pub fn active_reconcile_inc(&self) -> ActiveReconcileGuard {
283 let new = self.active_reconciles_count.fetch_add(1, Ordering::Relaxed) + 1;
284 self.active_reconciles
285 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
286 ActiveReconcileGuard {
287 count: self.active_reconciles_count.clone(),
288 gauge: self.active_reconciles.clone(),
289 controller: self.controller.clone(),
290 }
291 }
292
293 pub fn objects_in_backoff_inc(&self) {
294 let new = self
295 .objects_in_backoff_count
296 .fetch_add(1, Ordering::Relaxed)
297 + 1;
298 self.objects_in_backoff
299 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
300 }
301
302 pub fn objects_in_backoff_dec(&self) {
303 let prev = self
304 .objects_in_backoff_count
305 .fetch_sub(1, Ordering::Relaxed);
306 let new = (prev - 1).max(0);
307 self.objects_in_backoff
308 .record(new, &[KeyValue::new("controller", self.controller.clone())]);
309 }
310
311 pub fn active_reconciles_count(&self) -> i64 {
312 self.active_reconciles_count.load(Ordering::Relaxed)
313 }
314
315 pub fn objects_in_backoff_count(&self) -> i64 {
316 self.objects_in_backoff_count.load(Ordering::Relaxed)
317 }
318
319 pub fn kanidm_sdk_call_inc(
320 &self,
321 resource: &'static str,
322 operation: &'static str,
323 outcome: &'static str,
324 ) {
325 self.kanidm_sdk_calls.add(
326 1,
327 &[
328 KeyValue::new("resource", resource),
329 KeyValue::new("operation", operation),
330 KeyValue::new("outcome", outcome),
331 ],
332 );
333 }
334
335 pub fn record_kanidm_sdk_outcome(
336 &self,
337 resource: &'static str,
338 operation: &'static str,
339 outcome: &'static str,
340 duration: std::time::Duration,
341 ) {
342 self.kanidm_sdk_calls.add(
343 1,
344 &[
345 KeyValue::new("resource", resource),
346 KeyValue::new("operation", operation),
347 KeyValue::new("outcome", outcome),
348 ],
349 );
350 self.kanidm_sdk_call_duration.record(
351 duration.as_secs_f64(),
352 &[
353 KeyValue::new("resource", resource),
354 KeyValue::new("operation", operation),
355 ],
356 );
357 }
358
359 pub fn reconcile_outcome_record(&self, outcome: &'static str) {
360 self.reconcile_outcome.add(
361 1,
362 &[
363 KeyValue::new("controller", self.controller.clone()),
364 KeyValue::new("outcome", outcome),
365 ],
366 );
367 }
368}
369
370#[derive(Clone)]
371pub struct ReconcileMetrics {
372 pub operations: Counter<u64>,
373 pub failures: Counter<u64>,
374 pub duration: Histogram<f64>,
375 pub deploy_delete_create: Counter<u64>,
376}
377
378impl ReconcileMetrics {
379 pub fn new(meter: &Meter) -> Self {
380 debug!("Initializing reconcile metrics");
381 let operations = meter
382 .u64_counter("reconcile_operations")
383 .with_description("Total number of reconcile operations")
384 .build();
385
386 let failures = meter
387 .u64_counter("reconcile_failures")
388 .with_description("Number of errors that occurred during reconcile operations")
389 .build();
390
391 let duration = meter
392 .f64_histogram("reconcile_duration_seconds")
393 .with_description("Histogram of reconcile operations")
394 .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
395 .build();
396
397 let deploy_delete_create = meter
398 .u64_counter("reconcile_deploy_delete_create")
399 .with_description("Number of explicit resource delete/recreate operations (not limited to deployments)")
400 .build();
401
402 debug!("Reconcile metrics initialized");
403 Self {
404 operations,
405 failures,
406 duration,
407 deploy_delete_create,
408 }
409 }
410}
411
412pub struct ReconcileMeasurer {
416 start: Instant,
417 controller: String,
418 metric: Histogram<f64>,
419}
420
421impl Drop for ReconcileMeasurer {
422 fn drop(&mut self) {
423 let duration = self.start.elapsed().as_secs_f64();
424 self.metric.record(
425 duration,
426 &[KeyValue::new("controller", self.controller.clone())],
427 );
428 }
429}
430
431pub struct ActiveReconcileGuard {
432 count: Arc<AtomicI64>,
433 gauge: Gauge<i64>,
434 controller: String,
435}
436
437impl Drop for ActiveReconcileGuard {
438 fn drop(&mut self) {
439 let prev = self.count.fetch_sub(1, Ordering::Relaxed);
440 self.gauge.record(
441 (prev - 1).max(0),
442 &[KeyValue::new("controller", self.controller.clone())],
443 );
444 }
445}
446
447#[derive(Clone, Debug)]
448pub enum Action {
449 Apply,
450 Delete,
451}
452
453impl Action {
454 pub fn as_str(&self) -> &'static str {
455 match self {
456 Action::Apply => "apply",
457 Action::Delete => "delete",
458 }
459 }
460}
461
462#[cfg(test)]
463mod tests {
464 use super::*;
465 use opentelemetry::metrics::MeterProvider;
466
467 fn test_metrics() -> ControllerMetrics {
468 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
469 let meter = provider.meter("test");
470 ControllerMetrics::new("test-controller", &meter)
471 }
472
473 #[test]
474 fn active_reconcile_guard_increments_on_creation() {
475 let m = test_metrics();
476 assert_eq!(m.active_reconciles_count(), 0);
477 let _guard = m.active_reconcile_inc();
478 assert_eq!(m.active_reconciles_count(), 1);
479 }
480
481 #[test]
482 fn active_reconcile_guard_decrements_on_drop() {
483 let m = test_metrics();
484 let guard = m.active_reconcile_inc();
485 assert_eq!(m.active_reconciles_count(), 1);
486 drop(guard);
487 assert_eq!(m.active_reconciles_count(), 0);
488 }
489
490 #[test]
491 fn active_reconcile_multiple_guards() {
492 let m = test_metrics();
493 let g1 = m.active_reconcile_inc();
494 let g2 = m.active_reconcile_inc();
495 let g3 = m.active_reconcile_inc();
496 assert_eq!(m.active_reconciles_count(), 3);
497 drop(g2);
498 assert_eq!(m.active_reconciles_count(), 2);
499 drop(g1);
500 assert_eq!(m.active_reconciles_count(), 1);
501 drop(g3);
502 assert_eq!(m.active_reconciles_count(), 0);
503 }
504
505 #[test]
506 fn active_reconcile_guard_does_not_underflow() {
507 let m = test_metrics();
508 let guard = m.active_reconcile_inc();
509 drop(guard);
510 assert_eq!(m.active_reconciles_count(), 0);
511 }
512
513 #[test]
514 fn objects_in_backoff_inc_dec() {
515 let m = test_metrics();
516 assert_eq!(m.objects_in_backoff_count(), 0);
517 m.objects_in_backoff_inc();
518 assert_eq!(m.objects_in_backoff_count(), 1);
519 m.objects_in_backoff_inc();
520 assert_eq!(m.objects_in_backoff_count(), 2);
521 m.objects_in_backoff_dec();
522 assert_eq!(m.objects_in_backoff_count(), 1);
523 m.objects_in_backoff_dec();
524 assert_eq!(m.objects_in_backoff_count(), 0);
525 }
526
527 #[test]
528 fn cloned_metrics_share_active_reconcile_counter() {
529 let m = test_metrics();
530 let m2 = m.clone();
531 let _g = m.active_reconcile_inc();
532 assert_eq!(m2.active_reconciles_count(), 1);
533 }
534
535 #[test]
536 fn cloned_metrics_share_backoff_counter() {
537 let m = test_metrics();
538 let m2 = m.clone();
539 m.objects_in_backoff_inc();
540 assert_eq!(m2.objects_in_backoff_count(), 1);
541 m2.objects_in_backoff_dec();
542 assert_eq!(m.objects_in_backoff_count(), 0);
543 }
544
545 #[test]
546 fn kanidm_sdk_call_inc_records_counter() {
547 let m = test_metrics();
548 m.kanidm_sdk_call_inc(
549 KANIDM_RESOURCE_PERSON,
550 KANIDM_OP_CREATE,
551 KANIDM_OUTCOME_CHANGED,
552 );
553 m.kanidm_sdk_call_inc(
554 KANIDM_RESOURCE_PERSON,
555 KANIDM_OP_GET,
556 KANIDM_OUTCOME_UNCHANGED,
557 );
558 m.kanidm_sdk_call_inc(
559 KANIDM_RESOURCE_GROUP,
560 KANIDM_OP_DELETE,
561 KANIDM_OUTCOME_ERROR,
562 );
563 }
564
565 #[test]
566 fn record_kanidm_sdk_outcome_records_count_and_duration() {
567 let m = test_metrics();
568 m.record_kanidm_sdk_outcome(
569 KANIDM_RESOURCE_PERSON,
570 KANIDM_OP_CREATE,
571 KANIDM_OUTCOME_CHANGED,
572 std::time::Duration::from_millis(50),
573 );
574 m.record_kanidm_sdk_outcome(
575 KANIDM_RESOURCE_PERSON,
576 KANIDM_OP_GET,
577 KANIDM_OUTCOME_UNCHANGED,
578 std::time::Duration::from_millis(10),
579 );
580 }
581
582 #[tokio::test]
583 async fn record_kanidm_sdk_call_helper_records_changed_on_success() {
584 use opentelemetry::metrics::MeterProvider;
585 use opentelemetry_sdk::metrics::{
586 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
587 };
588
589 let exporter = InMemoryMetricExporter::default();
590 let reader = PeriodicReader::builder(exporter.clone())
591 .with_interval(std::time::Duration::from_millis(50))
592 .build();
593 let provider = SdkMeterProvider::builder().with_reader(reader).build();
594 let meter = provider.meter("test");
595 let m = ControllerMetrics::new("test", &meter);
596
597 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
598 &m,
599 KANIDM_RESOURCE_PERSON,
600 KANIDM_OP_CREATE,
601 KANIDM_OUTCOME_CHANGED,
602 async { Ok(()) },
603 )
604 .await;
605 assert!(result.is_ok());
606
607 provider.force_flush().unwrap();
608 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
609
610 let metrics = exporter.get_finished_metrics().unwrap();
611 let text = format!("{:?}", metrics);
612 assert!(text.contains("kanidm_sdk_calls"));
613 assert!(text.contains("kanidm_sdk_call_duration_seconds"));
614 assert!(text.contains("changed"));
615 }
616
617 #[tokio::test]
618 async fn record_kanidm_sdk_call_helper_records_error_on_failure() {
619 use opentelemetry::metrics::MeterProvider;
620 use opentelemetry_sdk::metrics::{
621 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
622 };
623
624 let exporter = InMemoryMetricExporter::default();
625 let reader = PeriodicReader::builder(exporter.clone())
626 .with_interval(std::time::Duration::from_millis(50))
627 .build();
628 let provider = SdkMeterProvider::builder().with_reader(reader).build();
629 let meter = provider.meter("test");
630 let m = ControllerMetrics::new("test", &meter);
631
632 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
633 &m,
634 KANIDM_RESOURCE_GROUP,
635 KANIDM_OP_DELETE,
636 KANIDM_OUTCOME_CHANGED,
637 async { Err("something failed") },
638 )
639 .await;
640 assert!(result.is_err());
641
642 provider.force_flush().unwrap();
643 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
644
645 let metrics = exporter.get_finished_metrics().unwrap();
646 let text = format!("{:?}", metrics);
647 assert!(text.contains("error"));
648 }
649
650 #[tokio::test]
651 async fn record_kanidm_sdk_call_with_explicit_outcome() {
652 use opentelemetry::metrics::MeterProvider;
653 use opentelemetry_sdk::metrics::{
654 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
655 };
656
657 let exporter = InMemoryMetricExporter::default();
658 let reader = PeriodicReader::builder(exporter.clone())
659 .with_interval(std::time::Duration::from_millis(50))
660 .build();
661 let provider = SdkMeterProvider::builder().with_reader(reader).build();
662 let meter = provider.meter("test");
663 let m = ControllerMetrics::new("test", &meter);
664
665 let result: std::result::Result<(), &str> = record_kanidm_sdk_call(
666 &m,
667 KANIDM_RESOURCE_OAUTH2,
668 KANIDM_OP_GET_SECRET,
669 KANIDM_OUTCOME_UNCHANGED,
670 async { Ok(()) },
671 )
672 .await;
673 assert!(result.is_ok());
674
675 provider.force_flush().unwrap();
676 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
677
678 let metrics = exporter.get_finished_metrics().unwrap();
679 let text = format!("{:?}", metrics);
680 assert!(text.contains(r#""unchanged""#));
681 assert!(!text.contains(r#""changed""#));
682 }
683
684 #[tokio::test]
685 async fn concurrent_active_reconcile_guards_track_correctly() {
686 let m = Arc::new(test_metrics());
687 let mut handles = Vec::new();
688 for _ in 0..100 {
689 let m = m.clone();
690 handles.push(tokio::spawn(async move {
691 let _guard = m.active_reconcile_inc();
692 tokio::task::yield_now().await;
693 }));
694 }
695 for h in handles {
696 h.await.unwrap();
697 }
698 assert_eq!(m.active_reconciles_count(), 0);
699 }
700
701 #[tokio::test]
702 async fn concurrent_backoff_inc_dec_stays_balanced() {
703 let m = Arc::new(test_metrics());
704 let mut handles = Vec::new();
705 for _ in 0..50 {
706 let m = m.clone();
707 handles.push(tokio::spawn(async move {
708 m.objects_in_backoff_inc();
709 tokio::task::yield_now().await;
710 m.objects_in_backoff_dec();
711 }));
712 }
713 for h in handles {
714 h.await.unwrap();
715 }
716 assert_eq!(m.objects_in_backoff_count(), 0);
717 }
718
719 #[tokio::test]
720 async fn entry_api_insert_only_increments_once() {
721 use std::collections::HashMap;
722 use std::collections::hash_map::Entry;
723 use tokio::sync::RwLock;
724
725 let m = Arc::new(test_metrics());
726 let cache: Arc<RwLock<HashMap<String, RwLock<u32>>>> =
727 Arc::new(RwLock::new(HashMap::new()));
728 let mut handles = Vec::new();
729
730 for _ in 0..100 {
731 let m = m.clone();
732 let cache = cache.clone();
733 handles.push(tokio::spawn(async move {
734 let mut guard = cache.write().await;
735 match guard.entry("same-key".to_string()) {
736 Entry::Vacant(v) => {
737 v.insert(RwLock::new(0));
738 m.objects_in_backoff_inc();
739 }
740 Entry::Occupied(_) => {}
741 }
742 }));
743 }
744 for h in handles {
745 h.await.unwrap();
746 }
747 assert_eq!(m.objects_in_backoff_count(), 1);
748 assert_eq!(cache.read().await.len(), 1);
749 }
750
751 #[tokio::test]
752 async fn remove_only_decrements_on_actual_removal() {
753 use std::collections::HashMap;
754 use tokio::sync::RwLock;
755
756 let m = Arc::new(test_metrics());
757 let cache: Arc<RwLock<HashMap<String, u32>>> = Arc::new(RwLock::new(HashMap::new()));
758 cache.write().await.insert("key".to_string(), 42);
759 m.objects_in_backoff_inc();
760
761 let mut handles = Vec::new();
762 for _ in 0..100 {
763 let m = m.clone();
764 let cache = cache.clone();
765 handles.push(tokio::spawn(async move {
766 let removed = cache.write().await.remove("key");
767 if removed.is_some() {
768 m.objects_in_backoff_dec();
769 }
770 }));
771 }
772 for h in handles {
773 h.await.unwrap();
774 }
775 assert_eq!(m.objects_in_backoff_count(), 0);
776 assert!(cache.read().await.is_empty());
777 }
778
779 #[tokio::test]
780 async fn reconcile_outcome_record_records_changed() {
781 use opentelemetry::metrics::MeterProvider;
782 use opentelemetry_sdk::metrics::{
783 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
784 };
785
786 let exporter = InMemoryMetricExporter::default();
787 let reader = PeriodicReader::builder(exporter.clone())
788 .with_interval(std::time::Duration::from_millis(50))
789 .build();
790 let provider = SdkMeterProvider::builder().with_reader(reader).build();
791 let meter = provider.meter("test");
792 let m = ControllerMetrics::new("test-ctrl", &meter);
793
794 m.reconcile_outcome_record(KANIDM_OUTCOME_CHANGED);
795
796 provider.force_flush().unwrap();
797 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
798
799 let metrics = exporter.get_finished_metrics().unwrap();
800 let text = format!("{:?}", metrics);
801 assert!(text.contains("reconcile_outcome"));
802 assert!(text.contains("changed"));
803 assert!(text.contains("test-ctrl"));
804 }
805
806 #[tokio::test]
807 async fn reconcile_outcome_record_records_unchanged() {
808 use opentelemetry::metrics::MeterProvider;
809 use opentelemetry_sdk::metrics::{
810 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
811 };
812
813 let exporter = InMemoryMetricExporter::default();
814 let reader = PeriodicReader::builder(exporter.clone())
815 .with_interval(std::time::Duration::from_millis(50))
816 .build();
817 let provider = SdkMeterProvider::builder().with_reader(reader).build();
818 let meter = provider.meter("test");
819 let m = ControllerMetrics::new("test-ctrl", &meter);
820
821 m.reconcile_outcome_record(KANIDM_OUTCOME_UNCHANGED);
822
823 provider.force_flush().unwrap();
824 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
825
826 let metrics = exporter.get_finished_metrics().unwrap();
827 let text = format!("{:?}", metrics);
828 assert!(text.contains("reconcile_outcome"));
829 assert!(text.contains("unchanged"));
830 }
831
832 #[tokio::test]
833 async fn reconcile_outcome_record_records_error() {
834 use opentelemetry::metrics::MeterProvider;
835 use opentelemetry_sdk::metrics::{
836 InMemoryMetricExporter, PeriodicReader, SdkMeterProvider,
837 };
838
839 let exporter = InMemoryMetricExporter::default();
840 let reader = PeriodicReader::builder(exporter.clone())
841 .with_interval(std::time::Duration::from_millis(50))
842 .build();
843 let provider = SdkMeterProvider::builder().with_reader(reader).build();
844 let meter = provider.meter("test");
845 let m = ControllerMetrics::new("test-ctrl", &meter);
846
847 m.reconcile_outcome_record(KANIDM_OUTCOME_ERROR);
848
849 provider.force_flush().unwrap();
850 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
851
852 let metrics = exporter.get_finished_metrics().unwrap();
853 let text = format!("{:?}", metrics);
854 assert!(text.contains("reconcile_outcome"));
855 assert!(text.contains("error"));
856 }
857}