Skip to main content

kaniop_operator/
metrics.rs

1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5
6use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
7use opentelemetry::{KeyValue, trace::TraceId};
8use tokio::time::Instant;
9use tracing::debug;
10
11#[derive(Clone)]
12pub struct Metrics {
13    pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
14}
15
16impl Metrics {
17    pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
18        debug!(
19            "Initializing operator metrics for controllers: {:?}",
20            controller_names
21        );
22        let controllers = controller_names
23            .iter()
24            .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
25            .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
26
27        debug!("Operator metrics initialized");
28        Self { controllers }
29    }
30}
31
32#[derive(Clone)]
33pub struct ControllerMetrics {
34    controller: String,
35    pub reconcile: ReconcileMetrics,
36    spec_replicas: Gauge<i64>,
37    status_update_errors: Counter<u64>,
38    triggered: Counter<u64>,
39    watch_operations_failed: Counter<u64>,
40    ready: Gauge<i64>,
41}
42
43impl ControllerMetrics {
44    pub fn new(controller: &str, meter: &Meter) -> Self {
45        let reconcile = ReconcileMetrics::new(meter);
46
47        let spec_replicas = meter
48            .i64_gauge("spec_replicas")
49            .with_description("Number of expected replicas for the object")
50            .build();
51
52        let status_update_errors = meter
53            .u64_counter("status_update_errors")
54            .with_description(
55                "Number of errors that occurred during update operations to status subresources",
56            )
57            .build();
58
59        let triggered = meter
60            .u64_counter("triggered")
61            .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
62            .build();
63
64        let watch_operations_failed = meter
65            .u64_counter("watch_operations_failed")
66            .with_description("Total number of watch operations that failed")
67            .build();
68
69        let ready = meter
70            .i64_gauge("ready")
71            .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
72            .build();
73
74        Self {
75            controller: controller.to_string(),
76            reconcile,
77            spec_replicas,
78            status_update_errors,
79            triggered,
80            watch_operations_failed,
81            ready,
82        }
83    }
84
85    pub fn reconcile_failure_inc(&self) {
86        self.reconcile
87            .failures
88            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
89    }
90
91    pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
92        self.reconcile
93            .operations
94            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
95        ReconcileMeasurer {
96            start: Instant::now(),
97            controller: self.controller.clone(),
98            metric: self.reconcile.duration.clone(),
99        }
100    }
101
102    pub fn reconcile_deploy_delete_create_inc(&self) {
103        self.reconcile
104            .deploy_delete_create
105            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
106    }
107
108    pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
109        self.spec_replicas.record(
110            replicas as i64,
111            &[
112                KeyValue::new("controller", self.controller.clone()),
113                KeyValue::new("namespace", namespace.to_string()),
114                KeyValue::new("name", name.to_string()),
115            ],
116        );
117    }
118
119    pub fn status_update_errors_inc(&self) {
120        self.status_update_errors
121            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
122    }
123
124    pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
125        self.triggered.add(
126            1,
127            &[
128                KeyValue::new("controller", self.controller.clone()),
129                KeyValue::new("action", action.as_str()),
130                KeyValue::new("triggered_by", triggered_by.to_string()),
131            ],
132        );
133    }
134
135    pub fn watch_operations_failed_inc(&self) {
136        self.watch_operations_failed
137            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
138    }
139
140    pub fn ready_set(&self, status: i64) {
141        self.ready.record(
142            status,
143            &[KeyValue::new("controller", self.controller.clone())],
144        );
145    }
146}
147
148#[derive(Clone)]
149pub struct ReconcileMetrics {
150    pub operations: Counter<u64>,
151    pub failures: Counter<u64>,
152    pub duration: Histogram<f64>,
153    pub deploy_delete_create: Counter<u64>,
154}
155
156impl ReconcileMetrics {
157    pub fn new(meter: &Meter) -> Self {
158        debug!("Initializing reconcile metrics");
159        let operations = meter
160            .u64_counter("reconcile_operations")
161            .with_description("Total number of reconcile operations")
162            .build();
163
164        let failures = meter
165            .u64_counter("reconcile_failures")
166            .with_description("Number of errors that occurred during reconcile operations")
167            .build();
168
169        let duration = meter
170            .f64_histogram("reconcile_duration_seconds")
171            .with_description("Histogram of reconcile operations")
172            .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
173            .build();
174
175        let deploy_delete_create = meter
176            .u64_counter("reconcile_deploy_delete_create")
177            .with_description("Number of times that reconciling a deployment required deleting and re-creating it")
178            .build();
179
180        debug!("Reconcile metrics initialized");
181        Self {
182            operations,
183            failures,
184            duration,
185            deploy_delete_create,
186        }
187    }
188}
189
190/// Smart function duration measurer
191///
192/// Relies on Drop to calculate duration and register the observation in the histogram
193pub struct ReconcileMeasurer {
194    start: Instant,
195    controller: String,
196    metric: Histogram<f64>,
197}
198
199impl Drop for ReconcileMeasurer {
200    fn drop(&mut self) {
201        let duration = self.start.elapsed().as_secs_f64();
202        self.metric.record(
203            duration,
204            &[KeyValue::new("controller", self.controller.clone())],
205        );
206    }
207}
208
209#[derive(Clone, Debug)]
210pub enum Action {
211    Apply,
212    Delete,
213}
214
215impl Action {
216    pub fn as_str(&self) -> &'static str {
217        match self {
218            Action::Apply => "apply",
219            Action::Delete => "delete",
220        }
221    }
222}