Skip to main content

kaniop_operator/
metrics.rs

1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5
6use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
7use opentelemetry::{KeyValue, trace::TraceId};
8use tokio::time::Instant;
9use tracing::debug;
10
11#[derive(Clone)]
12pub struct Metrics {
13    pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
14}
15
16impl Metrics {
17    pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
18        debug!(
19            "Initializing operator metrics for controllers: {:?}",
20            controller_names
21        );
22        let controllers = controller_names
23            .iter()
24            .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
25            .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
26
27        debug!("Operator metrics initialized");
28        Self { controllers }
29    }
30}
31
32#[derive(Clone)]
33pub struct ControllerMetrics {
34    controller: String,
35    pub reconcile: ReconcileMetrics,
36    spec_replicas: Gauge<i64>,
37    status_update_errors: Counter<u64>,
38    triggered: Counter<u64>,
39    watch_operations_failed: Counter<u64>,
40    ready: Gauge<i64>,
41}
42
43impl ControllerMetrics {
44    pub fn new(controller: &str, meter: &Meter) -> Self {
45        let reconcile = ReconcileMetrics::new(meter);
46
47        let spec_replicas = meter
48            .i64_gauge("spec_replicas")
49            .with_description("Number of expected replicas for the object")
50            .build();
51
52        let status_update_errors = meter
53            .u64_counter("status_update_errors")
54            .with_description(
55                "Number of errors that occurred during update operations to status subresources",
56            )
57            .build();
58
59        let triggered = meter
60            .u64_counter("triggered")
61            .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
62            .build();
63
64        let watch_operations_failed = meter
65            .u64_counter("watch_operations_failed")
66            .with_description("Total number of watch operations that failed")
67            .build();
68
69        let ready = meter
70            .i64_gauge("ready")
71            .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
72            .build();
73
74        Self {
75            controller: controller.to_string(),
76            reconcile,
77            spec_replicas,
78            status_update_errors,
79            triggered,
80            watch_operations_failed,
81            ready,
82        }
83    }
84
85    pub fn reconcile_failure_inc(&self) {
86        self.reconcile
87            .failures
88            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
89    }
90
91    pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
92        self.reconcile
93            .operations
94            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
95        ReconcileMeasurer {
96            start: Instant::now(),
97            controller: self.controller.clone(),
98            metric: self.reconcile.duration.clone(),
99        }
100    }
101
102    pub fn reconcile_deploy_delete_create_inc(&self, resource_kind: &str, reason: &str) {
103        self.reconcile.deploy_delete_create.add(
104            1,
105            &[
106                KeyValue::new("controller", self.controller.clone()),
107                KeyValue::new("resource_kind", resource_kind.to_string()),
108                KeyValue::new("reason", reason.to_string()),
109            ],
110        );
111    }
112
113    pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
114        self.spec_replicas.record(
115            replicas as i64,
116            &[
117                KeyValue::new("controller", self.controller.clone()),
118                KeyValue::new("namespace", namespace.to_string()),
119                KeyValue::new("name", name.to_string()),
120            ],
121        );
122    }
123
124    pub fn status_update_errors_inc(&self) {
125        self.status_update_errors
126            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
127    }
128
129    pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
130        self.triggered.add(
131            1,
132            &[
133                KeyValue::new("controller", self.controller.clone()),
134                KeyValue::new("action", action.as_str()),
135                KeyValue::new("triggered_by", triggered_by.to_string()),
136            ],
137        );
138    }
139
140    pub fn watch_operations_failed_inc(&self) {
141        self.watch_operations_failed
142            .add(1, &[KeyValue::new("controller", self.controller.clone())]);
143    }
144
145    pub fn ready_set(&self, status: i64) {
146        self.ready.record(
147            status,
148            &[KeyValue::new("controller", self.controller.clone())],
149        );
150    }
151}
152
153#[derive(Clone)]
154pub struct ReconcileMetrics {
155    pub operations: Counter<u64>,
156    pub failures: Counter<u64>,
157    pub duration: Histogram<f64>,
158    pub deploy_delete_create: Counter<u64>,
159}
160
161impl ReconcileMetrics {
162    pub fn new(meter: &Meter) -> Self {
163        debug!("Initializing reconcile metrics");
164        let operations = meter
165            .u64_counter("reconcile_operations")
166            .with_description("Total number of reconcile operations")
167            .build();
168
169        let failures = meter
170            .u64_counter("reconcile_failures")
171            .with_description("Number of errors that occurred during reconcile operations")
172            .build();
173
174        let duration = meter
175            .f64_histogram("reconcile_duration_seconds")
176            .with_description("Histogram of reconcile operations")
177            .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
178            .build();
179
180        let deploy_delete_create = meter
181            .u64_counter("reconcile_deploy_delete_create")
182            .with_description("Number of explicit resource delete/recreate operations (not limited to deployments)")
183            .build();
184
185        debug!("Reconcile metrics initialized");
186        Self {
187            operations,
188            failures,
189            duration,
190            deploy_delete_create,
191        }
192    }
193}
194
195/// Smart function duration measurer
196///
197/// Relies on Drop to calculate duration and register the observation in the histogram
198pub struct ReconcileMeasurer {
199    start: Instant,
200    controller: String,
201    metric: Histogram<f64>,
202}
203
204impl Drop for ReconcileMeasurer {
205    fn drop(&mut self) {
206        let duration = self.start.elapsed().as_secs_f64();
207        self.metric.record(
208            duration,
209            &[KeyValue::new("controller", self.controller.clone())],
210        );
211    }
212}
213
214#[derive(Clone, Debug)]
215pub enum Action {
216    Apply,
217    Delete,
218}
219
220impl Action {
221    pub fn as_str(&self) -> &'static str {
222        match self {
223            Action::Apply => "apply",
224            Action::Delete => "delete",
225        }
226    }
227}