kaniop_operator/
metrics.rs

1use crate::controller::ControllerId;
2use crate::error::Error;
3
4use std::collections::HashMap;
5use std::sync::Arc;
6use std::time::SystemTime;
7
8use opentelemetry::trace::TraceId;
9use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue};
10use prometheus_client::metrics::{
11    counter::Counter, exemplar::HistogramWithExemplars, family::Family, gauge::Gauge,
12};
13use prometheus_client::registry::{Registry, Unit};
14use tokio::time::Instant;
15
16#[derive(Clone)]
17pub struct Metrics {
18    pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
19    pub registry: Arc<Registry>,
20}
21
22impl Metrics {
23    pub fn new(mut registry: Registry, controller_names: &[&'static str]) -> Self {
24        let controllers = controller_names
25            .iter()
26            .map(|&id| {
27                (
28                    id,
29                    Arc::new(ControllerMetrics::new(id).register(&mut registry)),
30                )
31            })
32            .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
33
34        Self {
35            registry: Arc::new(registry),
36            controllers,
37        }
38    }
39}
40
41#[derive(Clone, Default)]
42pub struct ControllerMetrics {
43    controller: String,
44    pub reconcile: ReconcileMetrics,
45    pub spec_replicas: Family<ResourceLabels, Gauge>,
46    pub status_update_errors: Family<ControllerLabels, Counter>,
47    pub triggered: Family<TriggeredLabels, Counter>,
48    pub watch_operations_failed: Family<ControllerLabels, Counter>,
49    pub ready: Family<ControllerLabels, Gauge>,
50}
51
52impl ControllerMetrics {
53    pub fn new(controller: &str) -> Self {
54        Self {
55            controller: controller.to_string(),
56            ..Default::default()
57        }
58    }
59
60    /// Register API metrics to start tracking them.
61    pub fn register(self, r: &mut Registry) -> Self {
62        r.register(
63            "reconcile_operations",
64            "Total number of reconcile operations",
65            self.reconcile.operations.clone(),
66        );
67        r.register(
68            "reconcile_failures",
69            "Number of errors that occurred during reconcile operations",
70            self.reconcile.failures.clone(),
71        );
72        r.register_with_unit(
73            "reconcile_duration",
74            "Histogram of reconcile operations",
75            Unit::Seconds,
76            self.reconcile.duration.clone(),
77        );
78        r.register(
79            "reconcile_deploy_delete_create",
80            "Number of times that reconciling a deployment required deleting and re-creating it",
81            self.reconcile.deploy_delete_create.clone(),
82        );
83        r.register(
84            "spec_replicas",
85            "Number of expected replicas for the object",
86            self.spec_replicas.clone(),
87        );
88        r.register(
89            "status_update_errors",
90            "Number of errors that occurred during update operations to status subresources",
91            self.status_update_errors.clone(),
92        );
93        r.register(
94            "triggered",
95            "Number of times a Kubernetes object applied or delete event triggered to reconcile an object",
96            self.triggered.clone(),
97        );
98        r.register(
99            "watch_operations_failed",
100            "Total number of watch operations that failed",
101            self.watch_operations_failed.clone(),
102        );
103        r.register(
104            "ready",
105            "1 when the controller is ready to reconcile resources, 0 otherwise",
106            self.ready.clone(),
107        );
108        self
109    }
110
111    pub fn reconcile_failure_inc(&self) {
112        self.reconcile
113            .failures
114            .get_or_create(&ControllerLabels {
115                controller: self.controller.clone(),
116            })
117            .inc();
118    }
119
120    pub fn reconcile_count_and_measure(&self, trace_id: &TraceId) -> ReconcileMeasurer {
121        let controller_labels = ControllerLabels {
122            controller: self.controller.clone(),
123        };
124        self.reconcile
125            .operations
126            .get_or_create(&controller_labels)
127            .inc();
128        ReconcileMeasurer {
129            start: Instant::now(),
130            labels: trace_id.try_into().ok(),
131            metric: self
132                .reconcile
133                .duration
134                .get_or_create(&controller_labels)
135                .clone(),
136        }
137    }
138
139    pub fn reconcile_deploy_delete_create_inc(&self) {
140        let controller_labels = ControllerLabels {
141            controller: self.controller.clone(),
142        };
143        self.reconcile
144            .deploy_delete_create
145            .get_or_create(&controller_labels)
146            .inc();
147    }
148
149    pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
150        let resource_labels = ResourceLabels {
151            controller: self.controller.clone(),
152            namespace: namespace.to_string(),
153            name: name.to_string(),
154        };
155        self.spec_replicas
156            .get_or_create(&resource_labels)
157            .set(replicas as i64);
158    }
159
160    pub fn status_update_errors_inc(&self) {
161        let controller_labels = ControllerLabels {
162            controller: self.controller.clone(),
163        };
164        self.status_update_errors
165            .get_or_create(&controller_labels)
166            .inc();
167    }
168
169    pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
170        let triggered_labels = TriggeredLabels {
171            controller: self.controller.clone(),
172            action,
173            triggered_by: triggered_by.to_string(),
174        };
175        self.triggered.get_or_create(&triggered_labels).inc();
176    }
177
178    pub fn watch_operations_failed_inc(&self) {
179        let controller_labels = ControllerLabels {
180            controller: self.controller.clone(),
181        };
182        self.watch_operations_failed
183            .get_or_create(&controller_labels)
184            .inc();
185    }
186
187    pub fn ready_set(&self, status: i64) {
188        let controller_labels = ControllerLabels {
189            controller: self.controller.clone(),
190        };
191        self.ready.get_or_create(&controller_labels).set(status);
192    }
193}
194
195#[derive(Clone)]
196pub struct ReconcileMetrics {
197    pub operations: Family<ControllerLabels, Counter>,
198    pub failures: Family<ControllerLabels, Counter>,
199    pub duration: Family<ControllerLabels, HistogramWithExemplars<TraceLabel>>,
200    pub deploy_delete_create: Family<ControllerLabels, Counter>,
201}
202
203impl Default for ReconcileMetrics {
204    fn default() -> Self {
205        Self {
206            operations: Default::default(),
207            failures: Default::default(),
208            duration:
209                Family::<ControllerLabels, HistogramWithExemplars<TraceLabel>>::new_with_constructor(
210                    || HistogramWithExemplars::new([0.1, 0.5, 1., 5., 10.].into_iter()),
211                ),
212            deploy_delete_create: Default::default(),
213        }
214    }
215}
216
217/// Smart function duration measurer
218///
219/// Relies on Drop to calculate duration and register the observation in the histogram
220pub struct ReconcileMeasurer {
221    start: Instant,
222    labels: Option<TraceLabel>,
223    metric: HistogramWithExemplars<TraceLabel>,
224}
225
226impl Drop for ReconcileMeasurer {
227    fn drop(&mut self) {
228        let duration = self.start.elapsed().as_secs_f64();
229        let labels = self.labels.take();
230        self.metric
231            .observe(duration, labels, Some(SystemTime::now()));
232    }
233}
234
235#[derive(Clone, Hash, PartialEq, Eq, EncodeLabelSet, Debug, Default)]
236pub struct TraceLabel {
237    pub id: String,
238}
239impl TryFrom<&TraceId> for TraceLabel {
240    type Error = Error;
241
242    fn try_from(id: &TraceId) -> Result<TraceLabel, Self::Error> {
243        if std::matches!(id, &TraceId::INVALID) {
244            Err(Error::InvalidTraceId)
245        } else {
246            let trace_id = id.to_string();
247            Ok(Self { id: trace_id })
248        }
249    }
250}
251
252#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
253pub struct ControllerLabels {
254    pub controller: String,
255}
256
257#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
258pub struct ResourceLabels {
259    pub controller: String,
260    pub namespace: String,
261    pub name: String,
262}
263
264#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
265pub struct TriggeredLabels {
266    pub controller: String,
267    pub action: Action,
268    pub triggered_by: String,
269}
270
271#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelValue)]
272pub enum Action {
273    Apply,
274    Delete,
275}