1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5
6use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
7use opentelemetry::{KeyValue, trace::TraceId};
8use tokio::time::Instant;
9use tracing::debug;
10
11#[derive(Clone)]
12pub struct Metrics {
13 pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
14}
15
16impl Metrics {
17 pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
18 debug!(
19 "Initializing operator metrics for controllers: {:?}",
20 controller_names
21 );
22 let controllers = controller_names
23 .iter()
24 .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
25 .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
26
27 debug!("Operator metrics initialized");
28 Self { controllers }
29 }
30}
31
32#[derive(Clone)]
33pub struct ControllerMetrics {
34 controller: String,
35 pub reconcile: ReconcileMetrics,
36 spec_replicas: Gauge<i64>,
37 status_update_errors: Counter<u64>,
38 triggered: Counter<u64>,
39 watch_operations_failed: Counter<u64>,
40 ready: Gauge<i64>,
41}
42
43impl ControllerMetrics {
44 pub fn new(controller: &str, meter: &Meter) -> Self {
45 let reconcile = ReconcileMetrics::new(meter);
46
47 let spec_replicas = meter
48 .i64_gauge("spec_replicas")
49 .with_description("Number of expected replicas for the object")
50 .build();
51
52 let status_update_errors = meter
53 .u64_counter("status_update_errors")
54 .with_description(
55 "Number of errors that occurred during update operations to status subresources",
56 )
57 .build();
58
59 let triggered = meter
60 .u64_counter("triggered")
61 .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
62 .build();
63
64 let watch_operations_failed = meter
65 .u64_counter("watch_operations_failed")
66 .with_description("Total number of watch operations that failed")
67 .build();
68
69 let ready = meter
70 .i64_gauge("ready")
71 .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
72 .build();
73
74 Self {
75 controller: controller.to_string(),
76 reconcile,
77 spec_replicas,
78 status_update_errors,
79 triggered,
80 watch_operations_failed,
81 ready,
82 }
83 }
84
85 pub fn reconcile_failure_inc(&self) {
86 self.reconcile
87 .failures
88 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
89 }
90
91 pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
92 self.reconcile
93 .operations
94 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
95 ReconcileMeasurer {
96 start: Instant::now(),
97 controller: self.controller.clone(),
98 metric: self.reconcile.duration.clone(),
99 }
100 }
101
102 pub fn reconcile_deploy_delete_create_inc(&self) {
103 self.reconcile
104 .deploy_delete_create
105 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
106 }
107
108 pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
109 self.spec_replicas.record(
110 replicas as i64,
111 &[
112 KeyValue::new("controller", self.controller.clone()),
113 KeyValue::new("namespace", namespace.to_string()),
114 KeyValue::new("name", name.to_string()),
115 ],
116 );
117 }
118
119 pub fn status_update_errors_inc(&self) {
120 self.status_update_errors
121 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
122 }
123
124 pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
125 self.triggered.add(
126 1,
127 &[
128 KeyValue::new("controller", self.controller.clone()),
129 KeyValue::new("action", action.as_str()),
130 KeyValue::new("triggered_by", triggered_by.to_string()),
131 ],
132 );
133 }
134
135 pub fn watch_operations_failed_inc(&self) {
136 self.watch_operations_failed
137 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
138 }
139
140 pub fn ready_set(&self, status: i64) {
141 self.ready.record(
142 status,
143 &[KeyValue::new("controller", self.controller.clone())],
144 );
145 }
146}
147
148#[derive(Clone)]
149pub struct ReconcileMetrics {
150 pub operations: Counter<u64>,
151 pub failures: Counter<u64>,
152 pub duration: Histogram<f64>,
153 pub deploy_delete_create: Counter<u64>,
154}
155
156impl ReconcileMetrics {
157 pub fn new(meter: &Meter) -> Self {
158 debug!("Initializing reconcile metrics");
159 let operations = meter
160 .u64_counter("reconcile_operations")
161 .with_description("Total number of reconcile operations")
162 .build();
163
164 let failures = meter
165 .u64_counter("reconcile_failures")
166 .with_description("Number of errors that occurred during reconcile operations")
167 .build();
168
169 let duration = meter
170 .f64_histogram("reconcile_duration_seconds")
171 .with_description("Histogram of reconcile operations")
172 .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
173 .build();
174
175 let deploy_delete_create = meter
176 .u64_counter("reconcile_deploy_delete_create")
177 .with_description("Number of times that reconciling a deployment required deleting and re-creating it")
178 .build();
179
180 debug!("Reconcile metrics initialized");
181 Self {
182 operations,
183 failures,
184 duration,
185 deploy_delete_create,
186 }
187 }
188}
189
190pub struct ReconcileMeasurer {
194 start: Instant,
195 controller: String,
196 metric: Histogram<f64>,
197}
198
199impl Drop for ReconcileMeasurer {
200 fn drop(&mut self) {
201 let duration = self.start.elapsed().as_secs_f64();
202 self.metric.record(
203 duration,
204 &[KeyValue::new("controller", self.controller.clone())],
205 );
206 }
207}
208
209#[derive(Clone, Debug)]
210pub enum Action {
211 Apply,
212 Delete,
213}
214
215impl Action {
216 pub fn as_str(&self) -> &'static str {
217 match self {
218 Action::Apply => "apply",
219 Action::Delete => "delete",
220 }
221 }
222}