1use crate::controller::ControllerId;
2
3use std::collections::HashMap;
4use std::sync::Arc;
5
6use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter};
7use opentelemetry::{KeyValue, trace::TraceId};
8use tokio::time::Instant;
9use tracing::debug;
10
11#[derive(Clone)]
12pub struct Metrics {
13 pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
14}
15
16impl Metrics {
17 pub fn new(meter: &Meter, controller_names: &[&'static str]) -> Self {
18 debug!(
19 "Initializing operator metrics for controllers: {:?}",
20 controller_names
21 );
22 let controllers = controller_names
23 .iter()
24 .map(|&id| (id, Arc::new(ControllerMetrics::new(id, meter))))
25 .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
26
27 debug!("Operator metrics initialized");
28 Self { controllers }
29 }
30}
31
32#[derive(Clone)]
33pub struct ControllerMetrics {
34 controller: String,
35 pub reconcile: ReconcileMetrics,
36 spec_replicas: Gauge<i64>,
37 status_update_errors: Counter<u64>,
38 triggered: Counter<u64>,
39 watch_operations_failed: Counter<u64>,
40 ready: Gauge<i64>,
41}
42
43impl ControllerMetrics {
44 pub fn new(controller: &str, meter: &Meter) -> Self {
45 let reconcile = ReconcileMetrics::new(meter);
46
47 let spec_replicas = meter
48 .i64_gauge("spec_replicas")
49 .with_description("Number of expected replicas for the object")
50 .build();
51
52 let status_update_errors = meter
53 .u64_counter("status_update_errors")
54 .with_description(
55 "Number of errors that occurred during update operations to status subresources",
56 )
57 .build();
58
59 let triggered = meter
60 .u64_counter("triggered")
61 .with_description("Number of times a Kubernetes object applied or delete event triggered to reconcile an object")
62 .build();
63
64 let watch_operations_failed = meter
65 .u64_counter("watch_operations_failed")
66 .with_description("Total number of watch operations that failed")
67 .build();
68
69 let ready = meter
70 .i64_gauge("ready")
71 .with_description("1 when the controller is ready to reconcile resources, 0 otherwise")
72 .build();
73
74 Self {
75 controller: controller.to_string(),
76 reconcile,
77 spec_replicas,
78 status_update_errors,
79 triggered,
80 watch_operations_failed,
81 ready,
82 }
83 }
84
85 pub fn reconcile_failure_inc(&self) {
86 self.reconcile
87 .failures
88 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
89 }
90
91 pub fn reconcile_count_and_measure(&self, _trace_id: &TraceId) -> ReconcileMeasurer {
92 self.reconcile
93 .operations
94 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
95 ReconcileMeasurer {
96 start: Instant::now(),
97 controller: self.controller.clone(),
98 metric: self.reconcile.duration.clone(),
99 }
100 }
101
102 pub fn reconcile_deploy_delete_create_inc(&self, resource_kind: &str, reason: &str) {
103 self.reconcile.deploy_delete_create.add(
104 1,
105 &[
106 KeyValue::new("controller", self.controller.clone()),
107 KeyValue::new("resource_kind", resource_kind.to_string()),
108 KeyValue::new("reason", reason.to_string()),
109 ],
110 );
111 }
112
113 pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
114 self.spec_replicas.record(
115 replicas as i64,
116 &[
117 KeyValue::new("controller", self.controller.clone()),
118 KeyValue::new("namespace", namespace.to_string()),
119 KeyValue::new("name", name.to_string()),
120 ],
121 );
122 }
123
124 pub fn status_update_errors_inc(&self) {
125 self.status_update_errors
126 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
127 }
128
129 pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
130 self.triggered.add(
131 1,
132 &[
133 KeyValue::new("controller", self.controller.clone()),
134 KeyValue::new("action", action.as_str()),
135 KeyValue::new("triggered_by", triggered_by.to_string()),
136 ],
137 );
138 }
139
140 pub fn watch_operations_failed_inc(&self) {
141 self.watch_operations_failed
142 .add(1, &[KeyValue::new("controller", self.controller.clone())]);
143 }
144
145 pub fn ready_set(&self, status: i64) {
146 self.ready.record(
147 status,
148 &[KeyValue::new("controller", self.controller.clone())],
149 );
150 }
151}
152
153#[derive(Clone)]
154pub struct ReconcileMetrics {
155 pub operations: Counter<u64>,
156 pub failures: Counter<u64>,
157 pub duration: Histogram<f64>,
158 pub deploy_delete_create: Counter<u64>,
159}
160
161impl ReconcileMetrics {
162 pub fn new(meter: &Meter) -> Self {
163 debug!("Initializing reconcile metrics");
164 let operations = meter
165 .u64_counter("reconcile_operations")
166 .with_description("Total number of reconcile operations")
167 .build();
168
169 let failures = meter
170 .u64_counter("reconcile_failures")
171 .with_description("Number of errors that occurred during reconcile operations")
172 .build();
173
174 let duration = meter
175 .f64_histogram("reconcile_duration_seconds")
176 .with_description("Histogram of reconcile operations")
177 .with_boundaries(vec![0.1, 0.5, 1.0, 5.0, 10.0])
178 .build();
179
180 let deploy_delete_create = meter
181 .u64_counter("reconcile_deploy_delete_create")
182 .with_description("Number of explicit resource delete/recreate operations (not limited to deployments)")
183 .build();
184
185 debug!("Reconcile metrics initialized");
186 Self {
187 operations,
188 failures,
189 duration,
190 deploy_delete_create,
191 }
192 }
193}
194
195pub struct ReconcileMeasurer {
199 start: Instant,
200 controller: String,
201 metric: Histogram<f64>,
202}
203
204impl Drop for ReconcileMeasurer {
205 fn drop(&mut self) {
206 let duration = self.start.elapsed().as_secs_f64();
207 self.metric.record(
208 duration,
209 &[KeyValue::new("controller", self.controller.clone())],
210 );
211 }
212}
213
214#[derive(Clone, Debug)]
215pub enum Action {
216 Apply,
217 Delete,
218}
219
220impl Action {
221 pub fn as_str(&self) -> &'static str {
222 match self {
223 Action::Apply => "apply",
224 Action::Delete => "delete",
225 }
226 }
227}