1use crate::controller::ControllerId;
2use crate::error::Error;
3
4use std::collections::HashMap;
5use std::sync::Arc;
6use std::time::SystemTime;
7
8use opentelemetry::trace::TraceId;
9use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue};
10use prometheus_client::metrics::{
11 counter::Counter, exemplar::HistogramWithExemplars, family::Family, gauge::Gauge,
12};
13use prometheus_client::registry::{Registry, Unit};
14use tokio::time::Instant;
15
16#[derive(Clone)]
17pub struct Metrics {
18 pub controllers: HashMap<ControllerId, Arc<ControllerMetrics>>,
19 pub registry: Arc<Registry>,
20}
21
22impl Metrics {
23 pub fn new(mut registry: Registry, controller_names: &[&'static str]) -> Self {
24 let controllers = controller_names
25 .iter()
26 .map(|&id| {
27 (
28 id,
29 Arc::new(ControllerMetrics::new(id).register(&mut registry)),
30 )
31 })
32 .collect::<HashMap<ControllerId, Arc<ControllerMetrics>>>();
33
34 Self {
35 registry: Arc::new(registry),
36 controllers,
37 }
38 }
39}
40
41#[derive(Clone, Default)]
42pub struct ControllerMetrics {
43 controller: String,
44 pub reconcile: ReconcileMetrics,
45 pub spec_replicas: Family<ResourceLabels, Gauge>,
46 pub status_update_errors: Family<ControllerLabels, Counter>,
47 pub triggered: Family<TriggeredLabels, Counter>,
48 pub watch_operations_failed: Family<ControllerLabels, Counter>,
49 pub ready: Family<ControllerLabels, Gauge>,
50}
51
52impl ControllerMetrics {
53 pub fn new(controller: &str) -> Self {
54 Self {
55 controller: controller.to_string(),
56 ..Default::default()
57 }
58 }
59
60 pub fn register(self, r: &mut Registry) -> Self {
62 r.register(
63 "reconcile_operations",
64 "Total number of reconcile operations",
65 self.reconcile.operations.clone(),
66 );
67 r.register(
68 "reconcile_failures",
69 "Number of errors that occurred during reconcile operations",
70 self.reconcile.failures.clone(),
71 );
72 r.register_with_unit(
73 "reconcile_duration",
74 "Histogram of reconcile operations",
75 Unit::Seconds,
76 self.reconcile.duration.clone(),
77 );
78 r.register(
79 "reconcile_deploy_delete_create",
80 "Number of times that reconciling a deployment required deleting and re-creating it",
81 self.reconcile.deploy_delete_create.clone(),
82 );
83 r.register(
84 "spec_replicas",
85 "Number of expected replicas for the object",
86 self.spec_replicas.clone(),
87 );
88 r.register(
89 "status_update_errors",
90 "Number of errors that occurred during update operations to status subresources",
91 self.status_update_errors.clone(),
92 );
93 r.register(
94 "triggered",
95 "Number of times a Kubernetes object applied or delete event triggered to reconcile an object",
96 self.triggered.clone(),
97 );
98 r.register(
99 "watch_operations_failed",
100 "Total number of watch operations that failed",
101 self.watch_operations_failed.clone(),
102 );
103 r.register(
104 "ready",
105 "1 when the controller is ready to reconcile resources, 0 otherwise",
106 self.ready.clone(),
107 );
108 self
109 }
110
111 pub fn reconcile_failure_inc(&self) {
112 self.reconcile
113 .failures
114 .get_or_create(&ControllerLabels {
115 controller: self.controller.clone(),
116 })
117 .inc();
118 }
119
120 pub fn reconcile_count_and_measure(&self, trace_id: &TraceId) -> ReconcileMeasurer {
121 let controller_labels = ControllerLabels {
122 controller: self.controller.clone(),
123 };
124 self.reconcile
125 .operations
126 .get_or_create(&controller_labels)
127 .inc();
128 ReconcileMeasurer {
129 start: Instant::now(),
130 labels: trace_id.try_into().ok(),
131 metric: self
132 .reconcile
133 .duration
134 .get_or_create(&controller_labels)
135 .clone(),
136 }
137 }
138
139 pub fn reconcile_deploy_delete_create_inc(&self) {
140 let controller_labels = ControllerLabels {
141 controller: self.controller.clone(),
142 };
143 self.reconcile
144 .deploy_delete_create
145 .get_or_create(&controller_labels)
146 .inc();
147 }
148
149 pub fn spec_replicas_set(&self, namespace: &str, name: &str, replicas: i32) {
150 let resource_labels = ResourceLabels {
151 controller: self.controller.clone(),
152 namespace: namespace.to_string(),
153 name: name.to_string(),
154 };
155 self.spec_replicas
156 .get_or_create(&resource_labels)
157 .set(replicas as i64);
158 }
159
160 pub fn status_update_errors_inc(&self) {
161 let controller_labels = ControllerLabels {
162 controller: self.controller.clone(),
163 };
164 self.status_update_errors
165 .get_or_create(&controller_labels)
166 .inc();
167 }
168
169 pub fn triggered_inc(&self, action: Action, triggered_by: &str) {
170 let triggered_labels = TriggeredLabels {
171 controller: self.controller.clone(),
172 action,
173 triggered_by: triggered_by.to_string(),
174 };
175 self.triggered.get_or_create(&triggered_labels).inc();
176 }
177
178 pub fn watch_operations_failed_inc(&self) {
179 let controller_labels = ControllerLabels {
180 controller: self.controller.clone(),
181 };
182 self.watch_operations_failed
183 .get_or_create(&controller_labels)
184 .inc();
185 }
186
187 pub fn ready_set(&self, status: i64) {
188 let controller_labels = ControllerLabels {
189 controller: self.controller.clone(),
190 };
191 self.ready.get_or_create(&controller_labels).set(status);
192 }
193}
194
195#[derive(Clone)]
196pub struct ReconcileMetrics {
197 pub operations: Family<ControllerLabels, Counter>,
198 pub failures: Family<ControllerLabels, Counter>,
199 pub duration: Family<ControllerLabels, HistogramWithExemplars<TraceLabel>>,
200 pub deploy_delete_create: Family<ControllerLabels, Counter>,
201}
202
203impl Default for ReconcileMetrics {
204 fn default() -> Self {
205 Self {
206 operations: Default::default(),
207 failures: Default::default(),
208 duration:
209 Family::<ControllerLabels, HistogramWithExemplars<TraceLabel>>::new_with_constructor(
210 || HistogramWithExemplars::new([0.1, 0.5, 1., 5., 10.].into_iter()),
211 ),
212 deploy_delete_create: Default::default(),
213 }
214 }
215}
216
217pub struct ReconcileMeasurer {
221 start: Instant,
222 labels: Option<TraceLabel>,
223 metric: HistogramWithExemplars<TraceLabel>,
224}
225
226impl Drop for ReconcileMeasurer {
227 fn drop(&mut self) {
228 let duration = self.start.elapsed().as_secs_f64();
229 let labels = self.labels.take();
230 self.metric
231 .observe(duration, labels, Some(SystemTime::now()));
232 }
233}
234
235#[derive(Clone, Hash, PartialEq, Eq, EncodeLabelSet, Debug, Default)]
236pub struct TraceLabel {
237 pub id: String,
238}
239impl TryFrom<&TraceId> for TraceLabel {
240 type Error = Error;
241
242 fn try_from(id: &TraceId) -> Result<TraceLabel, Self::Error> {
243 if std::matches!(id, &TraceId::INVALID) {
244 Err(Error::InvalidTraceId)
245 } else {
246 let trace_id = id.to_string();
247 Ok(Self { id: trace_id })
248 }
249 }
250}
251
252#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
253pub struct ControllerLabels {
254 pub controller: String,
255}
256
257#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
258pub struct ResourceLabels {
259 pub controller: String,
260 pub namespace: String,
261 pub name: String,
262}
263
264#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
265pub struct TriggeredLabels {
266 pub controller: String,
267 pub action: Action,
268 pub triggered_by: String,
269}
270
271#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelValue)]
272pub enum Action {
273 Apply,
274 Delete,
275}