kaniop_operator/controller/
mod.rs1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use crate::prometheus_exporter;
9
10use kaniop_k8s_util::error::{Error, Result};
11
12use kaniop_k8s_util::types::short_type_name;
13
14use std::fmt::Debug;
15use std::sync::Arc;
16
17use futures::channel::mpsc;
18use futures::future::BoxFuture;
19use futures::{FutureExt, StreamExt};
20use k8s_openapi::api::core::v1::Namespace;
21use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
22use kube::Resource;
23use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
24use kube::client::Client;
25use kube::runtime::controller::Action;
26use kube::runtime::events::Recorder;
27use kube::runtime::reflector::store::Writer;
28use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
29use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
30use serde::de::DeserializeOwned;
31use tokio::sync::RwLock;
32use tokio::time::Duration;
33use tracing::{debug, error, info, trace};
34
35use std::sync::OnceLock;
36
37const DEFAULT_IDM_RECONCILE_INTERVAL_SECS: u64 = 60;
38const DEFAULT_CLUSTER_DOMAIN: &str = "cluster.local";
39const DEFAULT_BACKUP_DISCOVERY_SCAN_INTERVAL_SECS: u64 = 300;
40const DEFAULT_BACKUP_DISCOVERY_STALE_SECS: u64 = 300;
41
42static IDM_RECONCILE_INTERVAL: OnceLock<Duration> = OnceLock::new();
43static CLUSTER_DOMAIN: OnceLock<String> = OnceLock::new();
44static BACKUP_DISCOVERY_SCAN_INTERVAL: OnceLock<Duration> = OnceLock::new();
45static BACKUP_DISCOVERY_STALE_THRESHOLD: OnceLock<Duration> = OnceLock::new();
46static BACKUP_JOB_VOLUME_SIZE: OnceLock<Quantity> = OnceLock::new();
47
48pub fn set_idm_reconcile_interval(duration: Duration) {
49 let _ = IDM_RECONCILE_INTERVAL.set(duration);
50}
51
52pub fn idm_reconcile_interval() -> Duration {
53 *IDM_RECONCILE_INTERVAL.get_or_init(|| Duration::from_secs(DEFAULT_IDM_RECONCILE_INTERVAL_SECS))
54}
55
56pub fn set_cluster_domain(domain: String) {
57 let _ = CLUSTER_DOMAIN.set(domain);
58}
59
60pub fn cluster_domain() -> &'static str {
61 CLUSTER_DOMAIN.get_or_init(|| DEFAULT_CLUSTER_DOMAIN.to_string())
62}
63
64pub fn set_backup_discovery_scan_interval(duration: Duration) {
65 let _ = BACKUP_DISCOVERY_SCAN_INTERVAL.set(duration);
66}
67
68pub fn backup_discovery_scan_interval() -> Duration {
69 *BACKUP_DISCOVERY_SCAN_INTERVAL
70 .get_or_init(|| Duration::from_secs(DEFAULT_BACKUP_DISCOVERY_SCAN_INTERVAL_SECS))
71}
72
73pub fn set_backup_discovery_stale_threshold(duration: Duration) {
74 let _ = BACKUP_DISCOVERY_STALE_THRESHOLD.set(duration);
75}
76
77pub fn backup_discovery_stale_threshold() -> Duration {
78 *BACKUP_DISCOVERY_STALE_THRESHOLD
79 .get_or_init(|| Duration::from_secs(DEFAULT_BACKUP_DISCOVERY_STALE_SECS))
80}
81
82pub fn set_backup_job_volume_size(quantity: Quantity) {
83 let _ = BACKUP_JOB_VOLUME_SIZE.set(quantity);
84}
85
86pub fn backup_job_volume_size() -> Quantity {
87 BACKUP_JOB_VOLUME_SIZE
88 .get_or_init(|| Quantity("10Gi".to_string()))
89 .clone()
90}
91
92pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
93pub const RELOAD_BUFFER_SIZE: usize = 16;
94pub const NAME_LABEL: &str = "app.kubernetes.io/name";
95pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
96pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
97
98pub type ControllerId = &'static str;
99
100#[derive(Clone)]
103pub struct State {
104 metrics: Arc<metrics::Metrics>,
106 idm_clients: Arc<RwLock<KanidmClients>>,
108 system_clients: Arc<RwLock<KanidmClients>>,
111 pub namespace_store: Store<Namespace>,
113 pub kanidm_store: Store<Kanidm>,
115 pub client: Option<Client>,
117}
118
119pub struct ResourceReflector<K>
121where
122 K: Resource + Lookup + Clone + 'static,
123 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
124{
125 pub store: Store<K>,
126 pub writer: Writer<K>,
127 pub subscriber: ReflectHandle<K>,
128}
129
130impl State {
132 pub fn new(
133 metrics: metrics::Metrics,
134 namespace_store: Store<Namespace>,
135 kanidm_store: Store<Kanidm>,
136 client: Option<Client>,
137 ) -> Self {
138 Self {
139 metrics: Arc::new(metrics),
140 idm_clients: Arc::default(),
141 system_clients: Arc::default(),
142 namespace_store,
143 kanidm_store,
144 client,
145 }
146 }
147
148 pub fn metrics(&self) -> Result<String> {
150 prometheus_exporter::format_prometheus_metrics("kaniop").map_err(|e| {
151 Error::FormattingError(format!("failed to export metrics: {}", e), std::fmt::Error)
152 })
153 }
154
155 pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
157 where
158 K: Resource + Lookup + Clone + 'static,
159 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
160 {
161 Context::new(
162 controller_id,
163 client.clone(),
164 self.metrics
165 .controllers
166 .get(controller_id)
167 .expect("all CONTROLLER_IDs have to be registered")
168 .clone(),
169 Recorder::new(client.clone(), controller_id.into()),
170 self.idm_clients.clone(),
171 self.system_clients.clone(),
172 self.namespace_store.clone(),
173 self.kanidm_store.clone(),
174 )
175 }
176}
177
178pub async fn check_api_queryable<K>(client: Client) -> Api<K>
179where
180 K: Resource + Clone + DeserializeOwned + Debug,
181 <K as Resource>::DynamicType: Default,
182{
183 let api = Api::<K>::all(client.clone());
184 if let Err(e) = api.list(&ListParams::default().limit(1)).await {
185 error!(
186 "{} is not queryable; {e:?}. Check controller permissions",
187 short_type_name::<K>().unwrap_or("Unknown resource"),
188 );
189 std::process::exit(1);
190 }
191 api
192}
193
194pub async fn check_api_queryable_optional<K>(client: Client) -> Option<Api<K>>
195where
196 K: Resource + Clone + DeserializeOwned + Debug,
197 <K as Resource>::DynamicType: Default,
198{
199 let api = Api::<K>::all(client.clone());
200 if let Err(e) = api.list(&ListParams::default().limit(1)).await {
201 info!(
202 "{} is not queryable (optional resource); {e:?}. Skipping optional resource support.",
203 short_type_name::<K>().unwrap_or("Unknown resource"),
204 );
205 None
206 } else {
207 Some(api)
208 }
209}
210
211pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
212where
213 K: Resource + Lookup + Clone + 'static,
214 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
215 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
216{
217 let (store, writer) = reflector::store_shared::<K>(buffer_size);
218 let subscriber = writer
219 .subscribe()
220 .expect("subscribers can only be created from shared stores");
221
222 ResourceReflector {
223 store,
224 writer,
225 subscriber,
226 }
227}
228
229fn create_generic_watcher<K, W, T, S, StreamT>(
230 api: Api<K>,
231 writer: Writer<W>,
232 reload_tx: mpsc::Sender<()>,
233 controller_id: ControllerId,
234 ctx: Arc<Context<T>>,
235 stream_fn: S,
236) -> BoxFuture<'static, ()>
237where
238 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
239 W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
240 <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
241 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
242 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
243 <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
244 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
245 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
246 S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
247 StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
248 + Send
249 + 'static,
250{
251 let resource_name = short_type_name::<K>().unwrap_or("Unknown");
252
253 stream_fn(
254 api,
255 watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
256 )
257 .default_backoff()
258 .reflect_shared(writer)
259 .for_each(move |res| {
260 let mut reload_tx_clone = reload_tx.clone();
261 let ctx = ctx.clone();
262 async move {
263 match res {
264 Ok(event) => {
265 trace!(?event, "watched event");
266 match event {
267 watcher::Event::Delete(d) => {
268 debug!(
269 namespace = ResourceExt::namespace(&d).unwrap(),
270 name = d.name_any(),
271 "delete event for {resource_name} trigger reconcile"
272 );
273
274 let _ignore_errors = reload_tx_clone
278 .try_send(())
279 .map_err(|e| error!(%e, "failed to trigger reconcile on delete"));
280 ctx.metrics
281 .triggered_inc(metrics::Action::Delete, resource_name);
282 }
283 watcher::Event::Apply(d) => {
284 debug!(
285 namespace = ResourceExt::namespace(&d).unwrap(),
286 name = d.name_any(),
287 "apply event for {resource_name} trigger reconcile"
288 );
289 ctx.metrics
290 .triggered_inc(metrics::Action::Apply, resource_name);
291 }
292 _ => {}
293 }
294 }
295 Err(e) => {
296 error!(%e, "unexpected error when watching {resource_name}");
297 ctx.metrics.watch_operations_failed_inc();
298 }
299 }
300 }
301 })
302 .boxed()
303}
304
305pub fn create_watcher<K, T>(
306 api: Api<K>,
307 writer: Writer<K>,
308 reload_tx: mpsc::Sender<()>,
309 controller_id: ControllerId,
310 ctx: Arc<Context<T>>,
311) -> BoxFuture<'static, ()>
312where
313 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
314 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
315 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
316 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
317 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
318{
319 create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
320}
321
322pub fn create_metadata_watcher<K, T>(
323 api: Api<K>,
324 writer: Writer<PartialObjectMeta<K>>,
325 reload_tx: mpsc::Sender<()>,
326 controller_id: ControllerId,
327 ctx: Arc<Context<T>>,
328) -> BoxFuture<'static, ()>
329where
330 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
331 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
332 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
333 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
334 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
335{
336 create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
337 api,
338 writer,
339 reload_tx,
340 controller_id,
341 ctx,
342 metadata_watcher,
343 )
344}
345
346pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
347where
348 K: Resource + Lookup + Clone + 'static,
349 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
350{
351 tracing::warn!(
352 "error_policy called unexpectedly - all controllers should use backoff_reconciler! macro"
353 );
354 Action::requeue(Duration::from_secs(300))
355}
356
357#[macro_export]
358macro_rules! backoff_reconciler {
359 ($inner_reconciler:ident) => {
360 |obj, ctx| async move {
361 use $crate::controller::context::BackoffContext;
362 let _active_guard = ctx.metrics().active_reconcile_inc();
363 match $inner_reconciler(obj.clone(), ctx.clone()).await {
364 Ok((action, changed)) => {
365 ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
366 .await;
367 if changed {
368 ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_CHANGED);
369 } else {
370 ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_UNCHANGED);
371 }
372 Ok(action)
373 }
374 Err(error) => {
375 let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
377 let name = kube::ResourceExt::name_any(obj.as_ref());
378 tracing::error!(%namespace, %name, %error, "failed reconciliation");
379 ctx.metrics().reconcile_failure_inc();
380 ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_ERROR);
381 let backoff_duration = ctx
382 .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
383 .await;
384 tracing::trace!(%namespace, %name, "backoff duration: {backoff_duration:?}");
385 Ok(kube::runtime::controller::Action::requeue(backoff_duration))
386 }
387 }
388 }
389 };
390}