kaniop_operator/controller/
mod.rs1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use crate::prometheus_exporter;
9
10use kaniop_k8s_util::error::{Error, Result};
11
12use kaniop_k8s_util::types::short_type_name;
13
14use std::fmt::Debug;
15use std::sync::Arc;
16
17use futures::channel::mpsc;
18use futures::future::BoxFuture;
19use futures::{FutureExt, StreamExt};
20use k8s_openapi::api::core::v1::Namespace;
21use kube::Resource;
22use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
23use kube::client::Client;
24use kube::runtime::controller::Action;
25use kube::runtime::events::Recorder;
26use kube::runtime::reflector::store::Writer;
27use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
28use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
29use serde::de::DeserializeOwned;
30use tokio::sync::RwLock;
31use tokio::time::Duration;
32use tracing::{debug, error, info, trace};
33
34use std::sync::OnceLock;
35
36const DEFAULT_IDM_RECONCILE_INTERVAL_SECS: u64 = 60;
37const DEFAULT_CLUSTER_DOMAIN: &str = "cluster.local";
38const DEFAULT_BACKUP_DISCOVERY_SCAN_INTERVAL_SECS: u64 = 300;
39const DEFAULT_BACKUP_DISCOVERY_STALE_SECS: u64 = 900;
40
41static IDM_RECONCILE_INTERVAL: OnceLock<Duration> = OnceLock::new();
42static CLUSTER_DOMAIN: OnceLock<String> = OnceLock::new();
43static BACKUP_DISCOVERY_SCAN_INTERVAL: OnceLock<Duration> = OnceLock::new();
44static BACKUP_DISCOVERY_STALE_THRESHOLD: OnceLock<Duration> = OnceLock::new();
45
46pub fn set_idm_reconcile_interval(duration: Duration) {
47 let _ = IDM_RECONCILE_INTERVAL.set(duration);
48}
49
50pub fn idm_reconcile_interval() -> Duration {
51 *IDM_RECONCILE_INTERVAL.get_or_init(|| Duration::from_secs(DEFAULT_IDM_RECONCILE_INTERVAL_SECS))
52}
53
54pub fn set_cluster_domain(domain: String) {
55 let _ = CLUSTER_DOMAIN.set(domain);
56}
57
58pub fn cluster_domain() -> &'static str {
59 CLUSTER_DOMAIN.get_or_init(|| DEFAULT_CLUSTER_DOMAIN.to_string())
60}
61
62pub fn set_backup_discovery_scan_interval(duration: Duration) {
63 let _ = BACKUP_DISCOVERY_SCAN_INTERVAL.set(duration);
64}
65
66pub fn backup_discovery_scan_interval() -> Duration {
67 *BACKUP_DISCOVERY_SCAN_INTERVAL
68 .get_or_init(|| Duration::from_secs(DEFAULT_BACKUP_DISCOVERY_SCAN_INTERVAL_SECS))
69}
70
71pub fn set_backup_discovery_stale_threshold(duration: Duration) {
72 let _ = BACKUP_DISCOVERY_STALE_THRESHOLD.set(duration);
73}
74
75pub fn backup_discovery_stale_threshold() -> Duration {
76 *BACKUP_DISCOVERY_STALE_THRESHOLD
77 .get_or_init(|| Duration::from_secs(DEFAULT_BACKUP_DISCOVERY_STALE_SECS))
78}
79pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
80pub const RELOAD_BUFFER_SIZE: usize = 16;
81pub const NAME_LABEL: &str = "app.kubernetes.io/name";
82pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
83pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
84
85pub type ControllerId = &'static str;
86
87#[derive(Clone)]
90pub struct State {
91 metrics: Arc<metrics::Metrics>,
93 idm_clients: Arc<RwLock<KanidmClients>>,
95 system_clients: Arc<RwLock<KanidmClients>>,
98 pub namespace_store: Store<Namespace>,
100 pub kanidm_store: Store<Kanidm>,
102 pub client: Option<Client>,
104}
105
106pub struct ResourceReflector<K>
108where
109 K: Resource + Lookup + Clone + 'static,
110 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
111{
112 pub store: Store<K>,
113 pub writer: Writer<K>,
114 pub subscriber: ReflectHandle<K>,
115}
116
117impl State {
119 pub fn new(
120 metrics: metrics::Metrics,
121 namespace_store: Store<Namespace>,
122 kanidm_store: Store<Kanidm>,
123 client: Option<Client>,
124 ) -> Self {
125 Self {
126 metrics: Arc::new(metrics),
127 idm_clients: Arc::default(),
128 system_clients: Arc::default(),
129 namespace_store,
130 kanidm_store,
131 client,
132 }
133 }
134
135 pub fn metrics(&self) -> Result<String> {
137 prometheus_exporter::format_prometheus_metrics("kaniop").map_err(|e| {
138 Error::FormattingError(format!("failed to export metrics: {}", e), std::fmt::Error)
139 })
140 }
141
142 pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
144 where
145 K: Resource + Lookup + Clone + 'static,
146 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
147 {
148 Context::new(
149 controller_id,
150 client.clone(),
151 self.metrics
152 .controllers
153 .get(controller_id)
154 .expect("all CONTROLLER_IDs have to be registered")
155 .clone(),
156 Recorder::new(client.clone(), controller_id.into()),
157 self.idm_clients.clone(),
158 self.system_clients.clone(),
159 self.namespace_store.clone(),
160 self.kanidm_store.clone(),
161 )
162 }
163}
164
165pub async fn check_api_queryable<K>(client: Client) -> Api<K>
166where
167 K: Resource + Clone + DeserializeOwned + Debug,
168 <K as Resource>::DynamicType: Default,
169{
170 let api = Api::<K>::all(client.clone());
171 if let Err(e) = api.list(&ListParams::default().limit(1)).await {
172 error!(
173 "{} is not queryable; {e:?}. Check controller permissions",
174 short_type_name::<K>().unwrap_or("Unknown resource"),
175 );
176 std::process::exit(1);
177 }
178 api
179}
180
181pub async fn check_api_queryable_optional<K>(client: Client) -> Option<Api<K>>
182where
183 K: Resource + Clone + DeserializeOwned + Debug,
184 <K as Resource>::DynamicType: Default,
185{
186 let api = Api::<K>::all(client.clone());
187 if let Err(e) = api.list(&ListParams::default().limit(1)).await {
188 info!(
189 "{} is not queryable (optional resource); {e:?}. Skipping optional resource support.",
190 short_type_name::<K>().unwrap_or("Unknown resource"),
191 );
192 None
193 } else {
194 Some(api)
195 }
196}
197
198pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
199where
200 K: Resource + Lookup + Clone + 'static,
201 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
202 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
203{
204 let (store, writer) = reflector::store_shared::<K>(buffer_size);
205 let subscriber = writer
206 .subscribe()
207 .expect("subscribers can only be created from shared stores");
208
209 ResourceReflector {
210 store,
211 writer,
212 subscriber,
213 }
214}
215
216fn create_generic_watcher<K, W, T, S, StreamT>(
217 api: Api<K>,
218 writer: Writer<W>,
219 reload_tx: mpsc::Sender<()>,
220 controller_id: ControllerId,
221 ctx: Arc<Context<T>>,
222 stream_fn: S,
223) -> BoxFuture<'static, ()>
224where
225 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
226 W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
227 <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
228 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
229 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
230 <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
231 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
232 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
233 S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
234 StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
235 + Send
236 + 'static,
237{
238 let resource_name = short_type_name::<K>().unwrap_or("Unknown");
239
240 stream_fn(
241 api,
242 watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
243 )
244 .default_backoff()
245 .reflect_shared(writer)
246 .for_each(move |res| {
247 let mut reload_tx_clone = reload_tx.clone();
248 let ctx = ctx.clone();
249 async move {
250 match res {
251 Ok(event) => {
252 trace!(?event, "watched event");
253 match event {
254 watcher::Event::Delete(d) => {
255 debug!(
256 namespace = ResourceExt::namespace(&d).unwrap(),
257 name = d.name_any(),
258 "delete event for {resource_name} trigger reconcile"
259 );
260
261 let _ignore_errors = reload_tx_clone
265 .try_send(())
266 .map_err(|e| error!(%e, "failed to trigger reconcile on delete"));
267 ctx.metrics
268 .triggered_inc(metrics::Action::Delete, resource_name);
269 }
270 watcher::Event::Apply(d) => {
271 debug!(
272 namespace = ResourceExt::namespace(&d).unwrap(),
273 name = d.name_any(),
274 "apply event for {resource_name} trigger reconcile"
275 );
276 ctx.metrics
277 .triggered_inc(metrics::Action::Apply, resource_name);
278 }
279 _ => {}
280 }
281 }
282 Err(e) => {
283 error!(%e, "unexpected error when watching {resource_name}");
284 ctx.metrics.watch_operations_failed_inc();
285 }
286 }
287 }
288 })
289 .boxed()
290}
291
292pub fn create_watcher<K, T>(
293 api: Api<K>,
294 writer: Writer<K>,
295 reload_tx: mpsc::Sender<()>,
296 controller_id: ControllerId,
297 ctx: Arc<Context<T>>,
298) -> BoxFuture<'static, ()>
299where
300 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
301 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
302 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
303 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
304 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
305{
306 create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
307}
308
309pub fn create_metadata_watcher<K, T>(
310 api: Api<K>,
311 writer: Writer<PartialObjectMeta<K>>,
312 reload_tx: mpsc::Sender<()>,
313 controller_id: ControllerId,
314 ctx: Arc<Context<T>>,
315) -> BoxFuture<'static, ()>
316where
317 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
318 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
319 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
320 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
321 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
322{
323 create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
324 api,
325 writer,
326 reload_tx,
327 controller_id,
328 ctx,
329 metadata_watcher,
330 )
331}
332
333pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
334where
335 K: Resource + Lookup + Clone + 'static,
336 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
337{
338 tracing::warn!(
339 "error_policy called unexpectedly - all controllers should use backoff_reconciler! macro"
340 );
341 Action::requeue(Duration::from_secs(300))
342}
343
344#[macro_export]
345macro_rules! backoff_reconciler {
346 ($inner_reconciler:ident) => {
347 |obj, ctx| async move {
348 use $crate::controller::context::BackoffContext;
349 let _active_guard = ctx.metrics().active_reconcile_inc();
350 match $inner_reconciler(obj.clone(), ctx.clone()).await {
351 Ok((action, changed)) => {
352 ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
353 .await;
354 if changed {
355 ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_CHANGED);
356 } else {
357 ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_UNCHANGED);
358 }
359 Ok(action)
360 }
361 Err(error) => {
362 let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
364 let name = kube::ResourceExt::name_any(obj.as_ref());
365 tracing::error!(%namespace, %name, %error, "failed reconciliation");
366 ctx.metrics().reconcile_failure_inc();
367 ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_ERROR);
368 let backoff_duration = ctx
369 .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
370 .await;
371 tracing::trace!(%namespace, %name, "backoff duration: {backoff_duration:?}");
372 Ok(kube::runtime::controller::Action::requeue(backoff_duration))
373 }
374 }
375 }
376 };
377}