kaniop_operator/controller/
mod.rs1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use crate::prometheus_exporter;
9
10use kaniop_k8s_util::error::{Error, Result};
11
12use kaniop_k8s_util::types::short_type_name;
13
14use std::fmt::Debug;
15use std::sync::Arc;
16
17use futures::channel::mpsc;
18use futures::future::BoxFuture;
19use futures::{FutureExt, StreamExt};
20use k8s_openapi::api::core::v1::Namespace;
21use kube::Resource;
22use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
23use kube::client::Client;
24use kube::runtime::controller::Action;
25use kube::runtime::events::Recorder;
26use kube::runtime::reflector::store::Writer;
27use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
28use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
29use serde::de::DeserializeOwned;
30use tokio::sync::RwLock;
31use tokio::time::Duration;
32use tracing::{debug, error, info, trace};
33
34use std::sync::OnceLock;
35
36const DEFAULT_IDM_RECONCILE_INTERVAL_SECS: u64 = 60;
37
38static IDM_RECONCILE_INTERVAL: OnceLock<Duration> = OnceLock::new();
39
40pub fn set_idm_reconcile_interval(duration: Duration) {
41 let _ = IDM_RECONCILE_INTERVAL.set(duration);
42}
43
44pub fn idm_reconcile_interval() -> Duration {
45 *IDM_RECONCILE_INTERVAL.get_or_init(|| Duration::from_secs(DEFAULT_IDM_RECONCILE_INTERVAL_SECS))
46}
47pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
48pub const RELOAD_BUFFER_SIZE: usize = 16;
49pub const NAME_LABEL: &str = "app.kubernetes.io/name";
50pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
51pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
52
53pub type ControllerId = &'static str;
54
55#[derive(Clone)]
58pub struct State {
59 metrics: Arc<metrics::Metrics>,
61 idm_clients: Arc<RwLock<KanidmClients>>,
63 system_clients: Arc<RwLock<KanidmClients>>,
66 pub namespace_store: Store<Namespace>,
68 pub kanidm_store: Store<Kanidm>,
70 pub client: Option<Client>,
72}
73
74pub struct ResourceReflector<K>
76where
77 K: Resource + Lookup + Clone + 'static,
78 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
79{
80 pub store: Store<K>,
81 pub writer: Writer<K>,
82 pub subscriber: ReflectHandle<K>,
83}
84
85impl State {
87 pub fn new(
88 metrics: metrics::Metrics,
89 namespace_store: Store<Namespace>,
90 kanidm_store: Store<Kanidm>,
91 client: Option<Client>,
92 ) -> Self {
93 Self {
94 metrics: Arc::new(metrics),
95 idm_clients: Arc::default(),
96 system_clients: Arc::default(),
97 namespace_store,
98 kanidm_store,
99 client,
100 }
101 }
102
103 pub fn metrics(&self) -> Result<String> {
105 prometheus_exporter::format_prometheus_metrics("kaniop").map_err(|e| {
106 Error::FormattingError(format!("failed to export metrics: {}", e), std::fmt::Error)
107 })
108 }
109
110 pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
112 where
113 K: Resource + Lookup + Clone + 'static,
114 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
115 {
116 Context::new(
117 controller_id,
118 client.clone(),
119 self.metrics
120 .controllers
121 .get(controller_id)
122 .expect("all CONTROLLER_IDs have to be registered")
123 .clone(),
124 Recorder::new(client.clone(), controller_id.into()),
125 self.idm_clients.clone(),
126 self.system_clients.clone(),
127 self.namespace_store.clone(),
128 self.kanidm_store.clone(),
129 )
130 }
131}
132
133pub async fn check_api_queryable<K>(client: Client) -> Api<K>
134where
135 K: Resource + Clone + DeserializeOwned + Debug,
136 <K as Resource>::DynamicType: Default,
137{
138 let api = Api::<K>::all(client.clone());
139 if let Err(e) = api.list(&ListParams::default().limit(1)).await {
140 error!(
141 "{} is not queryable; {e:?}. Check controller permissions",
142 short_type_name::<K>().unwrap_or("Unknown resource"),
143 );
144 std::process::exit(1);
145 }
146 api
147}
148
149pub async fn check_api_queryable_optional<K>(client: Client) -> Option<Api<K>>
150where
151 K: Resource + Clone + DeserializeOwned + Debug,
152 <K as Resource>::DynamicType: Default,
153{
154 let api = Api::<K>::all(client.clone());
155 if let Err(e) = api.list(&ListParams::default().limit(1)).await {
156 info!(
157 "{} is not queryable (optional resource); {e:?}. Skipping optional resource support.",
158 short_type_name::<K>().unwrap_or("Unknown resource"),
159 );
160 None
161 } else {
162 Some(api)
163 }
164}
165
166pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
167where
168 K: Resource + Lookup + Clone + 'static,
169 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
170 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
171{
172 let (store, writer) = reflector::store_shared::<K>(buffer_size);
173 let subscriber = writer
174 .subscribe()
175 .expect("subscribers can only be created from shared stores");
176
177 ResourceReflector {
178 store,
179 writer,
180 subscriber,
181 }
182}
183
184fn create_generic_watcher<K, W, T, S, StreamT>(
185 api: Api<K>,
186 writer: Writer<W>,
187 reload_tx: mpsc::Sender<()>,
188 controller_id: ControllerId,
189 ctx: Arc<Context<T>>,
190 stream_fn: S,
191) -> BoxFuture<'static, ()>
192where
193 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
194 W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
195 <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
196 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
197 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
198 <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
199 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
200 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
201 S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
202 StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
203 + Send
204 + 'static,
205{
206 let resource_name = short_type_name::<K>().unwrap_or("Unknown");
207
208 stream_fn(
209 api,
210 watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
211 )
212 .default_backoff()
213 .reflect_shared(writer)
214 .for_each(move |res| {
215 let mut reload_tx_clone = reload_tx.clone();
216 let ctx = ctx.clone();
217 async move {
218 match res {
219 Ok(event) => {
220 trace!(msg = "watched event", ?event);
221 match event {
222 watcher::Event::Delete(d) => {
223 debug!(
224 msg = format!("delete event for {resource_name} trigger reconcile"),
225 namespace = ResourceExt::namespace(&d).unwrap(),
226 name = d.name_any()
227 );
228
229 let _ignore_errors = reload_tx_clone.try_send(()).map_err(
233 |e| error!(msg = "failed to trigger reconcile on delete", %e),
234 );
235 ctx.metrics
236 .triggered_inc(metrics::Action::Delete, resource_name);
237 }
238 watcher::Event::Apply(d) => {
239 debug!(
240 msg = format!("apply event for {resource_name} trigger reconcile"),
241 namespace = ResourceExt::namespace(&d).unwrap(),
242 name = d.name_any()
243 );
244 ctx.metrics
245 .triggered_inc(metrics::Action::Apply, resource_name);
246 }
247 _ => {}
248 }
249 }
250 Err(e) => {
251 error!(msg = format!("unexpected error when watching {resource_name}"), %e);
252 ctx.metrics.watch_operations_failed_inc();
253 }
254 }
255 }
256 })
257 .boxed()
258}
259
260pub fn create_watcher<K, T>(
261 api: Api<K>,
262 writer: Writer<K>,
263 reload_tx: mpsc::Sender<()>,
264 controller_id: ControllerId,
265 ctx: Arc<Context<T>>,
266) -> BoxFuture<'static, ()>
267where
268 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
269 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
270 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
271 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
272 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
273{
274 create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
275}
276
277pub fn create_metadata_watcher<K, T>(
278 api: Api<K>,
279 writer: Writer<PartialObjectMeta<K>>,
280 reload_tx: mpsc::Sender<()>,
281 controller_id: ControllerId,
282 ctx: Arc<Context<T>>,
283) -> BoxFuture<'static, ()>
284where
285 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
286 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
287 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
288 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
289 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
290{
291 create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
292 api,
293 writer,
294 reload_tx,
295 controller_id,
296 ctx,
297 metadata_watcher,
298 )
299}
300
301pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
302where
303 K: Resource + Lookup + Clone + 'static,
304 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
305{
306 unreachable!("Handle in backoff_reconciler macro")
307}
308
309#[macro_export]
310macro_rules! backoff_reconciler {
311 ($inner_reconciler:ident) => {
312 |obj, ctx| async move {
313 use $crate::controller::context::BackoffContext;
314 match $inner_reconciler(obj.clone(), ctx.clone()).await {
315 Ok(action) => {
316 ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
317 .await;
318 Ok(action)
319 }
320 Err(error) => {
321 let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
323 let name = kube::ResourceExt::name_any(obj.as_ref());
324 tracing::error!(msg = "failed reconciliation", %namespace, %name, %error);
325 ctx.metrics().reconcile_failure_inc();
326 let backoff_duration = ctx
327 .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
328 .await;
329 tracing::trace!(
330 msg = format!("backoff duration: {backoff_duration:?}"),
331 %namespace,
332 %name,
333 );
334 Ok(kube::runtime::controller::Action::requeue(backoff_duration))
335 }
336 }
337 }
338 };
339}