Skip to main content

kaniop_operator/controller/
context.rs

1use super::{
2    ControllerId, KanidmClients, idm_reconcile_interval,
3    kanidm::{ClientLockKey, KanidmKey, KanidmResource, KanidmUser},
4};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics::ControllerMetrics;
8use kaniop_k8s_util::error::{Error, Result};
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::collections::HashMap;
13use std::sync::Arc;
14
15use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder};
16use k8s_openapi::{NamespaceResourceScope, api::core::v1::Namespace};
17use kanidm_client::KanidmClient;
18use kube::runtime::events::{Event, EventType, Recorder};
19use kube::{Api, client::Client};
20use kube::{Resource, ResourceExt};
21use kube::{
22    api::{Patch, PatchParams},
23    runtime::reflector::{Lookup, ObjectRef, Store},
24};
25use serde::{Deserialize, Serialize};
26use tokio::sync::{Mutex, RwLock};
27use tokio::time::Duration;
28use tracing::{debug, error, info, trace};
29
30// Context for our reconciler
31#[derive(Clone)]
32pub struct Context<K: Resource> {
33    /// Controller ID
34    pub controller_id: ControllerId,
35    /// Kubernetes client
36    pub client: Client,
37    /// Prometheus metrics
38    pub metrics: Arc<ControllerMetrics>,
39    /// State of the error backoff policy per object
40    error_backoff_cache: Arc<RwLock<HashMap<ObjectRef<K>, RwLock<ExponentialBackoff>>>>,
41    /// Event recorder
42    pub recorder: Recorder,
43    /// Cache for Namespace resources
44    pub namespace_store: Store<Namespace>,
45    /// Cache for Kanidm resources
46    pub kanidm_store: Store<Kanidm>,
47    /// Shared Kanidm cache clients with the ability to manage users and their groups
48    idm_clients: Arc<RwLock<KanidmClients>>,
49    /// Shared Kanidm cache clients with the ability to manage the operation of Kanidm as a
50    /// database and service
51    system_clients: Arc<RwLock<KanidmClients>>,
52    /// Locks for client creation to prevent thundering herd problem
53    client_creation_locks: Arc<RwLock<HashMap<ClientLockKey, Arc<Mutex<()>>>>>,
54}
55
56impl<K> Context<K>
57where
58    K: Resource + ResourceExt + Lookup + Clone + 'static,
59    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
60{
61    #[allow(clippy::too_many_arguments)]
62    pub fn new(
63        controller_id: ControllerId,
64        client: Client,
65        metrics: Arc<ControllerMetrics>,
66        recorder: Recorder,
67        idm_clients: Arc<RwLock<KanidmClients>>,
68        system_clients: Arc<RwLock<KanidmClients>>,
69        namespace_store: Store<Namespace>,
70        kanidm_store: Store<Kanidm>,
71    ) -> Self {
72        Self {
73            controller_id,
74            client,
75            metrics,
76            recorder,
77            namespace_store,
78            kanidm_store,
79            idm_clients,
80            system_clients,
81            error_backoff_cache: Arc::default(),
82            client_creation_locks: Arc::default(),
83        }
84    }
85
86    pub async fn release_kanidm_clients(&self, kanidm: &Kanidm) {
87        let key = KanidmKey {
88            // safe unwrap: Kanidm is namespaced scoped
89            namespace: kube::ResourceExt::namespace(kanidm).unwrap(),
90            name: kanidm.name_any(),
91        };
92
93        self.idm_clients.write().await.remove(&key);
94        self.system_clients.write().await.remove(&key);
95        {
96            let mut locks = self.client_creation_locks.write().await;
97            locks.remove(&ClientLockKey {
98                namespace: key.namespace.clone(),
99                name: key.name.clone(),
100                user: KanidmUser::IdmAdmin,
101            });
102            locks.remove(&ClientLockKey {
103                namespace: key.namespace.clone(),
104                name: key.name.clone(),
105                user: KanidmUser::Admin,
106            });
107        }
108    }
109}
110
111impl<K> Context<K>
112where
113    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
114    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
115{
116    /// Check if a valid client exists in cache
117    async fn get_valid_cached_client(
118        cache: &Arc<RwLock<KanidmClients>>,
119        key: &KanidmKey,
120        namespace: &str,
121        name: &str,
122    ) -> Option<Arc<KanidmClient>> {
123        let client = cache.read().await.get(key).cloned()?;
124
125        trace!(
126            msg = "check existing Kanidm client session",
127            namespace, name
128        );
129        if client.auth_valid().await.is_ok() {
130            trace!(msg = "reuse Kanidm client session", namespace, name);
131            Some(client)
132        } else {
133            None
134        }
135    }
136
137    /// Return a valid client for the Kanidm cluster. This operation require to do at least a
138    /// request for validating the client, use it wisely.
139    async fn get_kanidm_client(&self, obj: &K, user: KanidmUser) -> Result<Arc<KanidmClient>> {
140        let namespace = obj.kanidm_namespace();
141        let name = obj.kanidm_name();
142        debug!(msg = "get Kanidm client", namespace, name);
143
144        let cache = match user {
145            KanidmUser::Admin => self.system_clients.clone(),
146            KanidmUser::IdmAdmin => self.idm_clients.clone(),
147        };
148        let key = KanidmKey {
149            namespace: namespace.clone(),
150            name: name.clone(),
151        };
152        if let Some(client) = Self::get_valid_cached_client(&cache, &key, &namespace, &name).await {
153            return Ok(client);
154        }
155
156        // Slow path: acquire lock for this specific client to prevent concurrent creation
157        let creation_lock = self
158            .client_creation_locks
159            .write()
160            .await
161            .entry(ClientLockKey {
162                namespace: namespace.clone(),
163                name: name.clone(),
164                user: user.clone(),
165            })
166            .or_insert_with(Arc::default)
167            .clone();
168        let _guard = creation_lock.lock().await;
169
170        // Double-check: another task may have created the client while we waited for the lock
171        if let Some(client) = Self::get_valid_cached_client(&cache, &key, &namespace, &name).await {
172            return Ok(client);
173        }
174
175        match KanidmClients::create_client(&namespace, &name, user, self.client.clone()).await {
176            Ok(client) => {
177                cache.write().await.insert(key.clone(), client.clone());
178                Ok(client)
179            }
180            Err(e) => {
181                self.recorder
182                    .publish(
183                        &Event {
184                            type_: EventType::Warning,
185                            reason: "KanidmClientError".to_string(),
186                            note: Some(e.to_string()),
187                            action: "KanidmClientCreating".into(),
188                            secondary: None,
189                        },
190                        &obj.object_ref(&()),
191                    )
192                    .await
193                    .map_err(|e| {
194                        error!(msg = "failed to create Kanidm client", %e);
195                        Error::KubeError("failed to publish event".to_string(), Box::new(e))
196                    })?;
197                Err(e)
198            }
199        }
200    }
201
202    /// Return [`Kanidm`] of the given object
203    ///
204    /// [`Kanidm`]: struct.Kanidm.html
205    pub fn get_kanidm(&self, obj: &K) -> Option<Arc<Kanidm>> {
206        let namespace = obj.kanidm_namespace();
207        let name = obj.kanidm_name();
208        self.kanidm_store.find(|k| {
209            kube::ResourceExt::namespace(k).as_ref() == Some(&namespace) && k.name_any() == name
210        })
211    }
212}
213
214#[allow(async_fn_in_trait)]
215pub trait BackoffContext<K: Resource> {
216    fn metrics(&self) -> &Arc<ControllerMetrics>;
217    async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration;
218    async fn reset_backoff(&self, obj_ref: ObjectRef<K>);
219}
220
221impl<K> BackoffContext<K> for Context<K>
222where
223    K: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
224    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
225{
226    fn metrics(&self) -> &Arc<ControllerMetrics> {
227        &self.metrics
228    }
229
230    /// Return next duration of the backoff policy for the given object
231    async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration {
232        {
233            let read_guard = self.error_backoff_cache.read().await;
234            if let Some(backoff) = read_guard.get(&obj_ref) {
235                if let Some(duration) = backoff.write().await.next() {
236                    return duration;
237                }
238            }
239        }
240
241        // Backoff policy: 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 300s, 300s...
242        let mut backoff = ExponentialBuilder::default()
243            .with_max_delay(idm_reconcile_interval())
244            .without_max_times()
245            .build();
246        // First backoff is always Some(Duration), but use defensive fallback
247        let duration = backoff.next().unwrap_or_else(|| {
248            trace!("backoff returned None, using default duration");
249            Duration::from_secs(1)
250        });
251        self.error_backoff_cache
252            .write()
253            .await
254            .insert(obj_ref.clone(), RwLock::new(backoff));
255        trace!(
256            msg = format!("recreate backoff policy"),
257            namespace = obj_ref.namespace.as_deref().unwrap(),
258            name = obj_ref.name,
259        );
260        duration
261    }
262
263    /// Reset the backoff policy for the given object
264    async fn reset_backoff(&self, obj_ref: ObjectRef<K>) {
265        let read_guard = self.error_backoff_cache.read().await;
266        if read_guard.get(&obj_ref).is_some() {
267            drop(read_guard);
268            trace!(
269                msg = "reset backoff policy",
270                namespace = obj_ref.namespace.as_deref().unwrap(),
271                name = obj_ref.name
272            );
273            self.error_backoff_cache.write().await.remove(&obj_ref);
274        }
275    }
276}
277
278#[allow(async_fn_in_trait)]
279pub trait IdmClientContext<K: Resource> {
280    async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
281}
282
283impl<K> IdmClientContext<K> for Context<K>
284where
285    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
286    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
287{
288    async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
289        self.get_kanidm_client(obj, KanidmUser::IdmAdmin).await
290    }
291}
292
293#[allow(async_fn_in_trait)]
294pub trait SystemClientContext<K: Resource> {
295    async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
296}
297
298impl<K> SystemClientContext<K> for Context<K>
299where
300    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
301    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
302{
303    async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
304        self.get_kanidm_client(obj, KanidmUser::Admin).await
305    }
306}
307
308fn is_immutable_update_message(message: &str) -> bool {
309    let message = message.to_ascii_lowercase();
310
311    message.contains("field is immutable")
312        || message.contains("may not change once set")
313        || message.contains("may not be changed once set")
314        || (message.contains("updates to statefulset spec for fields other than")
315            && message.contains("are forbidden"))
316}
317
318#[allow(async_fn_in_trait)]
319pub trait KubeOperations<T, K>
320where
321    T: Resource + ResourceExt + Lookup + Clone + 'static,
322    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
323    K: Resource<Scope = NamespaceResourceScope>
324        + Serialize
325        + Clone
326        + std::fmt::Debug
327        + for<'de> Deserialize<'de>,
328    <K as kube::Resource>::DynamicType: Default,
329    <K as Resource>::Scope: std::marker::Sized,
330{
331    async fn kube_delete(&self, client: Client, metrics: &ControllerMetrics, obj: &K)
332    -> Result<()>;
333    async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K>;
334    async fn kube_patch(
335        &self,
336        client: Client,
337        metrics: &ControllerMetrics,
338        obj: K,
339        operator_name: &str,
340    ) -> Result<K>;
341}
342
343impl<T, K> KubeOperations<T, K> for T
344where
345    T: Resource + ResourceExt + Lookup + Clone + 'static,
346    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
347    K: Resource<Scope = NamespaceResourceScope>
348        + Serialize
349        + Clone
350        + std::fmt::Debug
351        + for<'de> Deserialize<'de>,
352    <K as kube::Resource>::DynamicType: Default,
353    <K as Resource>::Scope: std::marker::Sized,
354{
355    async fn kube_delete(
356        &self,
357        client: Client,
358        _metrics: &ControllerMetrics,
359        obj: &K,
360    ) -> Result<()> {
361        let name = obj.name_any();
362        // safe unwrap: self is namespaced scoped
363        let namespace = kube::ResourceExt::namespace(self).unwrap();
364        trace!(
365            msg = format!("deleting {}", short_type_name::<K>().unwrap_or("Unknown")),
366            resource.name = &name,
367            resource.namespace = &namespace
368        );
369        let api = Api::<K>::namespaced(client, &namespace);
370        match api.delete(&name, &Default::default()).await {
371            Ok(_) => Ok(()),
372            Err(kube::Error::Api(ae)) if ae.code == 404 => {
373                trace!(
374                    msg = format!(
375                        "{} not found, treating delete as successful",
376                        short_type_name::<K>().unwrap_or("Unknown")
377                    ),
378                    resource.name = &name,
379                    resource.namespace = &namespace
380                );
381                Ok(())
382            }
383            Err(e) => Err(Error::KubeError(
384                format!(
385                    "failed to delete {} {namespace}/{name}",
386                    short_type_name::<K>().unwrap_or("Unknown")
387                ),
388                Box::new(e),
389            )),
390        }
391    }
392
393    async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K> {
394        let name = obj.name_any();
395        // safe unwrap: self is namespaced scoped
396        let namespace = kube::ResourceExt::namespace(self).unwrap();
397        trace!(
398            msg = format!("applying {}", short_type_name::<K>().unwrap_or("Unknown")),
399            resource.name = &name,
400            resource.namespace = &namespace
401        );
402        let resource_api = Api::<K>::namespaced(client, &namespace);
403
404        resource_api
405            .patch(
406                &name,
407                &PatchParams::apply(operator_name).force(),
408                &Patch::Apply(&obj),
409            )
410            .await
411            .map_err(|e| {
412                Error::KubeError(
413                    format!(
414                        "failed to patch {} {namespace}/{name}",
415                        short_type_name::<K>().unwrap_or("Unknown")
416                    ),
417                    Box::new(e),
418                )
419            })
420    }
421
422    async fn kube_patch(
423        &self,
424        client: Client,
425        metrics: &ControllerMetrics,
426        obj: K,
427        operator_name: &str,
428    ) -> Result<K> {
429        let name = obj.name_any();
430        // safe unwrap: self is namespaced scoped
431        let namespace = kube::ResourceExt::namespace(self).unwrap();
432        let result = self
433            .kube_apply(client.clone(), obj.clone(), operator_name)
434            .await;
435
436        if let Err(Error::KubeError(_, cause)) = &result
437            && let kube::Error::Api(ae) = cause.as_ref()
438            && ae.code == 422
439            && is_immutable_update_message(&ae.message)
440        {
441            info!(
442                msg = format!(
443                    "recreating {} because an immutable field changed",
444                    short_type_name::<K>().unwrap_or("Unknown")
445                ),
446                resource.name = &name,
447                resource.namespace = &namespace,
448                reason = %ae.reason,
449                message = %ae.message,
450            );
451            self.kube_delete(client.clone(), metrics, &obj).await?;
452            metrics.reconcile_deploy_delete_create_inc(
453                short_type_name::<K>().unwrap_or("Unknown"),
454                "immutable_api_error",
455            );
456            return self.kube_apply(client, obj, operator_name).await;
457        }
458
459        if let Err(Error::KubeError(_, cause)) = &result
460            && let kube::Error::Api(ae) = cause.as_ref()
461            && ae.code == 422
462        {
463            debug!(
464                msg = "server-side apply rejected resource with a non-immutable 422; preserving the existing resource",
465                resource.kind = short_type_name::<K>().unwrap_or("Unknown"),
466                resource.name = &name,
467                resource.namespace = &namespace,
468                reason = %ae.reason,
469                message = %ae.message,
470            );
471        }
472
473        result
474    }
475}
476
477#[cfg(test)]
478mod test {
479    use super::is_immutable_update_message;
480
481    #[test]
482    fn immutable_update_messages_are_classified() {
483        assert!(is_immutable_update_message(
484            "Deployment.apps \"example\" is invalid: spec.selector: Invalid value: v1.LabelSelector{}: field is immutable"
485        ));
486        assert!(is_immutable_update_message(
487            "Service \"example\" is invalid: spec.clusterIPs[0]: Invalid value: \"10.0.0.1\": may not change once set"
488        ));
489        assert!(is_immutable_update_message(
490            "StatefulSet.apps \"example\" is invalid: spec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'ordinals', 'template', 'updateStrategy', 'revisionHistoryLimit', 'persistentVolumeClaimRetentionPolicy' and 'minReadySeconds' are forbidden"
491        ));
492    }
493
494    #[test]
495    fn ordinary_validation_messages_are_not_classified_as_immutable() {
496        assert!(!is_immutable_update_message(
497            "StatefulSet.apps \"example\" is invalid: spec.minReadySeconds: Invalid value: -1: must be greater than or equal to 0"
498        ));
499        assert!(!is_immutable_update_message(
500            "StatefulSet.apps \"example\" is invalid: spec.template.spec.containers[0].image: Required value"
501        ));
502    }
503}