Skip to main content

kaniop_operator/controller/
context.rs

1use super::{
2    ControllerId, KanidmClients, idm_reconcile_interval,
3    kanidm::{ClientLockKey, KanidmKey, KanidmResource, KanidmUser},
4};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics::ControllerMetrics;
8use kaniop_k8s_util::error::{Error, Result};
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::collections::HashMap;
13use std::sync::Arc;
14
15use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder};
16use k8s_openapi::{NamespaceResourceScope, api::core::v1::Namespace};
17use kanidm_client::KanidmClient;
18use kube::runtime::events::{Event, EventType, Recorder};
19use kube::{Api, client::Client};
20use kube::{Resource, ResourceExt};
21use kube::{
22    api::{Patch, PatchParams},
23    runtime::reflector::{Lookup, ObjectRef, Store},
24};
25use serde::{Deserialize, Serialize};
26use tokio::sync::{Mutex, RwLock};
27use tokio::time::Duration;
28use tracing::{debug, error, info, trace};
29
30// Context for our reconciler
31#[derive(Clone)]
32pub struct Context<K: Resource> {
33    /// Controller ID
34    pub controller_id: ControllerId,
35    /// Kubernetes client
36    pub client: Client,
37    /// Prometheus metrics
38    pub metrics: Arc<ControllerMetrics>,
39    /// State of the error backoff policy per object
40    error_backoff_cache: Arc<RwLock<HashMap<ObjectRef<K>, RwLock<ExponentialBackoff>>>>,
41    /// Event recorder
42    pub recorder: Recorder,
43    /// Cache for Namespace resources
44    pub namespace_store: Store<Namespace>,
45    /// Cache for Kanidm resources
46    pub kanidm_store: Store<Kanidm>,
47    /// Shared Kanidm cache clients with the ability to manage users and their groups
48    idm_clients: Arc<RwLock<KanidmClients>>,
49    /// Shared Kanidm cache clients with the ability to manage the operation of Kanidm as a
50    /// database and service
51    system_clients: Arc<RwLock<KanidmClients>>,
52    /// Locks for client creation to prevent thundering herd problem
53    client_creation_locks: Arc<RwLock<HashMap<ClientLockKey, Arc<Mutex<()>>>>>,
54}
55
56impl<K> Context<K>
57where
58    K: Resource + ResourceExt + Lookup + Clone + 'static,
59    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
60{
61    #[allow(clippy::too_many_arguments)]
62    pub fn new(
63        controller_id: ControllerId,
64        client: Client,
65        metrics: Arc<ControllerMetrics>,
66        recorder: Recorder,
67        idm_clients: Arc<RwLock<KanidmClients>>,
68        system_clients: Arc<RwLock<KanidmClients>>,
69        namespace_store: Store<Namespace>,
70        kanidm_store: Store<Kanidm>,
71    ) -> Self {
72        Self {
73            controller_id,
74            client,
75            metrics,
76            recorder,
77            namespace_store,
78            kanidm_store,
79            idm_clients,
80            system_clients,
81            error_backoff_cache: Arc::default(),
82            client_creation_locks: Arc::default(),
83        }
84    }
85
86    pub async fn release_kanidm_clients(&self, kanidm: &Kanidm) {
87        let key = KanidmKey {
88            // safe unwrap: Kanidm is namespaced scoped
89            namespace: kube::ResourceExt::namespace(kanidm).unwrap(),
90            name: kanidm.name_any(),
91        };
92
93        self.idm_clients.write().await.remove(&key);
94        self.system_clients.write().await.remove(&key);
95        {
96            let mut locks = self.client_creation_locks.write().await;
97            locks.remove(&ClientLockKey {
98                namespace: key.namespace.clone(),
99                name: key.name.clone(),
100                user: KanidmUser::IdmAdmin,
101            });
102            locks.remove(&ClientLockKey {
103                namespace: key.namespace.clone(),
104                name: key.name.clone(),
105                user: KanidmUser::Admin,
106            });
107        }
108    }
109}
110
111impl<K> Context<K>
112where
113    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
114    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
115{
116    /// Check if a valid client exists in cache
117    async fn get_valid_cached_client(
118        cache: &Arc<RwLock<KanidmClients>>,
119        key: &KanidmKey,
120        namespace: &str,
121        name: &str,
122    ) -> Option<Arc<KanidmClient>> {
123        let client = cache.read().await.get(key).cloned()?;
124
125        trace!(namespace, name, "check existing Kanidm client session");
126        if client.auth_valid().await.is_ok() {
127            trace!(namespace, name, "reuse Kanidm client session");
128            Some(client)
129        } else {
130            None
131        }
132    }
133
134    /// Return a valid client for the Kanidm cluster. This operation require to do at least a
135    /// request for validating the client, use it wisely.
136    async fn get_kanidm_client(&self, obj: &K, user: KanidmUser) -> Result<Arc<KanidmClient>> {
137        let namespace = obj.kanidm_namespace();
138        let name = obj.kanidm_name();
139        debug!(namespace, name, "get Kanidm client");
140
141        let cache = match user {
142            KanidmUser::Admin => self.system_clients.clone(),
143            KanidmUser::IdmAdmin => self.idm_clients.clone(),
144        };
145        let key = KanidmKey {
146            namespace: namespace.clone(),
147            name: name.clone(),
148        };
149        if let Some(client) = Self::get_valid_cached_client(&cache, &key, &namespace, &name).await {
150            return Ok(client);
151        }
152
153        // Slow path: acquire lock for this specific client to prevent concurrent creation
154        let creation_lock = self
155            .client_creation_locks
156            .write()
157            .await
158            .entry(ClientLockKey {
159                namespace: namespace.clone(),
160                name: name.clone(),
161                user: user.clone(),
162            })
163            .or_insert_with(Arc::default)
164            .clone();
165        let _guard = creation_lock.lock().await;
166
167        // Double-check: another task may have created the client while we waited for the lock
168        if let Some(client) = Self::get_valid_cached_client(&cache, &key, &namespace, &name).await {
169            return Ok(client);
170        }
171
172        match KanidmClients::create_client(&namespace, &name, user, self.client.clone()).await {
173            Ok(client) => {
174                cache.write().await.insert(key.clone(), client.clone());
175                Ok(client)
176            }
177            Err(e) => {
178                self.recorder
179                    .publish(
180                        &Event {
181                            type_: EventType::Warning,
182                            reason: "KanidmClientError".to_string(),
183                            note: Some(e.to_string()),
184                            action: "KanidmClientCreating".into(),
185                            secondary: None,
186                        },
187                        &obj.object_ref(&()),
188                    )
189                    .await
190                    .map_err(|e| {
191                        error!(%e, "failed to create Kanidm client");
192                        Error::KubeError("failed to publish event".to_string(), Box::new(e))
193                    })?;
194                Err(e)
195            }
196        }
197    }
198
199    /// Return true when normal identity reconciliation may mutate the target Kanidm.
200    pub fn kanidm_write_allowed(&self, obj: &K) -> bool {
201        self.get_kanidm(obj).is_none_or(|kanidm| {
202            !kanidm
203                .annotations()
204                .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
205        })
206    }
207
208    /// Return [`Kanidm`] of the given object
209    ///
210    /// [`Kanidm`]: struct.Kanidm.html
211    pub fn get_kanidm(&self, obj: &K) -> Option<Arc<Kanidm>> {
212        let namespace = obj.kanidm_namespace();
213        let name = obj.kanidm_name();
214        self.kanidm_store.find(|k| {
215            kube::ResourceExt::namespace(k).as_ref() == Some(&namespace) && k.name_any() == name
216        })
217    }
218}
219
220#[allow(async_fn_in_trait)]
221pub trait BackoffContext<K: Resource> {
222    fn metrics(&self) -> &Arc<ControllerMetrics>;
223    async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration;
224    async fn reset_backoff(&self, obj_ref: ObjectRef<K>);
225}
226
227impl<K> BackoffContext<K> for Context<K>
228where
229    K: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
230    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
231{
232    fn metrics(&self) -> &Arc<ControllerMetrics> {
233        &self.metrics
234    }
235
236    /// Return next duration of the backoff policy for the given object
237    async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration {
238        {
239            let read_guard = self.error_backoff_cache.read().await;
240            if let Some(backoff) = read_guard.get(&obj_ref) {
241                if let Some(duration) = backoff.write().await.next() {
242                    return duration;
243                }
244            }
245        }
246
247        // Backoff policy: 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 300s, 300s...
248        let mut backoff = ExponentialBuilder::default()
249            .with_max_delay(idm_reconcile_interval())
250            .without_max_times()
251            .build();
252        // First backoff is always Some(Duration), but use defensive fallback
253        let duration = backoff.next().unwrap_or_else(|| {
254            trace!("backoff returned None, using default duration");
255            Duration::from_secs(1)
256        });
257        let mut cache = self.error_backoff_cache.write().await;
258        use std::collections::hash_map::Entry;
259        match cache.entry(obj_ref.clone()) {
260            Entry::Vacant(vacant) => {
261                vacant.insert(RwLock::new(backoff));
262                self.metrics.objects_in_backoff_inc();
263            }
264            Entry::Occupied(_occupied) => {}
265        }
266        trace!(
267            namespace = obj_ref.namespace.as_deref().unwrap(),
268            name = obj_ref.name,
269            "recreate backoff policy"
270        );
271        duration
272    }
273
274    /// Reset the backoff policy for the given object
275    async fn reset_backoff(&self, obj_ref: ObjectRef<K>) {
276        let removed = self.error_backoff_cache.write().await.remove(&obj_ref);
277        if removed.is_some() {
278            trace!(
279                namespace = obj_ref.namespace.as_deref().unwrap(),
280                name = obj_ref.name,
281                "reset backoff policy"
282            );
283            self.metrics.objects_in_backoff_dec();
284        }
285    }
286}
287
288#[allow(async_fn_in_trait)]
289pub trait IdmClientContext<K: Resource> {
290    async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
291}
292
293impl<K> IdmClientContext<K> for Context<K>
294where
295    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
296    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
297{
298    async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
299        self.get_kanidm_client(obj, KanidmUser::IdmAdmin).await
300    }
301}
302
303#[allow(async_fn_in_trait)]
304pub trait SystemClientContext<K: Resource> {
305    async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
306}
307
308impl<K> SystemClientContext<K> for Context<K>
309where
310    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
311    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
312{
313    async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
314        self.get_kanidm_client(obj, KanidmUser::Admin).await
315    }
316}
317
318fn is_immutable_update_message(message: &str) -> bool {
319    let message = message.to_ascii_lowercase();
320
321    message.contains("field is immutable")
322        || message.contains("may not change once set")
323        || message.contains("may not be changed once set")
324        || (message.contains("updates to statefulset spec for fields other than")
325            && message.contains("are forbidden"))
326}
327
328#[allow(async_fn_in_trait)]
329pub trait KubeOperations<T, K>
330where
331    T: Resource + ResourceExt + Lookup + Clone + 'static,
332    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
333    K: Resource<Scope = NamespaceResourceScope>
334        + Serialize
335        + Clone
336        + std::fmt::Debug
337        + for<'de> Deserialize<'de>,
338    <K as kube::Resource>::DynamicType: Default,
339    <K as Resource>::Scope: std::marker::Sized,
340{
341    async fn kube_delete(&self, client: Client, metrics: &ControllerMetrics, obj: &K)
342    -> Result<()>;
343    async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K>;
344    async fn kube_patch(
345        &self,
346        client: Client,
347        metrics: &ControllerMetrics,
348        obj: K,
349        operator_name: &str,
350    ) -> Result<K>;
351}
352
353impl<T, K> KubeOperations<T, K> for T
354where
355    T: Resource + ResourceExt + Lookup + Clone + 'static,
356    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
357    K: Resource<Scope = NamespaceResourceScope>
358        + Serialize
359        + Clone
360        + std::fmt::Debug
361        + for<'de> Deserialize<'de>,
362    <K as kube::Resource>::DynamicType: Default,
363    <K as Resource>::Scope: std::marker::Sized,
364{
365    async fn kube_delete(
366        &self,
367        client: Client,
368        _metrics: &ControllerMetrics,
369        obj: &K,
370    ) -> Result<()> {
371        let name = obj.name_any();
372        // safe unwrap: self is namespaced scoped
373        let namespace = kube::ResourceExt::namespace(self).unwrap();
374        trace!(
375            resource.name = &name,
376            resource.namespace = &namespace,
377            "deleting {}",
378            short_type_name::<K>().unwrap_or("Unknown")
379        );
380        let api = Api::<K>::namespaced(client, &namespace);
381        match api.delete(&name, &Default::default()).await {
382            Ok(_) => Ok(()),
383            Err(kube::Error::Api(ae)) if ae.code == 404 => {
384                trace!(
385                    resource.name = &name,
386                    resource.namespace = &namespace,
387                    "{} not found, treating delete as successful",
388                    short_type_name::<K>().unwrap_or("Unknown")
389                );
390                Ok(())
391            }
392            Err(e) => Err(Error::KubeError(
393                format!(
394                    "failed to delete {} {namespace}/{name}",
395                    short_type_name::<K>().unwrap_or("Unknown")
396                ),
397                Box::new(e),
398            )),
399        }
400    }
401
402    async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K> {
403        let name = obj.name_any();
404        // safe unwrap: self is namespaced scoped
405        let namespace = kube::ResourceExt::namespace(self).unwrap();
406        trace!(
407            resource.name = &name,
408            resource.namespace = &namespace,
409            "applying {}",
410            short_type_name::<K>().unwrap_or("Unknown")
411        );
412        let resource_api = Api::<K>::namespaced(client, &namespace);
413
414        resource_api
415            .patch(
416                &name,
417                &PatchParams::apply(operator_name).force(),
418                &Patch::Apply(&obj),
419            )
420            .await
421            .map_err(|e| {
422                Error::KubeError(
423                    format!(
424                        "failed to patch {} {namespace}/{name}",
425                        short_type_name::<K>().unwrap_or("Unknown")
426                    ),
427                    Box::new(e),
428                )
429            })
430    }
431
432    async fn kube_patch(
433        &self,
434        client: Client,
435        metrics: &ControllerMetrics,
436        obj: K,
437        operator_name: &str,
438    ) -> Result<K> {
439        let name = obj.name_any();
440        // safe unwrap: self is namespaced scoped
441        let namespace = kube::ResourceExt::namespace(self).unwrap();
442        let result = self
443            .kube_apply(client.clone(), obj.clone(), operator_name)
444            .await;
445
446        if let Err(Error::KubeError(_, cause)) = &result
447            && let kube::Error::Api(ae) = cause.as_ref()
448            && ae.code == 422
449            && is_immutable_update_message(&ae.message)
450        {
451            info!(resource.name = &name, resource.namespace = &namespace, reason = %ae.reason, api_message = %ae.message, "recreating {} because an immutable field changed",
452                    short_type_name::<K>().unwrap_or("Unknown"));
453            self.kube_delete(client.clone(), metrics, &obj).await?;
454            metrics.reconcile_deploy_delete_create_inc(
455                short_type_name::<K>().unwrap_or("Unknown"),
456                "immutable_api_error",
457            );
458            return self.kube_apply(client, obj, operator_name).await;
459        }
460
461        if let Err(Error::KubeError(_, cause)) = &result
462            && let kube::Error::Api(ae) = cause.as_ref()
463            && ae.code == 422
464        {
465            debug!(resource.kind = short_type_name::<K>().unwrap_or("Unknown"), resource.name = &name, resource.namespace = &namespace, reason = %ae.reason, api_message = %ae.message, "server-side apply rejected resource with a non-immutable 422; preserving the existing resource");
466        }
467
468        result
469    }
470}
471
472#[cfg(test)]
473mod test {
474    use super::is_immutable_update_message;
475
476    #[test]
477    fn immutable_update_messages_are_classified() {
478        assert!(is_immutable_update_message(
479            "Deployment.apps \"example\" is invalid: spec.selector: Invalid value: v1.LabelSelector{}: field is immutable"
480        ));
481        assert!(is_immutable_update_message(
482            "Service \"example\" is invalid: spec.clusterIPs[0]: Invalid value: \"10.0.0.1\": may not change once set"
483        ));
484        assert!(is_immutable_update_message(
485            "StatefulSet.apps \"example\" is invalid: spec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'ordinals', 'template', 'updateStrategy', 'revisionHistoryLimit', 'persistentVolumeClaimRetentionPolicy' and 'minReadySeconds' are forbidden"
486        ));
487    }
488
489    #[test]
490    fn ordinary_validation_messages_are_not_classified_as_immutable() {
491        assert!(!is_immutable_update_message(
492            "StatefulSet.apps \"example\" is invalid: spec.minReadySeconds: Invalid value: -1: must be greater than or equal to 0"
493        ));
494        assert!(!is_immutable_update_message(
495            "StatefulSet.apps \"example\" is invalid: spec.template.spec.containers[0].image: Required value"
496        ));
497    }
498}