1use super::{
2 ControllerId, KanidmClients, idm_reconcile_interval,
3 kanidm::{ClientLockKey, KanidmKey, KanidmResource, KanidmUser},
4};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics::ControllerMetrics;
8use kaniop_k8s_util::error::{Error, Result};
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::collections::HashMap;
13use std::sync::Arc;
14
15use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder};
16use k8s_openapi::{NamespaceResourceScope, api::core::v1::Namespace};
17use kanidm_client::KanidmClient;
18use kube::runtime::events::{Event, EventType, Recorder};
19use kube::{Api, client::Client};
20use kube::{Resource, ResourceExt};
21use kube::{
22 api::{Patch, PatchParams},
23 runtime::reflector::{Lookup, ObjectRef, Store},
24};
25use serde::{Deserialize, Serialize};
26use tokio::sync::{Mutex, RwLock};
27use tokio::time::Duration;
28use tracing::{debug, error, info, trace};
29
30#[derive(Clone)]
32pub struct Context<K: Resource> {
33 pub controller_id: ControllerId,
35 pub client: Client,
37 pub metrics: Arc<ControllerMetrics>,
39 error_backoff_cache: Arc<RwLock<HashMap<ObjectRef<K>, RwLock<ExponentialBackoff>>>>,
41 pub recorder: Recorder,
43 pub namespace_store: Store<Namespace>,
45 pub kanidm_store: Store<Kanidm>,
47 idm_clients: Arc<RwLock<KanidmClients>>,
49 system_clients: Arc<RwLock<KanidmClients>>,
52 client_creation_locks: Arc<RwLock<HashMap<ClientLockKey, Arc<Mutex<()>>>>>,
54}
55
56impl<K> Context<K>
57where
58 K: Resource + ResourceExt + Lookup + Clone + 'static,
59 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
60{
61 #[allow(clippy::too_many_arguments)]
62 pub fn new(
63 controller_id: ControllerId,
64 client: Client,
65 metrics: Arc<ControllerMetrics>,
66 recorder: Recorder,
67 idm_clients: Arc<RwLock<KanidmClients>>,
68 system_clients: Arc<RwLock<KanidmClients>>,
69 namespace_store: Store<Namespace>,
70 kanidm_store: Store<Kanidm>,
71 ) -> Self {
72 Self {
73 controller_id,
74 client,
75 metrics,
76 recorder,
77 namespace_store,
78 kanidm_store,
79 idm_clients,
80 system_clients,
81 error_backoff_cache: Arc::default(),
82 client_creation_locks: Arc::default(),
83 }
84 }
85
86 pub async fn release_kanidm_clients(&self, kanidm: &Kanidm) {
87 let key = KanidmKey {
88 namespace: kube::ResourceExt::namespace(kanidm).unwrap(),
90 name: kanidm.name_any(),
91 };
92
93 self.idm_clients.write().await.remove(&key);
94 self.system_clients.write().await.remove(&key);
95 {
96 let mut locks = self.client_creation_locks.write().await;
97 locks.remove(&ClientLockKey {
98 namespace: key.namespace.clone(),
99 name: key.name.clone(),
100 user: KanidmUser::IdmAdmin,
101 });
102 locks.remove(&ClientLockKey {
103 namespace: key.namespace.clone(),
104 name: key.name.clone(),
105 user: KanidmUser::Admin,
106 });
107 }
108 }
109}
110
111impl<K> Context<K>
112where
113 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
114 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
115{
116 async fn get_valid_cached_client(
118 cache: &Arc<RwLock<KanidmClients>>,
119 key: &KanidmKey,
120 namespace: &str,
121 name: &str,
122 ) -> Option<Arc<KanidmClient>> {
123 let client = cache.read().await.get(key).cloned()?;
124
125 trace!(
126 msg = "check existing Kanidm client session",
127 namespace, name
128 );
129 if client.auth_valid().await.is_ok() {
130 trace!(msg = "reuse Kanidm client session", namespace, name);
131 Some(client)
132 } else {
133 None
134 }
135 }
136
137 async fn get_kanidm_client(&self, obj: &K, user: KanidmUser) -> Result<Arc<KanidmClient>> {
140 let namespace = obj.kanidm_namespace();
141 let name = obj.kanidm_name();
142 debug!(msg = "get Kanidm client", namespace, name);
143
144 let cache = match user {
145 KanidmUser::Admin => self.system_clients.clone(),
146 KanidmUser::IdmAdmin => self.idm_clients.clone(),
147 };
148 let key = KanidmKey {
149 namespace: namespace.clone(),
150 name: name.clone(),
151 };
152 if let Some(client) = Self::get_valid_cached_client(&cache, &key, &namespace, &name).await {
153 return Ok(client);
154 }
155
156 let creation_lock = self
158 .client_creation_locks
159 .write()
160 .await
161 .entry(ClientLockKey {
162 namespace: namespace.clone(),
163 name: name.clone(),
164 user: user.clone(),
165 })
166 .or_insert_with(Arc::default)
167 .clone();
168 let _guard = creation_lock.lock().await;
169
170 if let Some(client) = Self::get_valid_cached_client(&cache, &key, &namespace, &name).await {
172 return Ok(client);
173 }
174
175 match KanidmClients::create_client(&namespace, &name, user, self.client.clone()).await {
176 Ok(client) => {
177 cache.write().await.insert(key.clone(), client.clone());
178 Ok(client)
179 }
180 Err(e) => {
181 self.recorder
182 .publish(
183 &Event {
184 type_: EventType::Warning,
185 reason: "KanidmClientError".to_string(),
186 note: Some(e.to_string()),
187 action: "KanidmClientCreating".into(),
188 secondary: None,
189 },
190 &obj.object_ref(&()),
191 )
192 .await
193 .map_err(|e| {
194 error!(msg = "failed to create Kanidm client", %e);
195 Error::KubeError("failed to publish event".to_string(), Box::new(e))
196 })?;
197 Err(e)
198 }
199 }
200 }
201
202 pub fn kanidm_write_allowed(&self, obj: &K) -> bool {
204 self.get_kanidm(obj).is_none_or(|kanidm| {
205 !kanidm
206 .annotations()
207 .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
208 })
209 }
210
211 pub fn get_kanidm(&self, obj: &K) -> Option<Arc<Kanidm>> {
215 let namespace = obj.kanidm_namespace();
216 let name = obj.kanidm_name();
217 self.kanidm_store.find(|k| {
218 kube::ResourceExt::namespace(k).as_ref() == Some(&namespace) && k.name_any() == name
219 })
220 }
221}
222
223#[allow(async_fn_in_trait)]
224pub trait BackoffContext<K: Resource> {
225 fn metrics(&self) -> &Arc<ControllerMetrics>;
226 async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration;
227 async fn reset_backoff(&self, obj_ref: ObjectRef<K>);
228}
229
230impl<K> BackoffContext<K> for Context<K>
231where
232 K: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
233 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
234{
235 fn metrics(&self) -> &Arc<ControllerMetrics> {
236 &self.metrics
237 }
238
239 async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration {
241 {
242 let read_guard = self.error_backoff_cache.read().await;
243 if let Some(backoff) = read_guard.get(&obj_ref) {
244 if let Some(duration) = backoff.write().await.next() {
245 return duration;
246 }
247 }
248 }
249
250 let mut backoff = ExponentialBuilder::default()
252 .with_max_delay(idm_reconcile_interval())
253 .without_max_times()
254 .build();
255 let duration = backoff.next().unwrap_or_else(|| {
257 trace!("backoff returned None, using default duration");
258 Duration::from_secs(1)
259 });
260 let mut cache = self.error_backoff_cache.write().await;
261 use std::collections::hash_map::Entry;
262 match cache.entry(obj_ref.clone()) {
263 Entry::Vacant(vacant) => {
264 vacant.insert(RwLock::new(backoff));
265 self.metrics.objects_in_backoff_inc();
266 }
267 Entry::Occupied(_occupied) => {}
268 }
269 trace!(
270 msg = "recreate backoff policy".to_string(),
271 namespace = obj_ref.namespace.as_deref().unwrap(),
272 name = obj_ref.name,
273 );
274 duration
275 }
276
277 async fn reset_backoff(&self, obj_ref: ObjectRef<K>) {
279 let removed = self.error_backoff_cache.write().await.remove(&obj_ref);
280 if removed.is_some() {
281 trace!(
282 msg = "reset backoff policy",
283 namespace = obj_ref.namespace.as_deref().unwrap(),
284 name = obj_ref.name
285 );
286 self.metrics.objects_in_backoff_dec();
287 }
288 }
289}
290
291#[allow(async_fn_in_trait)]
292pub trait IdmClientContext<K: Resource> {
293 async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
294}
295
296impl<K> IdmClientContext<K> for Context<K>
297where
298 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
299 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
300{
301 async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
302 self.get_kanidm_client(obj, KanidmUser::IdmAdmin).await
303 }
304}
305
306#[allow(async_fn_in_trait)]
307pub trait SystemClientContext<K: Resource> {
308 async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
309}
310
311impl<K> SystemClientContext<K> for Context<K>
312where
313 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
314 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
315{
316 async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
317 self.get_kanidm_client(obj, KanidmUser::Admin).await
318 }
319}
320
321fn is_immutable_update_message(message: &str) -> bool {
322 let message = message.to_ascii_lowercase();
323
324 message.contains("field is immutable")
325 || message.contains("may not change once set")
326 || message.contains("may not be changed once set")
327 || (message.contains("updates to statefulset spec for fields other than")
328 && message.contains("are forbidden"))
329}
330
331#[allow(async_fn_in_trait)]
332pub trait KubeOperations<T, K>
333where
334 T: Resource + ResourceExt + Lookup + Clone + 'static,
335 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
336 K: Resource<Scope = NamespaceResourceScope>
337 + Serialize
338 + Clone
339 + std::fmt::Debug
340 + for<'de> Deserialize<'de>,
341 <K as kube::Resource>::DynamicType: Default,
342 <K as Resource>::Scope: std::marker::Sized,
343{
344 async fn kube_delete(&self, client: Client, metrics: &ControllerMetrics, obj: &K)
345 -> Result<()>;
346 async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K>;
347 async fn kube_patch(
348 &self,
349 client: Client,
350 metrics: &ControllerMetrics,
351 obj: K,
352 operator_name: &str,
353 ) -> Result<K>;
354}
355
356impl<T, K> KubeOperations<T, K> for T
357where
358 T: Resource + ResourceExt + Lookup + Clone + 'static,
359 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
360 K: Resource<Scope = NamespaceResourceScope>
361 + Serialize
362 + Clone
363 + std::fmt::Debug
364 + for<'de> Deserialize<'de>,
365 <K as kube::Resource>::DynamicType: Default,
366 <K as Resource>::Scope: std::marker::Sized,
367{
368 async fn kube_delete(
369 &self,
370 client: Client,
371 _metrics: &ControllerMetrics,
372 obj: &K,
373 ) -> Result<()> {
374 let name = obj.name_any();
375 let namespace = kube::ResourceExt::namespace(self).unwrap();
377 trace!(
378 msg = format!("deleting {}", short_type_name::<K>().unwrap_or("Unknown")),
379 resource.name = &name,
380 resource.namespace = &namespace
381 );
382 let api = Api::<K>::namespaced(client, &namespace);
383 match api.delete(&name, &Default::default()).await {
384 Ok(_) => Ok(()),
385 Err(kube::Error::Api(ae)) if ae.code == 404 => {
386 trace!(
387 msg = format!(
388 "{} not found, treating delete as successful",
389 short_type_name::<K>().unwrap_or("Unknown")
390 ),
391 resource.name = &name,
392 resource.namespace = &namespace
393 );
394 Ok(())
395 }
396 Err(e) => Err(Error::KubeError(
397 format!(
398 "failed to delete {} {namespace}/{name}",
399 short_type_name::<K>().unwrap_or("Unknown")
400 ),
401 Box::new(e),
402 )),
403 }
404 }
405
406 async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K> {
407 let name = obj.name_any();
408 let namespace = kube::ResourceExt::namespace(self).unwrap();
410 trace!(
411 msg = format!("applying {}", short_type_name::<K>().unwrap_or("Unknown")),
412 resource.name = &name,
413 resource.namespace = &namespace
414 );
415 let resource_api = Api::<K>::namespaced(client, &namespace);
416
417 resource_api
418 .patch(
419 &name,
420 &PatchParams::apply(operator_name).force(),
421 &Patch::Apply(&obj),
422 )
423 .await
424 .map_err(|e| {
425 Error::KubeError(
426 format!(
427 "failed to patch {} {namespace}/{name}",
428 short_type_name::<K>().unwrap_or("Unknown")
429 ),
430 Box::new(e),
431 )
432 })
433 }
434
435 async fn kube_patch(
436 &self,
437 client: Client,
438 metrics: &ControllerMetrics,
439 obj: K,
440 operator_name: &str,
441 ) -> Result<K> {
442 let name = obj.name_any();
443 let namespace = kube::ResourceExt::namespace(self).unwrap();
445 let result = self
446 .kube_apply(client.clone(), obj.clone(), operator_name)
447 .await;
448
449 if let Err(Error::KubeError(_, cause)) = &result
450 && let kube::Error::Api(ae) = cause.as_ref()
451 && ae.code == 422
452 && is_immutable_update_message(&ae.message)
453 {
454 info!(
455 msg = format!(
456 "recreating {} because an immutable field changed",
457 short_type_name::<K>().unwrap_or("Unknown")
458 ),
459 resource.name = &name,
460 resource.namespace = &namespace,
461 reason = %ae.reason,
462 message = %ae.message,
463 );
464 self.kube_delete(client.clone(), metrics, &obj).await?;
465 metrics.reconcile_deploy_delete_create_inc(
466 short_type_name::<K>().unwrap_or("Unknown"),
467 "immutable_api_error",
468 );
469 return self.kube_apply(client, obj, operator_name).await;
470 }
471
472 if let Err(Error::KubeError(_, cause)) = &result
473 && let kube::Error::Api(ae) = cause.as_ref()
474 && ae.code == 422
475 {
476 debug!(
477 msg = "server-side apply rejected resource with a non-immutable 422; preserving the existing resource",
478 resource.kind = short_type_name::<K>().unwrap_or("Unknown"),
479 resource.name = &name,
480 resource.namespace = &namespace,
481 reason = %ae.reason,
482 message = %ae.message,
483 );
484 }
485
486 result
487 }
488}
489
490#[cfg(test)]
491mod test {
492 use super::is_immutable_update_message;
493
494 #[test]
495 fn immutable_update_messages_are_classified() {
496 assert!(is_immutable_update_message(
497 "Deployment.apps \"example\" is invalid: spec.selector: Invalid value: v1.LabelSelector{}: field is immutable"
498 ));
499 assert!(is_immutable_update_message(
500 "Service \"example\" is invalid: spec.clusterIPs[0]: Invalid value: \"10.0.0.1\": may not change once set"
501 ));
502 assert!(is_immutable_update_message(
503 "StatefulSet.apps \"example\" is invalid: spec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'ordinals', 'template', 'updateStrategy', 'revisionHistoryLimit', 'persistentVolumeClaimRetentionPolicy' and 'minReadySeconds' are forbidden"
504 ));
505 }
506
507 #[test]
508 fn ordinary_validation_messages_are_not_classified_as_immutable() {
509 assert!(!is_immutable_update_message(
510 "StatefulSet.apps \"example\" is invalid: spec.minReadySeconds: Invalid value: -1: must be greater than or equal to 0"
511 ));
512 assert!(!is_immutable_update_message(
513 "StatefulSet.apps \"example\" is invalid: spec.template.spec.containers[0].image: Required value"
514 ));
515 }
516}