1use super::{
2 ControllerId, KanidmClients, idm_reconcile_interval,
3 kanidm::{ClientLockKey, KanidmKey, KanidmResource, KanidmUser},
4};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics::ControllerMetrics;
8use kaniop_k8s_util::error::{Error, Result};
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::collections::HashMap;
13use std::sync::Arc;
14
15use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder};
16use k8s_openapi::{NamespaceResourceScope, api::core::v1::Namespace};
17use kanidm_client::KanidmClient;
18use kube::runtime::events::{Event, EventType, Recorder};
19use kube::{Api, client::Client};
20use kube::{Resource, ResourceExt};
21use kube::{
22 api::{Patch, PatchParams},
23 runtime::reflector::{Lookup, ObjectRef, Store},
24};
25use serde::{Deserialize, Serialize};
26use tokio::sync::{Mutex, RwLock};
27use tokio::time::Duration;
28use tracing::{debug, error, info, trace};
29
30#[derive(Clone)]
32pub struct Context<K: Resource> {
33 pub controller_id: ControllerId,
35 pub client: Client,
37 pub metrics: Arc<ControllerMetrics>,
39 error_backoff_cache: Arc<RwLock<HashMap<ObjectRef<K>, RwLock<ExponentialBackoff>>>>,
41 pub recorder: Recorder,
43 pub namespace_store: Store<Namespace>,
45 pub kanidm_store: Store<Kanidm>,
47 idm_clients: Arc<RwLock<KanidmClients>>,
49 system_clients: Arc<RwLock<KanidmClients>>,
52 client_creation_locks: Arc<RwLock<HashMap<ClientLockKey, Arc<Mutex<()>>>>>,
54}
55
56impl<K> Context<K>
57where
58 K: Resource + ResourceExt + Lookup + Clone + 'static,
59 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
60{
61 #[allow(clippy::too_many_arguments)]
62 pub fn new(
63 controller_id: ControllerId,
64 client: Client,
65 metrics: Arc<ControllerMetrics>,
66 recorder: Recorder,
67 idm_clients: Arc<RwLock<KanidmClients>>,
68 system_clients: Arc<RwLock<KanidmClients>>,
69 namespace_store: Store<Namespace>,
70 kanidm_store: Store<Kanidm>,
71 ) -> Self {
72 Self {
73 controller_id,
74 client,
75 metrics,
76 recorder,
77 namespace_store,
78 kanidm_store,
79 idm_clients,
80 system_clients,
81 error_backoff_cache: Arc::default(),
82 client_creation_locks: Arc::default(),
83 }
84 }
85
86 pub async fn release_kanidm_clients(&self, kanidm: &Kanidm) {
87 let key = KanidmKey {
88 namespace: kube::ResourceExt::namespace(kanidm).unwrap(),
90 name: kanidm.name_any(),
91 };
92
93 self.idm_clients.write().await.remove(&key);
94 self.system_clients.write().await.remove(&key);
95 {
96 let mut locks = self.client_creation_locks.write().await;
97 locks.remove(&ClientLockKey {
98 namespace: key.namespace.clone(),
99 name: key.name.clone(),
100 user: KanidmUser::IdmAdmin,
101 });
102 locks.remove(&ClientLockKey {
103 namespace: key.namespace.clone(),
104 name: key.name.clone(),
105 user: KanidmUser::Admin,
106 });
107 }
108 }
109}
110
111impl<K> Context<K>
112where
113 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
114 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
115{
116 async fn get_valid_cached_client(
118 cache: &Arc<RwLock<KanidmClients>>,
119 key: &KanidmKey,
120 namespace: &str,
121 name: &str,
122 ) -> Option<Arc<KanidmClient>> {
123 let client = cache.read().await.get(key).cloned()?;
124
125 trace!(namespace, name, "check existing Kanidm client session");
126 if client.auth_valid().await.is_ok() {
127 trace!(namespace, name, "reuse Kanidm client session");
128 Some(client)
129 } else {
130 None
131 }
132 }
133
134 async fn get_kanidm_client(&self, obj: &K, user: KanidmUser) -> Result<Arc<KanidmClient>> {
137 let namespace = obj.kanidm_namespace();
138 let name = obj.kanidm_name();
139 debug!(namespace, name, "get Kanidm client");
140
141 let cache = match user {
142 KanidmUser::Admin => self.system_clients.clone(),
143 KanidmUser::IdmAdmin => self.idm_clients.clone(),
144 };
145 let key = KanidmKey {
146 namespace: namespace.clone(),
147 name: name.clone(),
148 };
149 if let Some(client) = Self::get_valid_cached_client(&cache, &key, &namespace, &name).await {
150 return Ok(client);
151 }
152
153 let creation_lock = self
155 .client_creation_locks
156 .write()
157 .await
158 .entry(ClientLockKey {
159 namespace: namespace.clone(),
160 name: name.clone(),
161 user: user.clone(),
162 })
163 .or_insert_with(Arc::default)
164 .clone();
165 let _guard = creation_lock.lock().await;
166
167 if let Some(client) = Self::get_valid_cached_client(&cache, &key, &namespace, &name).await {
169 return Ok(client);
170 }
171
172 match KanidmClients::create_client(&namespace, &name, user, self.client.clone()).await {
173 Ok(client) => {
174 cache.write().await.insert(key.clone(), client.clone());
175 Ok(client)
176 }
177 Err(e) => {
178 self.recorder
179 .publish(
180 &Event {
181 type_: EventType::Warning,
182 reason: "KanidmClientError".to_string(),
183 note: Some(e.to_string()),
184 action: "KanidmClientCreating".into(),
185 secondary: None,
186 },
187 &obj.object_ref(&()),
188 )
189 .await
190 .map_err(|e| {
191 error!(%e, "failed to create Kanidm client");
192 Error::KubeError("failed to publish event".to_string(), Box::new(e))
193 })?;
194 Err(e)
195 }
196 }
197 }
198
199 pub fn kanidm_write_allowed(&self, obj: &K) -> bool {
201 self.get_kanidm(obj).is_none_or(|kanidm| {
202 !kanidm
203 .annotations()
204 .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
205 })
206 }
207
208 pub fn get_kanidm(&self, obj: &K) -> Option<Arc<Kanidm>> {
212 let namespace = obj.kanidm_namespace();
213 let name = obj.kanidm_name();
214 self.kanidm_store.find(|k| {
215 kube::ResourceExt::namespace(k).as_ref() == Some(&namespace) && k.name_any() == name
216 })
217 }
218}
219
220#[allow(async_fn_in_trait)]
221pub trait BackoffContext<K: Resource> {
222 fn metrics(&self) -> &Arc<ControllerMetrics>;
223 async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration;
224 async fn reset_backoff(&self, obj_ref: ObjectRef<K>);
225}
226
227impl<K> BackoffContext<K> for Context<K>
228where
229 K: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
230 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
231{
232 fn metrics(&self) -> &Arc<ControllerMetrics> {
233 &self.metrics
234 }
235
236 async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration {
238 {
239 let read_guard = self.error_backoff_cache.read().await;
240 if let Some(backoff) = read_guard.get(&obj_ref) {
241 if let Some(duration) = backoff.write().await.next() {
242 return duration;
243 }
244 }
245 }
246
247 let mut backoff = ExponentialBuilder::default()
249 .with_max_delay(idm_reconcile_interval())
250 .without_max_times()
251 .build();
252 let duration = backoff.next().unwrap_or_else(|| {
254 trace!("backoff returned None, using default duration");
255 Duration::from_secs(1)
256 });
257 let mut cache = self.error_backoff_cache.write().await;
258 use std::collections::hash_map::Entry;
259 match cache.entry(obj_ref.clone()) {
260 Entry::Vacant(vacant) => {
261 vacant.insert(RwLock::new(backoff));
262 self.metrics.objects_in_backoff_inc();
263 }
264 Entry::Occupied(_occupied) => {}
265 }
266 trace!(
267 namespace = obj_ref.namespace.as_deref().unwrap(),
268 name = obj_ref.name,
269 "recreate backoff policy"
270 );
271 duration
272 }
273
274 async fn reset_backoff(&self, obj_ref: ObjectRef<K>) {
276 let removed = self.error_backoff_cache.write().await.remove(&obj_ref);
277 if removed.is_some() {
278 trace!(
279 namespace = obj_ref.namespace.as_deref().unwrap(),
280 name = obj_ref.name,
281 "reset backoff policy"
282 );
283 self.metrics.objects_in_backoff_dec();
284 }
285 }
286}
287
288#[allow(async_fn_in_trait)]
289pub trait IdmClientContext<K: Resource> {
290 async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
291}
292
293impl<K> IdmClientContext<K> for Context<K>
294where
295 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
296 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
297{
298 async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
299 self.get_kanidm_client(obj, KanidmUser::IdmAdmin).await
300 }
301}
302
303#[allow(async_fn_in_trait)]
304pub trait SystemClientContext<K: Resource> {
305 async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
306}
307
308impl<K> SystemClientContext<K> for Context<K>
309where
310 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
311 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
312{
313 async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
314 self.get_kanidm_client(obj, KanidmUser::Admin).await
315 }
316}
317
318fn is_immutable_update_message(message: &str) -> bool {
319 let message = message.to_ascii_lowercase();
320
321 message.contains("field is immutable")
322 || message.contains("may not change once set")
323 || message.contains("may not be changed once set")
324 || (message.contains("updates to statefulset spec for fields other than")
325 && message.contains("are forbidden"))
326}
327
328#[allow(async_fn_in_trait)]
329pub trait KubeOperations<T, K>
330where
331 T: Resource + ResourceExt + Lookup + Clone + 'static,
332 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
333 K: Resource<Scope = NamespaceResourceScope>
334 + Serialize
335 + Clone
336 + std::fmt::Debug
337 + for<'de> Deserialize<'de>,
338 <K as kube::Resource>::DynamicType: Default,
339 <K as Resource>::Scope: std::marker::Sized,
340{
341 async fn kube_delete(&self, client: Client, metrics: &ControllerMetrics, obj: &K)
342 -> Result<()>;
343 async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K>;
344 async fn kube_patch(
345 &self,
346 client: Client,
347 metrics: &ControllerMetrics,
348 obj: K,
349 operator_name: &str,
350 ) -> Result<K>;
351}
352
353impl<T, K> KubeOperations<T, K> for T
354where
355 T: Resource + ResourceExt + Lookup + Clone + 'static,
356 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
357 K: Resource<Scope = NamespaceResourceScope>
358 + Serialize
359 + Clone
360 + std::fmt::Debug
361 + for<'de> Deserialize<'de>,
362 <K as kube::Resource>::DynamicType: Default,
363 <K as Resource>::Scope: std::marker::Sized,
364{
365 async fn kube_delete(
366 &self,
367 client: Client,
368 _metrics: &ControllerMetrics,
369 obj: &K,
370 ) -> Result<()> {
371 let name = obj.name_any();
372 let namespace = kube::ResourceExt::namespace(self).unwrap();
374 trace!(
375 resource.name = &name,
376 resource.namespace = &namespace,
377 "deleting {}",
378 short_type_name::<K>().unwrap_or("Unknown")
379 );
380 let api = Api::<K>::namespaced(client, &namespace);
381 match api.delete(&name, &Default::default()).await {
382 Ok(_) => Ok(()),
383 Err(kube::Error::Api(ae)) if ae.code == 404 => {
384 trace!(
385 resource.name = &name,
386 resource.namespace = &namespace,
387 "{} not found, treating delete as successful",
388 short_type_name::<K>().unwrap_or("Unknown")
389 );
390 Ok(())
391 }
392 Err(e) => Err(Error::KubeError(
393 format!(
394 "failed to delete {} {namespace}/{name}",
395 short_type_name::<K>().unwrap_or("Unknown")
396 ),
397 Box::new(e),
398 )),
399 }
400 }
401
402 async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K> {
403 let name = obj.name_any();
404 let namespace = kube::ResourceExt::namespace(self).unwrap();
406 trace!(
407 resource.name = &name,
408 resource.namespace = &namespace,
409 "applying {}",
410 short_type_name::<K>().unwrap_or("Unknown")
411 );
412 let resource_api = Api::<K>::namespaced(client, &namespace);
413
414 resource_api
415 .patch(
416 &name,
417 &PatchParams::apply(operator_name).force(),
418 &Patch::Apply(&obj),
419 )
420 .await
421 .map_err(|e| {
422 Error::KubeError(
423 format!(
424 "failed to patch {} {namespace}/{name}",
425 short_type_name::<K>().unwrap_or("Unknown")
426 ),
427 Box::new(e),
428 )
429 })
430 }
431
432 async fn kube_patch(
433 &self,
434 client: Client,
435 metrics: &ControllerMetrics,
436 obj: K,
437 operator_name: &str,
438 ) -> Result<K> {
439 let name = obj.name_any();
440 let namespace = kube::ResourceExt::namespace(self).unwrap();
442 let result = self
443 .kube_apply(client.clone(), obj.clone(), operator_name)
444 .await;
445
446 if let Err(Error::KubeError(_, cause)) = &result
447 && let kube::Error::Api(ae) = cause.as_ref()
448 && ae.code == 422
449 && is_immutable_update_message(&ae.message)
450 {
451 info!(resource.name = &name, resource.namespace = &namespace, reason = %ae.reason, api_message = %ae.message, "recreating {} because an immutable field changed",
452 short_type_name::<K>().unwrap_or("Unknown"));
453 self.kube_delete(client.clone(), metrics, &obj).await?;
454 metrics.reconcile_deploy_delete_create_inc(
455 short_type_name::<K>().unwrap_or("Unknown"),
456 "immutable_api_error",
457 );
458 return self.kube_apply(client, obj, operator_name).await;
459 }
460
461 if let Err(Error::KubeError(_, cause)) = &result
462 && let kube::Error::Api(ae) = cause.as_ref()
463 && ae.code == 422
464 {
465 debug!(resource.kind = short_type_name::<K>().unwrap_or("Unknown"), resource.name = &name, resource.namespace = &namespace, reason = %ae.reason, api_message = %ae.message, "server-side apply rejected resource with a non-immutable 422; preserving the existing resource");
466 }
467
468 result
469 }
470}
471
472#[cfg(test)]
473mod test {
474 use super::is_immutable_update_message;
475
476 #[test]
477 fn immutable_update_messages_are_classified() {
478 assert!(is_immutable_update_message(
479 "Deployment.apps \"example\" is invalid: spec.selector: Invalid value: v1.LabelSelector{}: field is immutable"
480 ));
481 assert!(is_immutable_update_message(
482 "Service \"example\" is invalid: spec.clusterIPs[0]: Invalid value: \"10.0.0.1\": may not change once set"
483 ));
484 assert!(is_immutable_update_message(
485 "StatefulSet.apps \"example\" is invalid: spec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'ordinals', 'template', 'updateStrategy', 'revisionHistoryLimit', 'persistentVolumeClaimRetentionPolicy' and 'minReadySeconds' are forbidden"
486 ));
487 }
488
489 #[test]
490 fn ordinary_validation_messages_are_not_classified_as_immutable() {
491 assert!(!is_immutable_update_message(
492 "StatefulSet.apps \"example\" is invalid: spec.minReadySeconds: Invalid value: -1: must be greater than or equal to 0"
493 ));
494 assert!(!is_immutable_update_message(
495 "StatefulSet.apps \"example\" is invalid: spec.template.spec.containers[0].image: Required value"
496 ));
497 }
498}