1use super::{
2 ControllerId, KanidmClients, idm_reconcile_interval,
3 kanidm::{ClientLockKey, KanidmKey, KanidmResource, KanidmUser},
4};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics::ControllerMetrics;
8use kaniop_k8s_util::error::{Error, Result};
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::collections::HashMap;
13use std::sync::Arc;
14
15use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder};
16use k8s_openapi::{NamespaceResourceScope, api::core::v1::Namespace};
17use kanidm_client::KanidmClient;
18use kube::runtime::events::{Event, EventType, Recorder};
19use kube::{Api, client::Client};
20use kube::{Resource, ResourceExt};
21use kube::{
22 api::{Patch, PatchParams},
23 runtime::reflector::{Lookup, ObjectRef, Store},
24};
25use serde::{Deserialize, Serialize};
26use tokio::sync::{Mutex, RwLock};
27use tokio::time::Duration;
28use tracing::{debug, error, info, trace};
29
30#[derive(Clone)]
32pub struct Context<K: Resource> {
33 pub controller_id: ControllerId,
35 pub client: Client,
37 pub metrics: Arc<ControllerMetrics>,
39 error_backoff_cache: Arc<RwLock<HashMap<ObjectRef<K>, RwLock<ExponentialBackoff>>>>,
41 pub recorder: Recorder,
43 pub namespace_store: Store<Namespace>,
45 pub kanidm_store: Store<Kanidm>,
47 idm_clients: Arc<RwLock<KanidmClients>>,
49 system_clients: Arc<RwLock<KanidmClients>>,
52 client_creation_locks: Arc<RwLock<HashMap<ClientLockKey, Arc<Mutex<()>>>>>,
54}
55
56impl<K> Context<K>
57where
58 K: Resource + ResourceExt + Lookup + Clone + 'static,
59 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
60{
61 #[allow(clippy::too_many_arguments)]
62 pub fn new(
63 controller_id: ControllerId,
64 client: Client,
65 metrics: Arc<ControllerMetrics>,
66 recorder: Recorder,
67 idm_clients: Arc<RwLock<KanidmClients>>,
68 system_clients: Arc<RwLock<KanidmClients>>,
69 namespace_store: Store<Namespace>,
70 kanidm_store: Store<Kanidm>,
71 ) -> Self {
72 Self {
73 controller_id,
74 client,
75 metrics,
76 recorder,
77 namespace_store,
78 kanidm_store,
79 idm_clients,
80 system_clients,
81 error_backoff_cache: Arc::default(),
82 client_creation_locks: Arc::default(),
83 }
84 }
85
86 pub async fn release_kanidm_clients(&self, kanidm: &Kanidm) {
87 let key = KanidmKey {
88 namespace: kube::ResourceExt::namespace(kanidm).unwrap(),
90 name: kanidm.name_any(),
91 };
92
93 self.idm_clients.write().await.remove(&key);
94 self.system_clients.write().await.remove(&key);
95 {
96 let mut locks = self.client_creation_locks.write().await;
97 locks.remove(&ClientLockKey {
98 namespace: key.namespace.clone(),
99 name: key.name.clone(),
100 user: KanidmUser::IdmAdmin,
101 });
102 locks.remove(&ClientLockKey {
103 namespace: key.namespace.clone(),
104 name: key.name.clone(),
105 user: KanidmUser::Admin,
106 });
107 }
108 }
109
110 pub async fn get_kanidm_cluster_client(
114 &self,
115 kanidm: &Kanidm,
116 user: KanidmUser,
117 ) -> Result<Arc<KanidmClient>> {
118 let namespace = kube::ResourceExt::namespace(kanidm).unwrap();
120 self.get_cached_client(&namespace, &kanidm.name_any(), user)
121 .await
122 }
123
124 async fn get_valid_cached_client(
126 cache: &Arc<RwLock<KanidmClients>>,
127 key: &KanidmKey,
128 namespace: &str,
129 name: &str,
130 ) -> Option<Arc<KanidmClient>> {
131 let client = cache.read().await.get(key).cloned()?;
132
133 trace!(namespace, name, "check existing Kanidm client session");
134 if client.auth_valid().await.is_ok() {
135 trace!(namespace, name, "reuse Kanidm client session");
136 Some(client)
137 } else {
138 None
139 }
140 }
141
142 async fn get_cached_client(
145 &self,
146 namespace: &str,
147 name: &str,
148 user: KanidmUser,
149 ) -> Result<Arc<KanidmClient>> {
150 debug!(namespace, name, "get Kanidm client");
151
152 let cache = match user {
153 KanidmUser::Admin => self.system_clients.clone(),
154 KanidmUser::IdmAdmin => self.idm_clients.clone(),
155 };
156 let key = KanidmKey {
157 namespace: namespace.to_string(),
158 name: name.to_string(),
159 };
160 if let Some(client) = Self::get_valid_cached_client(&cache, &key, namespace, name).await {
161 return Ok(client);
162 }
163
164 let creation_lock = self
166 .client_creation_locks
167 .write()
168 .await
169 .entry(ClientLockKey {
170 namespace: namespace.to_string(),
171 name: name.to_string(),
172 user: user.clone(),
173 })
174 .or_insert_with(Arc::default)
175 .clone();
176 let _guard = creation_lock.lock().await;
177
178 if let Some(client) = Self::get_valid_cached_client(&cache, &key, namespace, name).await {
180 return Ok(client);
181 }
182
183 let client =
184 KanidmClients::create_client(namespace, name, user, self.client.clone()).await?;
185 cache.write().await.insert(key, client.clone());
186 Ok(client)
187 }
188}
189
190impl<K> Context<K>
191where
192 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
193 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
194{
195 async fn get_kanidm_client(&self, obj: &K, user: KanidmUser) -> Result<Arc<KanidmClient>> {
198 let namespace = obj.kanidm_namespace();
199 let name = obj.kanidm_name();
200
201 match self.get_cached_client(&namespace, &name, user).await {
202 Ok(client) => Ok(client),
203 Err(e) => {
204 self.recorder
205 .publish(
206 &Event {
207 type_: EventType::Warning,
208 reason: "KanidmClientError".to_string(),
209 note: Some(e.to_string()),
210 action: "KanidmClientCreating".into(),
211 secondary: None,
212 },
213 &obj.object_ref(&()),
214 )
215 .await
216 .map_err(|e| {
217 error!(%e, "failed to create Kanidm client");
218 Error::KubeError("failed to publish event".to_string(), Box::new(e))
219 })?;
220 Err(e)
221 }
222 }
223 }
224
225 pub fn kanidm_write_allowed(&self, obj: &K) -> bool {
227 self.get_kanidm(obj).is_none_or(|kanidm| {
228 !kanidm
229 .annotations()
230 .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
231 })
232 }
233
234 pub fn get_kanidm(&self, obj: &K) -> Option<Arc<Kanidm>> {
238 let namespace = obj.kanidm_namespace();
239 let name = obj.kanidm_name();
240 self.kanidm_store.find(|k| {
241 kube::ResourceExt::namespace(k).as_ref() == Some(&namespace) && k.name_any() == name
242 })
243 }
244}
245
246#[allow(async_fn_in_trait)]
247pub trait BackoffContext<K: Resource> {
248 fn metrics(&self) -> &Arc<ControllerMetrics>;
249 async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration;
250 async fn reset_backoff(&self, obj_ref: ObjectRef<K>);
251}
252
253impl<K> BackoffContext<K> for Context<K>
254where
255 K: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
256 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
257{
258 fn metrics(&self) -> &Arc<ControllerMetrics> {
259 &self.metrics
260 }
261
262 async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration {
264 {
265 let read_guard = self.error_backoff_cache.read().await;
266 if let Some(backoff) = read_guard.get(&obj_ref) {
267 if let Some(duration) = backoff.write().await.next() {
268 return duration;
269 }
270 }
271 }
272
273 let mut backoff = ExponentialBuilder::default()
275 .with_max_delay(idm_reconcile_interval())
276 .without_max_times()
277 .build();
278 let duration = backoff.next().unwrap_or_else(|| {
280 trace!("backoff returned None, using default duration");
281 Duration::from_secs(1)
282 });
283 let mut cache = self.error_backoff_cache.write().await;
284 use std::collections::hash_map::Entry;
285 match cache.entry(obj_ref.clone()) {
286 Entry::Vacant(vacant) => {
287 vacant.insert(RwLock::new(backoff));
288 self.metrics.objects_in_backoff_inc();
289 }
290 Entry::Occupied(_occupied) => {}
291 }
292 trace!(
293 namespace = obj_ref.namespace.as_deref().unwrap(),
294 name = obj_ref.name,
295 "recreate backoff policy"
296 );
297 duration
298 }
299
300 async fn reset_backoff(&self, obj_ref: ObjectRef<K>) {
302 let removed = self.error_backoff_cache.write().await.remove(&obj_ref);
303 if removed.is_some() {
304 trace!(
305 namespace = obj_ref.namespace.as_deref().unwrap(),
306 name = obj_ref.name,
307 "reset backoff policy"
308 );
309 self.metrics.objects_in_backoff_dec();
310 }
311 }
312}
313
314#[allow(async_fn_in_trait)]
315pub trait IdmClientContext<K: Resource> {
316 async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
317}
318
319impl<K> IdmClientContext<K> for Context<K>
320where
321 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
322 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
323{
324 async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
325 self.get_kanidm_client(obj, KanidmUser::IdmAdmin).await
326 }
327}
328
329#[allow(async_fn_in_trait)]
330pub trait SystemClientContext<K: Resource> {
331 async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
332}
333
334impl<K> SystemClientContext<K> for Context<K>
335where
336 K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
337 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
338{
339 async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
340 self.get_kanidm_client(obj, KanidmUser::Admin).await
341 }
342}
343
344fn is_immutable_update_message(message: &str) -> bool {
345 let message = message.to_ascii_lowercase();
346
347 message.contains("field is immutable")
348 || message.contains("may not change once set")
349 || message.contains("may not be changed once set")
350 || (message.contains("updates to statefulset spec for fields other than")
351 && message.contains("are forbidden"))
352}
353
354#[allow(async_fn_in_trait)]
355pub trait KubeOperations<T, K>
356where
357 T: Resource + ResourceExt + Lookup + Clone + 'static,
358 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
359 K: Resource<Scope = NamespaceResourceScope>
360 + Serialize
361 + Clone
362 + std::fmt::Debug
363 + for<'de> Deserialize<'de>,
364 <K as kube::Resource>::DynamicType: Default,
365 <K as Resource>::Scope: std::marker::Sized,
366{
367 async fn kube_delete(&self, client: Client, metrics: &ControllerMetrics, obj: &K)
368 -> Result<()>;
369 async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K>;
370 async fn kube_patch(
371 &self,
372 client: Client,
373 metrics: &ControllerMetrics,
374 obj: K,
375 operator_name: &str,
376 ) -> Result<K>;
377}
378
379impl<T, K> KubeOperations<T, K> for T
380where
381 T: Resource + ResourceExt + Lookup + Clone + 'static,
382 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
383 K: Resource<Scope = NamespaceResourceScope>
384 + Serialize
385 + Clone
386 + std::fmt::Debug
387 + for<'de> Deserialize<'de>,
388 <K as kube::Resource>::DynamicType: Default,
389 <K as Resource>::Scope: std::marker::Sized,
390{
391 async fn kube_delete(
392 &self,
393 client: Client,
394 _metrics: &ControllerMetrics,
395 obj: &K,
396 ) -> Result<()> {
397 let name = obj.name_any();
398 let namespace = kube::ResourceExt::namespace(self).unwrap();
400 trace!(
401 resource.name = &name,
402 resource.namespace = &namespace,
403 "deleting {}",
404 short_type_name::<K>().unwrap_or("Unknown")
405 );
406 let api = Api::<K>::namespaced(client, &namespace);
407 match api.delete(&name, &Default::default()).await {
408 Ok(_) => Ok(()),
409 Err(kube::Error::Api(ae)) if ae.code == 404 => {
410 trace!(
411 resource.name = &name,
412 resource.namespace = &namespace,
413 "{} not found, treating delete as successful",
414 short_type_name::<K>().unwrap_or("Unknown")
415 );
416 Ok(())
417 }
418 Err(e) => Err(Error::KubeError(
419 format!(
420 "failed to delete {} {namespace}/{name}",
421 short_type_name::<K>().unwrap_or("Unknown")
422 ),
423 Box::new(e),
424 )),
425 }
426 }
427
428 async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K> {
429 let name = obj.name_any();
430 let namespace = kube::ResourceExt::namespace(self).unwrap();
432 trace!(
433 resource.name = &name,
434 resource.namespace = &namespace,
435 "applying {}",
436 short_type_name::<K>().unwrap_or("Unknown")
437 );
438 let resource_api = Api::<K>::namespaced(client, &namespace);
439
440 resource_api
441 .patch(
442 &name,
443 &PatchParams::apply(operator_name).force(),
444 &Patch::Apply(&obj),
445 )
446 .await
447 .map_err(|e| {
448 Error::KubeError(
449 format!(
450 "failed to patch {} {namespace}/{name}",
451 short_type_name::<K>().unwrap_or("Unknown")
452 ),
453 Box::new(e),
454 )
455 })
456 }
457
458 async fn kube_patch(
459 &self,
460 client: Client,
461 metrics: &ControllerMetrics,
462 obj: K,
463 operator_name: &str,
464 ) -> Result<K> {
465 let name = obj.name_any();
466 let namespace = kube::ResourceExt::namespace(self).unwrap();
468 let result = self
469 .kube_apply(client.clone(), obj.clone(), operator_name)
470 .await;
471
472 if let Err(Error::KubeError(_, cause)) = &result
473 && let kube::Error::Api(ae) = cause.as_ref()
474 && ae.code == 422
475 && is_immutable_update_message(&ae.message)
476 {
477 info!(resource.name = &name, resource.namespace = &namespace, reason = %ae.reason, api_message = %ae.message, "recreating {} because an immutable field changed",
478 short_type_name::<K>().unwrap_or("Unknown"));
479 self.kube_delete(client.clone(), metrics, &obj).await?;
480 metrics.reconcile_deploy_delete_create_inc(
481 short_type_name::<K>().unwrap_or("Unknown"),
482 "immutable_api_error",
483 );
484 return self.kube_apply(client, obj, operator_name).await;
485 }
486
487 if let Err(Error::KubeError(_, cause)) = &result
488 && let kube::Error::Api(ae) = cause.as_ref()
489 && ae.code == 422
490 {
491 debug!(resource.kind = short_type_name::<K>().unwrap_or("Unknown"), resource.name = &name, resource.namespace = &namespace, reason = %ae.reason, api_message = %ae.message, "server-side apply rejected resource with a non-immutable 422; preserving the existing resource");
492 }
493
494 result
495 }
496}
497
498#[cfg(test)]
499mod test {
500 use super::{Context, is_immutable_update_message};
501
502 use crate::controller::State;
503 use crate::controller::kanidm::{KanidmKey, KanidmUser};
504 use crate::kanidm::crd::Kanidm;
505
506 use std::sync::{Arc, Mutex};
507 use std::time::Duration;
508
509 use futures::FutureExt;
510 use http::{Request, Response};
511 use kanidm_client::{KanidmClient, KanidmClientBuilder};
512 use kube::runtime::reflector::store::Writer;
513 use kube::{Client, ResourceExt, client::Body};
514 use opentelemetry::metrics::MeterProvider;
515 use tokio::io::{AsyncReadExt, AsyncWriteExt};
516 use tokio::net::TcpListener;
517
518 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
519
520 const KANIDM_VERSION: &str = "1.11.2";
523
524 fn test_context() -> (Context<Kanidm>, ApiServerHandle) {
525 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
526 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
527 let metrics = crate::metrics::Metrics::new(&provider.meter("test"), &["test"]);
528 let state = State::new(
529 metrics,
530 Writer::default().as_reader(),
531 Writer::default().as_reader(),
532 None,
533 );
534 (
535 state.to_context(Client::new(mock_service, "default"), "test"),
536 handle,
537 )
538 }
539
540 fn kanidm_key(kanidm: &Kanidm) -> KanidmKey {
541 KanidmKey {
542 namespace: kanidm.namespace().unwrap(),
543 name: kanidm.name_any(),
544 }
545 }
546
547 async fn start_fake_kanidm(status: u16) -> (Arc<KanidmClient>, Arc<Mutex<Vec<String>>>) {
550 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
551 let address = listener.local_addr().unwrap();
552 let requests = Arc::new(Mutex::new(Vec::new()));
553 let seen = requests.clone();
554 tokio::spawn(async move {
555 while let Ok((mut stream, _)) = listener.accept().await {
556 let mut buffer = [0u8; 4096];
557 let n = stream.read(&mut buffer).await.unwrap_or(0);
558 if let Some(line) = String::from_utf8_lossy(&buffer[..n]).lines().next() {
559 seen.lock().unwrap().push(line.to_string());
560 }
561 let response = format!(
562 "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\nx-kanidm-version: {KANIDM_VERSION}\r\ncontent-length: 4\r\nconnection: close\r\n\r\nnull"
563 );
564 let _ = stream.write_all(response.as_bytes()).await;
565 let _ = stream.shutdown().await;
566 }
567 });
568 let client = KanidmClientBuilder::new()
569 .no_proxy()
570 .address(format!("http://{address}"))
571 .build()
572 .unwrap();
573 (Arc::new(client), requests)
574 }
575
576 #[tokio::test]
577 async fn kanidm_cluster_client_reuses_valid_cached_session() {
578 let (ctx, mut api_server) = test_context();
579 let kanidm = Kanidm::test();
580 let (cached, requests) = start_fake_kanidm(200).await;
581 ctx.system_clients
582 .write()
583 .await
584 .insert(kanidm_key(&kanidm), cached.clone());
585
586 let client = tokio::time::timeout(
588 Duration::from_secs(5),
589 ctx.get_kanidm_cluster_client(&kanidm, KanidmUser::Admin),
590 )
591 .await
592 .expect("cached client is reused without creating a new session")
593 .expect("valid cached client");
594
595 assert!(Arc::ptr_eq(&client, &cached));
596 assert_eq!(
597 *requests.lock().unwrap(),
598 vec!["GET /v1/auth/valid HTTP/1.1".to_string()]
599 );
600 assert!(api_server.next_request().now_or_never().is_none());
601
602 ctx.release_kanidm_clients(&kanidm).await;
603 assert!(
604 ctx.system_clients
605 .read()
606 .await
607 .get(&kanidm_key(&kanidm))
608 .is_none()
609 );
610 }
611
612 #[tokio::test]
613 async fn kanidm_cluster_client_recreates_invalid_cached_session() {
614 let (ctx, mut api_server) = test_context();
615 let kanidm = Kanidm::test();
616 let (stale, requests) = start_fake_kanidm(401).await;
617 ctx.idm_clients
618 .write()
619 .await
620 .insert(kanidm_key(&kanidm), stale);
621
622 let api_server = async move {
623 let (request, send) = api_server
624 .next_request()
625 .await
626 .expect("Kanidm is read to create a new client");
627 assert_eq!(
628 request.uri().path(),
629 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/test"
630 );
631 send.send_response(
632 Response::builder()
633 .status(404)
634 .body(Body::from(
635 br#"{"kind":"Status","apiVersion":"v1","status":"Failure","reason":"NotFound","code":404}"#
636 .to_vec(),
637 ))
638 .unwrap(),
639 );
640 };
641 let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async {
642 tokio::join!(
643 ctx.get_kanidm_cluster_client(&kanidm, KanidmUser::IdmAdmin),
644 api_server
645 )
646 })
647 .await
648 .expect("invalid cached session falls back to client creation");
649
650 assert!(result.is_err());
651 assert_eq!(
653 *requests.lock().unwrap(),
654 vec!["GET /v1/auth/valid HTTP/1.1".to_string(); 2]
655 );
656 }
657
658 #[test]
659 fn immutable_update_messages_are_classified() {
660 assert!(is_immutable_update_message(
661 "Deployment.apps \"example\" is invalid: spec.selector: Invalid value: v1.LabelSelector{}: field is immutable"
662 ));
663 assert!(is_immutable_update_message(
664 "Service \"example\" is invalid: spec.clusterIPs[0]: Invalid value: \"10.0.0.1\": may not change once set"
665 ));
666 assert!(is_immutable_update_message(
667 "StatefulSet.apps \"example\" is invalid: spec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'ordinals', 'template', 'updateStrategy', 'revisionHistoryLimit', 'persistentVolumeClaimRetentionPolicy' and 'minReadySeconds' are forbidden"
668 ));
669 }
670
671 #[test]
672 fn ordinary_validation_messages_are_not_classified_as_immutable() {
673 assert!(!is_immutable_update_message(
674 "StatefulSet.apps \"example\" is invalid: spec.minReadySeconds: Invalid value: -1: must be greater than or equal to 0"
675 ));
676 assert!(!is_immutable_update_message(
677 "StatefulSet.apps \"example\" is invalid: spec.template.spec.containers[0].image: Required value"
678 ));
679 }
680}