1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12 cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::{
21 StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
22 preserve_defaulted_statefulset_fields,
23};
24use self::status::StatusExt;
25use self::status::{is_kanidm_available, is_kanidm_initialized};
26
27use crate::controller::context::KubeOperations;
28use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
29use crate::kanidm::crd::{
30 Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
31};
32use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
33use crate::telemetry;
34
35use kaniop_k8s_util::client::get_output;
36use kaniop_k8s_util::error::{Error, Result};
37use kaniop_k8s_util::parse::parse_semver;
38use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
39
40use std::collections::{BTreeMap, BTreeSet};
41use std::fmt::Debug;
42use std::sync::{Arc, LazyLock};
43
44use futures::future::{TryJoinAll, join_all, try_join_all};
45use futures::stream::{self, StreamExt};
46use futures::try_join;
47use k8s_openapi::NamespaceResourceScope;
48use k8s_openapi::api::apps::v1::StatefulSet;
49use k8s_openapi::api::core::v1::{Pod, Secret, Service};
50use kube::Resource;
51use kube::ResourceExt;
52use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
53use kube::runtime::controller::Action;
54use kube::runtime::events::{Event, EventType};
55use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
56use serde::{Deserialize, Serialize};
57use tokio::time::{Duration, sleep};
58use tracing::{Span, debug, field, info, instrument, trace, warn};
59
60const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
61const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
62const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
63const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
64const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
65const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
66const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
67const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
68const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
69
70pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
71const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
72pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
73
74const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
75
76static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
77 BTreeMap::from([
78 (NAME_LABEL.to_string(), "kanidm".to_string()),
79 (
80 MANAGED_BY_LABEL.to_string(),
81 format!("kaniop-{CONTROLLER_ID}"),
82 ),
83 ])
84});
85
86async fn restart_statefulsets_after_replica_secret_update(
87 ctx: Arc<Context>,
88 sts_api: &Api<StatefulSet>,
89 namespace: &str,
90 statefulset_names: &[String],
91 sequential: bool,
92) -> Result<()> {
93 let restart_futures = statefulset_names
96 .iter()
97 .map(|sts_name| sts_api.restart(sts_name));
98
99 if sequential {
100 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
101 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
102 return Err(Error::kube_error(
103 "restart",
104 "StatefulSet",
105 namespace,
106 "replica certificate secret update statefulsets",
107 first_err,
108 ));
109 }
110 } else {
111 let results: Vec<_> = join_all(restart_futures).await;
112 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
113 return Err(Error::kube_error(
114 "restart",
115 "StatefulSet",
116 namespace,
117 "replica certificate secret update statefulsets",
118 first_err,
119 ));
120 }
121 }
122
123 let rollout_futures = statefulset_names
124 .iter()
125 .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
126 try_join_all(rollout_futures).await?;
127
128 Ok(())
129}
130
131pub async fn reconcile_admins_secret(
132 kanidm: Arc<Kanidm>,
133 ctx: Arc<Context>,
134 status: &KanidmStatus,
135) -> Result<()> {
136 if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
137 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
138 kanidm.patch(&ctx, admins_secret).await?;
139 }
140 Ok(())
141}
142
143pub async fn reconcile_replication_secrets(
144 kanidm: Arc<Kanidm>,
145 ctx: Arc<Context>,
146 status: &KanidmStatus,
147) -> Result<()> {
148 let expected_secret_names = if kanidm.is_replication_enabled() {
149 status
150 .replica_statuses
151 .iter()
152 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
153 .collect::<Vec<_>>()
154 } else {
155 Vec::new()
156 };
157 let deprecated_secrets = ctx
158 .stores
159 .secret_store
160 .state()
161 .into_iter()
162 .filter(|secret| {
163 secret.namespace() == kanidm.namespace()
164 && kanidm.admins_secret_name() != secret.name_any()
165 && !expected_secret_names.contains(&secret.name_any())
166 && secret
167 .metadata
168 .labels
169 .as_ref()
170 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
171 })
172 .collect::<Vec<_>>();
173 let secret_delete_future = deprecated_secrets
174 .iter()
175 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
176 .collect::<TryJoinAll<_>>();
177 try_join!(secret_delete_future)?;
178
179 if kanidm.is_replication_enabled() {
180 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
181 let statefulset_names = status
182 .replica_statuses
183 .iter()
184 .map(|rs| rs.statefulset_name.clone())
185 .collect::<BTreeSet<_>>()
186 .into_iter()
187 .collect::<Vec<_>>();
188
189 let has_pending = status
190 .replica_statuses
191 .iter()
192 .any(|rs| rs.state == KanidmReplicaState::Pending);
193
194 let sts_api =
195 Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
196
197 if has_pending {
198 stream::iter(
199 status
200 .replica_statuses
201 .iter()
202 .filter(|rs| rs.state == KanidmReplicaState::Pending),
203 )
204 .then(|rs| async {
205 let secret = kanidm
206 .generate_replica_secret(ctx.clone(), &rs.pod_name)
207 .await?;
208 kanidm.patch(&ctx, secret).await
209 })
210 .collect::<Vec<_>>()
211 .await
212 .into_iter()
213 .collect::<Result<Vec<_>>>()?;
214
215 let restart_futures = status
216 .replica_statuses
217 .iter()
218 .map(|rs| sts_api.restart(&rs.statefulset_name));
219
220 if has_single_replica {
221 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
222 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
223 return Err(Error::kube_error(
224 "restart",
225 "StatefulSet",
226 kanidm.get_namespace(),
227 "pending replica statefulsets",
228 first_err,
229 ));
230 }
231 } else {
232 let results: Vec<_> = join_all(restart_futures).await;
233 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
234 return Err(Error::kube_error(
235 "restart",
236 "StatefulSet",
237 kanidm.get_namespace(),
238 "pending replica statefulsets",
239 first_err,
240 ));
241 }
242 }
243 }
244
245 let has_certificate_host_invalid = status
246 .replica_statuses
247 .iter()
248 .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
249
250 let has_certificate_expiring = status
251 .replica_statuses
252 .iter()
253 .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
254
255 let has_non_ready_replicas = status
256 .replica_statuses
257 .iter()
258 .any(|rs| rs.state != KanidmReplicaState::Ready);
259
260 let host_invalid_replicas: Vec<_> = status
261 .replica_statuses
262 .iter()
263 .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
264 .collect();
265
266 let cert_expiring_replicas: Vec<_> = status
267 .replica_statuses
268 .iter()
269 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
270 .collect();
271
272 if !host_invalid_replicas.is_empty() {
273 let restart_futures = host_invalid_replicas
274 .iter()
275 .map(|rs| sts_api.restart(&rs.statefulset_name));
276
277 if has_single_replica {
278 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
279 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
280 return Err(Error::kube_error(
281 "restart",
282 "StatefulSet",
283 kanidm.get_namespace(),
284 "certificate host invalid statefulsets",
285 first_err,
286 ));
287 }
288 } else {
289 let results: Vec<_> = join_all(restart_futures).await;
290 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
291 return Err(Error::kube_error(
292 "restart",
293 "StatefulSet",
294 kanidm.get_namespace(),
295 "certificate host invalid statefulsets",
296 first_err,
297 ));
298 }
299 }
300
301 let restarted_sts_names: Vec<_> = host_invalid_replicas
302 .iter()
303 .map(|rs| rs.statefulset_name.clone())
304 .collect::<BTreeSet<_>>()
305 .into_iter()
306 .collect();
307
308 let namespace = kanidm.get_namespace();
309 let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
310 wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
311 });
312 try_join_all(rollout_futures).await?;
313
314 let renew_futures = host_invalid_replicas
315 .iter()
316 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
317 let results: Vec<_> = join_all(renew_futures).await;
318 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
319 return Err(first_err);
320 }
321
322 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
323
324 let cert_futures = host_invalid_replicas.iter().map(|rs| async {
325 let cert = show_replica_cert_with_retries(
326 &kanidm,
327 ctx.clone(),
328 &rs.pod_name,
329 CERT_SHOW_RETRY_ATTEMPTS,
330 )
331 .await?;
332 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
333 kanidm.patch(&ctx, secret.clone()).await
334 });
335 try_join_all(cert_futures).await?;
336
337 restart_statefulsets_after_replica_secret_update(
338 ctx.clone(),
339 &sts_api,
340 &kanidm.get_namespace(),
341 &statefulset_names,
342 has_single_replica,
343 )
344 .await?;
345
346 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
347 }
348
349 if !cert_expiring_replicas.is_empty() {
350 let renew_futures = cert_expiring_replicas
351 .iter()
352 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
353 let results: Vec<_> = join_all(renew_futures).await;
354 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
355 return Err(first_err);
356 }
357
358 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
359
360 let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
361 let cert = show_replica_cert_with_retries(
362 &kanidm,
363 ctx.clone(),
364 &rs.pod_name,
365 CERT_SHOW_RETRY_ATTEMPTS,
366 )
367 .await?;
368 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
369 kanidm.patch(&ctx, secret.clone()).await
370 });
371 try_join_all(cert_futures).await?;
372
373 restart_statefulsets_after_replica_secret_update(
374 ctx.clone(),
375 &sts_api,
376 &kanidm.get_namespace(),
377 &statefulset_names,
378 has_single_replica,
379 )
380 .await?;
381
382 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
383 }
384
385 if has_non_ready_replicas
386 && !has_pending
387 && !has_certificate_host_invalid
388 && !has_certificate_expiring
389 {
390 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
391
392 let restart_futures = status
393 .replica_statuses
394 .iter()
395 .map(|rs| sts_api.restart(&rs.statefulset_name));
396
397 if has_single_replica {
398 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
399 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
400 return Err(Error::kube_error(
401 "restart",
402 "StatefulSet",
403 kanidm.get_namespace(),
404 "replica statefulsets",
405 first_err,
406 ));
407 }
408 } else {
409 let results: Vec<_> = join_all(restart_futures).await;
410 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
411 return Err(Error::kube_error(
412 "restart",
413 "StatefulSet",
414 kanidm.get_namespace(),
415 "replica statefulsets",
416 first_err,
417 ));
418 }
419 }
420 }
421 }
422
423 Ok(())
424}
425
426#[instrument(skip(ctx, kanidm))]
427pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
428 let trace_id = telemetry::get_trace_id();
429 Span::current().record("trace_id", field::display(&trace_id));
430 let _timer = ctx
431 .kaniop_ctx
432 .metrics
433 .reconcile_count_and_measure(&trace_id);
434 info!(msg = "reconciling Kanidm");
435
436 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
437 debug!(msg = "failed to reconcile status", %e);
438 ctx.kaniop_ctx.metrics.status_update_errors_inc();
439 e
440 })?;
441 let kanidm_api: Api<Kanidm> =
442 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
443 finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
444 match event {
445 Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
446 Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
447 }
448 })
449 .await
450 .or_else(|e| match e {
451 FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
452 debug!(msg = "resource already removed during finalizer cleanup");
453 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
454 }
455 _ => Err(Error::FinalizerError(
456 "failed on kanidm account finalizer".to_string(),
457 Box::new(e),
458 )),
459 })
460}
461
462async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
466 let secret_name = kanidm.effective_tls_secret_name();
467 let secret_api =
468 Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
469 let secret = secret_api
470 .get_opt(&secret_name)
471 .await
472 .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
473 Ok(secret.as_ref().map(hash_secret_data))
474}
475
476fn previous_tls_secret_hash(
481 kanidm: &Kanidm,
482 stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
483) -> Option<String> {
484 stateful_sets
485 .into_iter()
486 .filter(|sts| {
487 sts.namespace() == kanidm.namespace()
488 && sts
489 .metadata
490 .labels
491 .as_ref()
492 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
493 })
494 .find_map(|sts| {
495 sts.spec
496 .as_ref()?
497 .template
498 .metadata
499 .as_ref()?
500 .annotations
501 .as_ref()?
502 .get(TLS_SECRET_HASH_ANNOTATION)
503 .cloned()
504 })
505}
506
507fn is_headless_service(service: &Service) -> bool {
508 service
509 .spec
510 .as_ref()
511 .and_then(|spec| spec.cluster_ip.as_deref())
512 == Some("None")
513}
514
515fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
516 if is_headless_service(desired) != is_headless_service(current) {
517 return;
518 }
519 let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
520 else {
521 return;
522 };
523 if desired_spec.cluster_ip.is_none() {
524 desired_spec.cluster_ip = current_spec.cluster_ip.clone();
525 }
526 if desired_spec.cluster_ips.is_none() {
527 desired_spec.cluster_ips = current_spec.cluster_ips.clone();
528 }
529}
530
531async fn wait_for_statefulset_deletion(
532 client: kube::Client,
533 namespace: &str,
534 name: &str,
535) -> Result<()> {
536 let api = Api::<StatefulSet>::namespaced(client, namespace);
537 let start = std::time::Instant::now();
538 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
539 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
540
541 loop {
542 match api.get_opt(name).await {
543 Ok(None) => return Ok(()),
544 Ok(Some(_)) => {}
545 Err(e) => {
546 return Err(Error::kube_error(
547 "wait for deletion of",
548 "StatefulSet",
549 namespace,
550 name,
551 e,
552 ));
553 }
554 }
555
556 if start.elapsed() >= timeout {
557 return Err(Error::ReceiveOutput(format!(
558 "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
559 )));
560 }
561 sleep(poll_interval).await;
562 }
563}
564
565async fn reconcile_service(
566 kanidm: &Kanidm,
567 ctx: &Context,
568 mut desired: Service,
569) -> Result<Service> {
570 let namespace = kanidm.get_namespace();
571 let name = desired.name_any();
572 let desired_headless = is_headless_service(&desired);
573
574 if let Some(cached) = ctx
575 .stores
576 .service_store
577 .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
578 {
579 if is_headless_service(cached.as_ref()) == desired_headless {
580 preserve_defaulted_service_fields(&mut desired, cached.as_ref());
581 } else {
582 let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
583 let current = service_api
584 .get_opt(&name)
585 .await
586 .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
587
588 if let Some(current) = current {
589 let current_headless = is_headless_service(¤t);
590 if current_headless != desired_headless {
591 info!(
592 msg = "recreating Service to change headless mode",
593 resource.name = &name,
594 resource.namespace = &namespace,
595 current_headless,
596 desired_headless,
597 );
598 kanidm.delete(ctx, ¤t).await?;
599 ctx.kaniop_ctx
600 .metrics
601 .reconcile_deploy_delete_create_inc("Service", "headless_transition");
602 wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
603 .await?;
604 } else {
605 preserve_defaulted_service_fields(&mut desired, ¤t);
606 }
607 }
608 }
609 }
610
611 kanidm.patch(ctx, desired).await
612}
613
614async fn wait_for_service_deletion(
615 client: kube::Client,
616 namespace: &str,
617 name: &str,
618) -> Result<()> {
619 let api = Api::<Service>::namespaced(client, namespace);
620 let start = std::time::Instant::now();
621 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
622 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
623
624 loop {
625 match api.get_opt(name).await {
626 Ok(None) => return Ok(()),
627 Ok(Some(_)) => {}
628 Err(e) => {
629 return Err(Error::kube_error(
630 "wait for deletion of",
631 "Service",
632 namespace,
633 name,
634 e,
635 ));
636 }
637 }
638
639 if start.elapsed() >= timeout {
640 return Err(Error::ReceiveOutput(format!(
641 "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
642 )));
643 }
644 sleep(poll_interval).await;
645 }
646}
647
648async fn validate_statefulset_replacement(
649 api: &Api<StatefulSet>,
650 desired: &StatefulSet,
651 namespace: &str,
652 name: &str,
653) -> Result<()> {
654 let mut candidate = desired.clone();
655 candidate.metadata.name = None;
656 candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
657 candidate.metadata.namespace = None;
658 candidate.metadata.resource_version = None;
659 candidate.metadata.uid = None;
660 candidate.metadata.creation_timestamp = None;
661 candidate.metadata.generation = None;
662 candidate.metadata.managed_fields = None;
663 candidate.metadata.deletion_timestamp = None;
664 candidate.metadata.deletion_grace_period_seconds = None;
665 candidate.status = None;
666
667 api.create(
668 &PostParams {
669 dry_run: true,
670 ..Default::default()
671 },
672 &candidate,
673 )
674 .await
675 .map(|_| ())
676 .map_err(|e| {
677 Error::kube_error(
678 "validate replacement for",
679 "StatefulSet",
680 namespace,
681 name,
682 e,
683 )
684 })
685}
686
687async fn reconcile_statefulset(
688 kanidm: Arc<Kanidm>,
689 ctx: Arc<Context>,
690 desired: StatefulSet,
691) -> Result<StatefulSet> {
692 let namespace = kanidm.namespace().unwrap();
693 let name = desired.name_any();
694 let cached_current = ctx.stores.stateful_set_store.find(|current| {
695 current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
696 });
697
698 let cache_miss_422_error = if let Some(cached_current) = cached_current {
703 let mut cached_desired = desired.clone();
704 preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
705 if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
706 == StatefulSetApplyStrategy::Apply
707 {
708 return kanidm.apply(&ctx, cached_desired).await;
709 }
710 None
711 } else {
712 match kanidm.apply(&ctx, desired.clone()).await {
713 Ok(applied) => return Ok(applied),
714 Err(error) => {
715 if is_kube_422(&error) {
719 Some(error)
720 } else {
721 return Err(error);
722 }
723 }
724 }
725 };
726
727 let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
731 let Some(live_current) = api
732 .get_opt(&name)
733 .await
734 .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
735 else {
736 return match cache_miss_422_error {
737 Some(error) => Err(error),
738 None => kanidm.apply(&ctx, desired).await,
739 };
740 };
741
742 let mut live_desired = desired.clone();
745 preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
746 match classify_statefulset_change(&live_current, &live_desired) {
747 StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
748 Some(error) => Err(error),
749 None => kanidm.apply(&ctx, live_desired).await,
750 },
751 StatefulSetApplyStrategy::Recreate { .. } => {
752 validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
756
757 let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
760 Error::kube_error(
761 "re-read before deletion",
762 "StatefulSet",
763 &namespace,
764 &name,
765 e,
766 )
767 })?
768 else {
769 return kanidm.apply(&ctx, live_desired).await;
770 };
771 let mut latest_desired = live_desired.clone();
772 preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
773
774 match classify_statefulset_change(&latest_current, &latest_desired) {
775 StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
776 StatefulSetApplyStrategy::Recreate { immutable_fields } => {
777 validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
782 .await?;
783
784 info!(
785 msg = "recreating StatefulSet because immutable fields changed",
786 resource.name = %name,
787 resource.namespace = %namespace,
788 ?immutable_fields,
789 );
790 kanidm.delete(&ctx, &latest_current).await?;
791 ctx.kaniop_ctx
792 .metrics
793 .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
794 wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
795 .await?;
796 kanidm.apply(&ctx, latest_desired).await
797 }
798 }
799 }
800 }
801}
802
803fn is_kube_422(error: &Error) -> bool {
805 matches!(
806 error,
807 Error::KubeError(_, cause)
808 if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
809 )
810}
811
812#[cfg(test)]
813mod statefulset_recreate_recovery_tests {
814 use super::{Error, is_kube_422};
815
816 fn kube_api_error(code: u16) -> Error {
817 Error::KubeError(
818 "test error".to_string(),
819 Box::new(kube::Error::Api(Box::new(kube::error::Status {
820 code,
821 message: "test".to_string(),
822 reason: "test".to_string(),
823 ..Default::default()
824 }))),
825 )
826 }
827
828 #[test]
829 fn recreate_recovery_accepts_only_422() {
830 assert!(is_kube_422(&kube_api_error(422)));
831 assert!(!is_kube_422(&kube_api_error(403)));
832 assert!(!is_kube_422(&kube_api_error(429)));
833 assert!(!is_kube_422(&kube_api_error(500)));
834 }
835
836 #[test]
837 fn recreate_recovery_rejects_429_too_many_requests() {
838 assert!(!is_kube_422(&kube_api_error(429)));
839 }
840
841 #[test]
842 fn recreate_recovery_rejects_5xx_server_errors() {
843 assert!(!is_kube_422(&kube_api_error(500)));
844 assert!(!is_kube_422(&kube_api_error(502)));
845 assert!(!is_kube_422(&kube_api_error(503)));
846 assert!(!is_kube_422(&kube_api_error(504)));
847 }
848
849 #[test]
850 fn recreate_recovery_rejects_400_and_403() {
851 assert!(!is_kube_422(&kube_api_error(400)));
852 assert!(!is_kube_422(&kube_api_error(403)));
853 }
854}
855
856async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
857 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
858 let replication_secret_futures =
859 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
860
861 let sts_to_delete = {
862 let expected_sts_names = kanidm
863 .spec
864 .replica_groups
865 .iter()
866 .map(|rg| kanidm.statefulset_name(&rg.name))
867 .collect::<Vec<_>>();
868 ctx.stores
869 .stateful_set_store
870 .state()
871 .into_iter()
872 .filter(|sts| {
873 sts.namespace() == kanidm.namespace()
874 && sts
875 .metadata
876 .labels
877 .as_ref()
878 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
879 && !expected_sts_names.contains(&sts.name_any())
880 })
881 .collect::<Vec<_>>()
882 };
883 let sts_delete_futures = sts_to_delete
884 .iter()
885 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
886 .collect::<TryJoinAll<_>>();
887
888 let sts_futures = match kanidm.is_updatable(&status) {
889 true => {
890 let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
891 previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
892 });
893 kanidm
894 .spec
895 .replica_groups
896 .iter()
897 .map(|rg| {
898 let sts = kanidm.create_statefulset(rg, tls_secret_hash.as_deref())?;
899 Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
900 })
901 .collect::<Result<TryJoinAll<_>, _>>()?
902 }
903 false => {
904 let note = match status.version.as_ref() {
905 Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
906 format!(
907 "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
908 v.image_tag,
909 crate::version::KANIDM_CLIENT_VERSION
910 )
911 }
912 _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
913 };
914 let _ignore_error = ctx
915 .kaniop_ctx
916 .recorder
917 .publish(
918 &Event {
919 type_: EventType::Warning,
920 reason: "UpgradeBlocked".to_string(),
921 note: Some(note),
922 action: "ReconcileStatefulSet".to_string(),
923 secondary: None,
924 },
925 &kanidm.object_ref(&()),
926 )
927 .await
928 .map_err(|e| {
929 warn!(msg = "failed to publish KanidmError event", %e);
930 Error::KubeError("failed to publish event".to_string(), Box::new(e))
931 });
932 try_join_all([])
933 }
934 };
935 let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
936
937 let deprecated_rg_svcs = {
938 let expected_rg_svcs_names = kanidm
939 .spec
940 .replica_groups
941 .iter()
942 .filter(|rg| rg.services.is_some())
943 .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
944 .collect::<Vec<_>>();
945 ctx.stores
946 .service_store
947 .state()
948 .into_iter()
949 .filter(|svc| {
950 let labels = svc.metadata.labels.as_ref();
951 svc.namespace() == kanidm.namespace()
952 && !expected_rg_svcs_names.contains(&svc.name_any())
953 && labels.is_some_and(|l| {
954 l.contains_key(REPLICA_LABEL)
955 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
956 })
957 })
958 .collect::<Vec<_>>()
959 };
960 let rg_svcs_delete_futures = deprecated_rg_svcs
961 .iter()
962 .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
963 .collect::<TryJoinAll<_>>();
964
965 let rg_services_futures = kanidm
966 .spec
967 .replica_groups
968 .iter()
969 .filter(|rg| rg.services.is_some())
970 .flat_map(|rg| {
971 (0..rg.replicas).map(|i| {
972 let mut svc = kanidm.create_replica_group_service(rg, i);
973 if let Some(current) = ctx.stores.service_store.find(|current| {
974 current.namespace() == kanidm.namespace()
975 && current.name_any() == svc.name_any()
976 }) {
977 preserve_defaulted_service_fields(&mut svc, current.as_ref());
978 }
979 kanidm.patch(&ctx, svc)
980 })
981 })
982 .collect::<TryJoinAll<_>>();
983
984 let deprecated_ingresses = {
985 let names = [
986 kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
987 kanidm.generate_region_ingress_name(),
988 ]
989 .into_iter()
990 .flatten()
991 .collect::<Vec<_>>();
992 ctx.stores
993 .ingress_store
994 .state()
995 .into_iter()
996 .filter(|ing| {
997 ing.namespace() == kanidm.namespace()
998 && !names.contains(&ing.name_any())
999 && ing.metadata.labels == Some(kanidm.generate_labels())
1000 })
1001 .collect::<Vec<_>>()
1002 };
1003 let ingress_delete_futures = deprecated_ingresses
1004 .iter()
1005 .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1006 .collect::<TryJoinAll<_>>();
1007
1008 let ingress_futures = kanidm
1009 .create_ingress()
1010 .into_iter()
1011 .chain(kanidm.create_region_ingress())
1012 .map(|ingress| kanidm.patch(&ctx, ingress))
1013 .collect::<TryJoinAll<_>>();
1014
1015 let deprecated_http_routes = match &ctx.stores.http_route_store {
1016 Some(store) => {
1017 let expected_names = kanidm
1018 .spec
1019 .gateway
1020 .as_ref()
1021 .map(|_| vec![kanidm.name_any()])
1022 .unwrap_or_default();
1023 store
1024 .state()
1025 .into_iter()
1026 .filter(|route| {
1027 route.namespace() == kanidm.namespace()
1028 && !expected_names.contains(&route.name_any())
1029 && route.metadata.labels == Some(kanidm.generate_labels())
1030 })
1031 .collect::<Vec<_>>()
1032 }
1033 None => Vec::new(),
1034 };
1035 let http_route_delete_futures = deprecated_http_routes
1036 .iter()
1037 .map(|route| kanidm.delete(&ctx, route.as_ref()))
1038 .collect::<TryJoinAll<_>>();
1039
1040 let http_route_futures = if ctx.stores.http_route_store.is_some() {
1041 kanidm
1042 .create_http_route()
1043 .into_iter()
1044 .map(|route| kanidm.patch(&ctx, route))
1045 .collect::<TryJoinAll<_>>()
1046 } else {
1047 try_join_all(Vec::new())
1048 };
1049
1050 let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1051 Some(store) => {
1052 let expected_names = kanidm
1053 .spec
1054 .gateway
1055 .as_ref()
1056 .and_then(|g| g.backend_tls_policy.as_ref())
1057 .map(|_| vec![kanidm.name_any()])
1058 .unwrap_or_default();
1059 store
1060 .state()
1061 .into_iter()
1062 .filter(|policy| {
1063 policy.namespace() == kanidm.namespace()
1064 && !expected_names.contains(&policy.name_any())
1065 && policy.metadata.labels == Some(kanidm.generate_labels())
1066 })
1067 .collect::<Vec<_>>()
1068 }
1069 None => Vec::new(),
1070 };
1071 let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1072 .iter()
1073 .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1074 .collect::<TryJoinAll<_>>();
1075
1076 let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1077 kanidm
1078 .create_backend_tls_policy()
1079 .into_iter()
1080 .map(|policy| kanidm.patch(&ctx, policy))
1081 .collect::<TryJoinAll<_>>()
1082 } else {
1083 try_join_all(Vec::new())
1084 };
1085
1086 try_join!(
1087 sts_delete_futures,
1088 admin_secret_future,
1089 replication_secret_futures,
1090 sts_futures,
1091 service_future,
1092 rg_svcs_delete_futures,
1093 rg_services_futures,
1094 ingress_delete_futures,
1095 ingress_futures,
1096 http_route_delete_futures,
1097 http_route_futures,
1098 backend_tls_policy_delete_futures,
1099 backend_tls_policy_futures
1100 )?;
1101
1102 if is_kanidm_available(status.clone()) {
1103 let namespace = kanidm.namespace().unwrap();
1104 let name = kanidm.name_any();
1105 let system_client = crate::controller::kanidm::KanidmClients::create_client(
1106 &namespace,
1107 &name,
1108 crate::controller::kanidm::KanidmUser::Admin,
1109 ctx.kaniop_ctx.client.clone(),
1110 )
1111 .await?;
1112 reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
1113
1114 let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
1115 &namespace,
1116 &name,
1117 crate::controller::kanidm::KanidmUser::IdmAdmin,
1118 ctx.kaniop_ctx.client.clone(),
1119 )
1120 .await?;
1121 let mail_sender_status =
1122 reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
1123 let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1124 if mail_sender_status != current_mail_sender_status {
1125 let kanidm_api: Api<Kanidm> =
1126 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1127 let status_patch = serde_json::json!({
1128 "status": {
1129 "mailSender": mail_sender_status
1130 }
1131 });
1132 kanidm_api
1133 .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1134 .await
1135 .map_err(|e| {
1136 Error::KubeError(
1137 format!("failed to patch Kanidm/status {namespace}/{name}"),
1138 Box::new(e),
1139 )
1140 })?;
1141 }
1142 }
1143
1144 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
1145}
1146
1147async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
1148 debug!(msg = "cleanup");
1149
1150 cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1151
1152 let namespace = kanidm.namespace().unwrap();
1153 let name = kanidm.name_any();
1154
1155 if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1156 &namespace,
1157 &name,
1158 crate::controller::kanidm::KanidmUser::IdmAdmin,
1159 ctx.kaniop_ctx.client.clone(),
1160 )
1161 .await
1162 {
1163 cleanup_mail_sender_in_kanidm(&kanidm_client, &name).await?;
1164 }
1165
1166 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1167 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
1168}
1169
1170async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1171 let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1172 let start = std::time::Instant::now();
1173 let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1174 let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1175
1176 loop {
1177 let sts = sts_api.get(sts_name).await.map_err(|e| {
1178 Error::KubeError(
1179 format!("failed to get StatefulSet {namespace}/{sts_name}"),
1180 Box::new(e),
1181 )
1182 })?;
1183
1184 if sts.metadata.deletion_timestamp.is_some() {
1185 return Err(Error::ReceiveOutput(format!(
1186 "StatefulSet {namespace}/{sts_name} is being deleted"
1187 )));
1188 }
1189
1190 let spec = sts.spec.as_ref().ok_or_else(|| {
1191 Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1192 })?;
1193
1194 let status = match sts.status {
1195 Some(ref s) => s,
1196 None => {
1197 if start.elapsed() >= timeout {
1198 return Err(Error::ReceiveOutput(format!(
1199 "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1200 )));
1201 }
1202 sleep(poll_interval).await;
1203 continue;
1204 }
1205 };
1206
1207 let replicas = spec.replicas.unwrap_or(0);
1208
1209 if replicas == 0 {
1210 return Ok(());
1211 }
1212
1213 let updated_replicas = status.updated_replicas.unwrap_or(0);
1214 let current_revision = status.current_revision.as_deref().unwrap_or("");
1215 let update_revision = status.update_revision.as_deref().unwrap_or("");
1216 let ready_replicas = status.ready_replicas.unwrap_or(0);
1217
1218 if updated_replicas == replicas
1219 && current_revision == update_revision
1220 && ready_replicas == replicas
1221 {
1222 return Ok(());
1223 }
1224
1225 if start.elapsed() >= timeout {
1226 return Err(Error::ReceiveOutput(format!(
1227 "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1228 )));
1229 }
1230
1231 sleep(poll_interval).await;
1232 }
1233}
1234
1235async fn show_replica_cert_with_retries(
1236 kanidm: &Kanidm,
1237 ctx: Arc<Context>,
1238 pod_name: &str,
1239 max_attempts: u32,
1240) -> std::result::Result<String, Error> {
1241 let mut last_error = None;
1242 for attempt in 0..max_attempts {
1243 if attempt > 0 {
1244 sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1245 }
1246 match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1247 Ok(cert) => return Ok(cert),
1248 Err(e) => {
1249 info!(
1250 msg = format!(
1251 "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1252 attempt + 1,
1253 max_attempts
1254 )
1255 );
1256 last_error = Some(e);
1257 }
1258 }
1259 }
1260 Err(last_error.unwrap_or_else(|| {
1261 Error::ReceiveOutput(format!(
1262 "failed to get certificate for {pod_name} after {max_attempts} attempts"
1263 ))
1264 }))
1265}
1266
1267impl Kanidm {
1268 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1270 where
1271 K: Resource<Scope = NamespaceResourceScope>
1272 + Serialize
1273 + Clone
1274 + std::fmt::Debug
1275 + for<'de> Deserialize<'de>,
1276 <K as kube::Resource>::DynamicType: Default,
1277 <K as Resource>::Scope: std::marker::Sized,
1278 {
1279 self.kube_delete(
1280 ctx.kaniop_ctx.client.clone(),
1281 &ctx.kaniop_ctx.metrics,
1282 resource,
1283 )
1284 .await
1285 }
1286
1287 pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1290 where
1291 K: Resource<Scope = NamespaceResourceScope>
1292 + Serialize
1293 + Clone
1294 + std::fmt::Debug
1295 + for<'de> Deserialize<'de>,
1296 <K as kube::Resource>::DynamicType: Default,
1297 <K as Resource>::Scope: std::marker::Sized,
1298 {
1299 self.kube_apply(
1300 ctx.kaniop_ctx.client.clone(),
1301 resource,
1302 KANIDM_OPERATOR_NAME,
1303 )
1304 .await
1305 }
1306
1307 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1308 where
1309 K: Resource<Scope = NamespaceResourceScope>
1310 + Serialize
1311 + Clone
1312 + std::fmt::Debug
1313 + for<'de> Deserialize<'de>,
1314 <K as kube::Resource>::DynamicType: Default,
1315 <K as Resource>::Scope: std::marker::Sized,
1316 {
1317 self.kube_patch(
1318 ctx.kaniop_ctx.client.clone(),
1319 &ctx.kaniop_ctx.metrics,
1320 resource,
1321 KANIDM_OPERATOR_NAME,
1322 )
1323 .await
1324 }
1325
1326 #[inline]
1327 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1328 LABELS
1329 .clone()
1330 .into_iter()
1331 .chain([
1332 (INSTANCE_LABEL.to_string(), self.name_any()),
1333 (CLUSTER_LABEL.to_string(), self.name_any()),
1334 ])
1335 .collect()
1336 }
1337
1338 #[inline]
1339 fn generate_labels(&self) -> BTreeMap<String, String> {
1340 self.generate_resource_labels()
1341 }
1342
1343 #[inline]
1344 fn get_tls_secret_name(&self) -> String {
1345 format!("{}-tls", self.name_any())
1346 }
1347
1348 pub(crate) fn effective_tls_secret_name(&self) -> String {
1351 self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1352 self.spec
1353 .ingress
1354 .as_ref()
1355 .and_then(|i| i.tls_secret_name.clone())
1356 .unwrap_or_else(|| self.get_tls_secret_name())
1357 })
1358 }
1359
1360 #[inline]
1361 fn get_namespace(&self) -> String {
1362 self.namespace().unwrap()
1364 }
1365
1366 #[inline]
1367 fn is_replication_enabled(&self) -> bool {
1368 self.spec.replica_groups.len() > 1
1369 || self
1370 .spec
1371 .replica_groups
1372 .first()
1373 .is_some_and(|rg| rg.replicas > 1)
1374 || !self.spec.external_replication_nodes.is_empty()
1375 }
1376
1377 #[inline]
1378 fn is_updatable(&self, status: &KanidmStatus) -> bool {
1379 status
1380 .version
1381 .as_ref()
1382 .map(|v| {
1383 if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1384 return false;
1385 }
1386 let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1387 let versions = (parse_semver(¤t_tag), parse_semver(&v.image_tag));
1388 match v.upgrade_check_result {
1389 KanidmUpgradeCheckResult::Passed => {
1390 if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1391 minor <= v_minor + 1
1392 } else {
1393 true
1394 }
1395 }
1396 KanidmUpgradeCheckResult::Failed => {
1397 if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1398 versions
1399 {
1400 major == v_major && minor == v_minor && patch >= v_patch
1401 } else {
1402 v.image_tag == current_tag
1403 }
1404 }
1405 }
1406 })
1407 .unwrap_or(true)
1408 }
1409
1410 async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1411 let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1412 let pod = pod_api.get(pod_name).await.map_err(|e| {
1413 Error::KubeError(
1414 format!("failed to get pod {namespace}/{pod_name}"),
1415 Box::new(e),
1416 )
1417 })?;
1418 Ok(pod
1419 .status
1420 .as_ref()
1421 .and_then(|s| s.conditions.as_ref())
1422 .is_some_and(|conditions| {
1423 conditions
1424 .iter()
1425 .any(|c| c.type_ == "Ready" && c.status == "True")
1426 }))
1427 }
1428
1429 async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1430 let start = std::time::Instant::now();
1431 let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1432 let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1433
1434 loop {
1435 match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1436 Ok(true) => return Ok(()),
1437 Ok(false) => {}
1438 Err(e) => {
1439 if !e.is_retryable() {
1440 return Err(e);
1441 }
1442 trace!(msg = "pod readiness check failed, retrying", %e);
1443 }
1444 }
1445
1446 if start.elapsed() >= timeout {
1447 return Err(Error::ReceiveOutput(format!(
1448 "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1449 )));
1450 }
1451
1452 sleep(poll_interval).await;
1453 }
1454 }
1455
1456 async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1457 where
1458 I: IntoIterator<Item = T> + Debug,
1459 T: Into<String>,
1460 {
1461 let namespace = &self.get_namespace();
1462 trace!(
1463 msg = "pod exec",
1464 resource.name = &pod_name,
1465 resource.namespace = &namespace,
1466 ?command
1467 );
1468 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1469 let attached = pod
1470 .exec(
1471 pod_name,
1472 command,
1473 &AttachParams::default().container("kanidm"),
1474 )
1475 .await
1476 .map_err(|e| {
1477 Error::KubeError(
1478 format!("failed to exec pod {namespace}/{pod_name}"),
1479 Box::new(e),
1480 )
1481 })?;
1482 get_output(attached).await
1483 }
1484
1485 async fn exec_with_wait<I, T>(
1486 &self,
1487 ctx: Arc<Context>,
1488 pod_name: &str,
1489 command: I,
1490 ) -> Result<String>
1491 where
1492 I: IntoIterator<Item = T> + Debug,
1493 T: Into<String>,
1494 {
1495 let namespace = &self.get_namespace();
1496 Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1497 self.exec(ctx, pod_name, command).await
1498 }
1499
1500 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1501 where
1502 I: IntoIterator<Item = T> + Debug,
1503 T: Into<String>,
1504 {
1505 let first_replica_group = self
1507 .spec
1508 .replica_groups
1509 .first()
1510 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1511 let sts_name = self.statefulset_name(&first_replica_group.name);
1512 let pod_name = format!("{sts_name}-0");
1513 self.exec(ctx, &pod_name, command).await
1514 }
1515
1516 async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1517 where
1518 I: IntoIterator<Item = T> + Debug,
1519 T: Into<String>,
1520 {
1521 let first_replica_group = self
1522 .spec
1523 .replica_groups
1524 .first()
1525 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1526 let sts_name = self.statefulset_name(&first_replica_group.name);
1527 let pod_name = format!("{sts_name}-0");
1528 self.exec_with_wait(ctx, &pod_name, command).await
1529 }
1530}
1531
1532#[cfg(test)]
1533mod test {
1534 use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1535 use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1536
1537 use crate::controller::State;
1538 use crate::kanidm::controller::context::{Context, Stores};
1539 use crate::kanidm::crd::KanidmStatus;
1540
1541 use kaniop_k8s_util::error::Result;
1542 use kaniop_k8s_util::resources::hash_secret_data;
1543
1544 use std::collections::BTreeMap;
1545 use std::sync::Arc;
1546
1547 use http::{Request, Response};
1548 use k8s_openapi::ByteString;
1549 use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1550 use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1551 use k8s_openapi::api::networking::v1::Ingress;
1552 use kube::api::ObjectMeta;
1553 use kube::runtime::reflector::store::Writer;
1554 use kube::{Client, Resource, ResourceExt, client::Body};
1555 use opentelemetry::metrics::MeterProvider;
1556 use serde_json::json;
1557
1558 impl Kanidm {
1559 pub fn test() -> Self {
1561 let mut e = Kanidm::new(
1562 "test",
1563 serde_json::from_value(json!({
1564 "domain": "idm.example.com",
1565 "replicaGroups": [
1566 {
1567 "name": "default",
1568 "replicas": 1
1569 }
1570 ]
1571 }))
1572 .unwrap(),
1573 );
1574 e.meta_mut().namespace = Some("default".into());
1575 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1578 e
1579 }
1580
1581 pub fn with_ingress(mut self) -> Self {
1582 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1583 self
1584 }
1585
1586 pub fn with_replicas(mut self, replicas: i32) -> Self {
1588 self.spec.replica_groups[0].replicas = replicas;
1589 self
1590 }
1591
1592 pub fn needs_delete(mut self) -> Self {
1594 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1595 use k8s_openapi::jiff::Timestamp;
1596 let now = Timestamp::from_second(1491138632).unwrap(); self.meta_mut().deletion_timestamp = Some(Time(now));
1598 self
1599 }
1600
1601 pub fn with_status(mut self, status: KanidmStatus) -> Self {
1603 self.status = Some(status);
1604 self
1605 }
1606 }
1607
1608 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1610 pub struct ApiServerVerifier(ApiServerHandle);
1611
1612 pub enum Scenario {
1614 Create(Kanidm),
1615 CreateWithTwoReplicas(Kanidm),
1616 CreateWithIngress(Kanidm),
1617 CreateWithIngressWithTwoReplicas(Kanidm),
1618 CreateWithTlsSecret(Kanidm, Box<Secret>),
1619 }
1620
1621 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1622 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1623 .await
1624 .expect("timeout on mock apiserver")
1625 .expect("scenario succeeded")
1626 }
1627
1628 impl ApiServerVerifier {
1629 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1640 tokio::spawn(async move {
1641 match scenario {
1643 Scenario::Create(kanidm) => {
1644 self.handle_kanidm_get(kanidm.clone())
1645 .await
1646 .unwrap()
1647 .handle_kanidm_status_patch(kanidm.clone())
1648 .await
1649 .unwrap()
1650 .handle_tls_secret_get_not_found(kanidm.clone())
1651 .await
1652 .unwrap()
1653 .handle_statefulset_patch(kanidm.clone(), None)
1654 .await
1655 .unwrap()
1656 .handle_service_patch(kanidm.clone())
1657 .await
1658 }
1659 Scenario::CreateWithTwoReplicas(kanidm) => {
1660 self.handle_kanidm_get(kanidm.clone())
1661 .await
1662 .unwrap()
1663 .handle_kanidm_status_patch(kanidm.clone())
1664 .await
1665 .unwrap()
1666 .handle_tls_secret_get_not_found(kanidm.clone())
1667 .await
1668 .unwrap()
1669 .handle_statefulset_patch(kanidm.clone(), None)
1670 .await
1671 .unwrap()
1672 .handle_service_patch(kanidm.clone())
1673 .await
1674 }
1675 Scenario::CreateWithIngress(kanidm) => {
1676 self.handle_kanidm_get(kanidm.clone())
1677 .await
1678 .unwrap()
1679 .handle_kanidm_status_patch(kanidm.clone())
1680 .await
1681 .unwrap()
1682 .handle_tls_secret_get_not_found(kanidm.clone())
1683 .await
1684 .unwrap()
1685 .handle_statefulset_patch(kanidm.clone(), None)
1686 .await
1687 .unwrap()
1688 .handle_service_patch(kanidm.clone())
1689 .await
1690 .unwrap()
1691 .handle_ingress_patch(kanidm.clone())
1692 .await
1693 }
1694 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1695 self.handle_kanidm_get(kanidm.clone())
1696 .await
1697 .unwrap()
1698 .handle_kanidm_status_patch(kanidm.clone())
1699 .await
1700 .unwrap()
1701 .handle_tls_secret_get_not_found(kanidm.clone())
1702 .await
1703 .unwrap()
1704 .handle_statefulset_patch(kanidm.clone(), None)
1705 .await
1706 .unwrap()
1707 .handle_service_patch(kanidm.clone())
1708 .await
1709 .unwrap()
1710 .handle_ingress_patch(kanidm.clone())
1711 .await
1712 }
1713 Scenario::CreateWithTlsSecret(kanidm, secret) => {
1714 let expected_hash = hash_secret_data(&secret);
1715 self.handle_kanidm_get(kanidm.clone())
1716 .await
1717 .unwrap()
1718 .handle_kanidm_status_patch(kanidm.clone())
1719 .await
1720 .unwrap()
1721 .handle_tls_secret_get(kanidm.clone(), *secret)
1722 .await
1723 .unwrap()
1724 .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1725 .await
1726 .unwrap()
1727 .handle_service_patch(kanidm.clone())
1728 .await
1729 }
1730 }
1731 .expect("scenario completed without errors");
1732 })
1733 }
1734
1735 async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1736 let (request, send) = self.0.next_request().await.expect("service not called");
1737 assert_eq!(request.method(), http::Method::GET);
1738 let expected_uri_prefix = format!(
1739 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1740 kanidm.name_any()
1741 );
1742 let uri = request.uri().to_string();
1743 assert!(
1744 uri.starts_with(&expected_uri_prefix),
1745 "expected uri to start with {}, got {}",
1746 expected_uri_prefix,
1747 uri
1748 );
1749 let response = serde_json::to_vec(&kanidm).unwrap();
1750 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1751 Ok(self)
1752 }
1753
1754 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1755 let (request, send) = self.0.next_request().await.expect("service not called");
1756 assert_eq!(request.method(), http::Method::PATCH);
1757 assert_eq!(
1758 request.uri().to_string(),
1759 format!(
1760 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1761 kanidm.name_any()
1762 )
1763 );
1764
1765 let req_body = request.into_body().collect_bytes().await.unwrap();
1766 let json: serde_json::Value =
1767 serde_json::from_slice(&req_body).expect("patch object is json");
1768 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1769 .expect("valid kanidm status");
1770 let mut kanidm = Kanidm::test();
1771 kanidm.status = Some(status.clone());
1772 let response = serde_json::to_vec(&kanidm).unwrap();
1773 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1775 Ok(self)
1776 }
1777
1778 async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1779 let (request, send) = self.0.next_request().await.expect("service not called");
1780 assert_eq!(request.method(), http::Method::GET);
1781 let expected_uri_prefix = format!(
1782 "/api/v1/namespaces/default/secrets/{}",
1783 kanidm.effective_tls_secret_name()
1784 );
1785 let uri = request.uri().to_string();
1786 assert!(
1787 uri.starts_with(&expected_uri_prefix),
1788 "expected uri to start with {expected_uri_prefix}, got {uri}"
1789 );
1790 let status = serde_json::json!({
1791 "kind": "Status",
1792 "apiVersion": "v1",
1793 "metadata": {},
1794 "status": "Failure",
1795 "message": "secret not found",
1796 "reason": "NotFound",
1797 "code": 404
1798 });
1799 send.send_response(
1800 Response::builder()
1801 .status(404)
1802 .body(Body::from(serde_json::to_vec(&status).unwrap()))
1803 .unwrap(),
1804 );
1805 Ok(self)
1806 }
1807
1808 async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1809 let (request, send) = self.0.next_request().await.expect("service not called");
1810 assert_eq!(request.method(), http::Method::GET);
1811 let expected_uri_prefix = format!(
1812 "/api/v1/namespaces/default/secrets/{}",
1813 kanidm.effective_tls_secret_name()
1814 );
1815 let uri = request.uri().to_string();
1816 assert!(
1817 uri.starts_with(&expected_uri_prefix),
1818 "expected uri to start with {expected_uri_prefix}, got {uri}"
1819 );
1820 let response = serde_json::to_vec(&secret).unwrap();
1821 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1822 Ok(self)
1823 }
1824
1825 async fn handle_statefulset_patch(
1826 mut self,
1827 kanidm: Kanidm,
1828 expected_tls_hash: Option<String>,
1829 ) -> Result<Self> {
1830 for rg in kanidm.spec.replica_groups.iter() {
1831 let (request, send) = self.0.next_request().await.expect("service not called");
1832 assert_eq!(request.method(), http::Method::PATCH);
1833 assert_eq!(
1834 request.uri().to_string(),
1835 format!(
1836 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1837 kanidm.statefulset_name(&rg.name)
1838 )
1839 );
1840 let req_body = request.into_body().collect_bytes().await.unwrap();
1841 let json: serde_json::Value =
1842 serde_json::from_slice(&req_body).expect("patch object is json");
1843 let statefulset: StatefulSet =
1844 serde_json::from_value(json).expect("valid statefulset");
1845 assert_eq!(
1846 statefulset.clone().spec.unwrap().replicas.unwrap(),
1847 rg.replicas
1848 );
1849 let tls_hash_annotation = statefulset
1850 .spec
1851 .as_ref()
1852 .unwrap()
1853 .template
1854 .metadata
1855 .as_ref()
1856 .unwrap()
1857 .annotations
1858 .as_ref()
1859 .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
1860 .cloned();
1861 assert_eq!(tls_hash_annotation, expected_tls_hash);
1862 let response = serde_json::to_vec(&statefulset).unwrap();
1863 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1865 }
1866
1867 Ok(self)
1868 }
1869
1870 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1871 let (request, send) = self.0.next_request().await.expect("service not called");
1872 assert_eq!(request.method(), http::Method::PATCH);
1873 assert_eq!(
1874 request.uri().to_string(),
1875 format!(
1876 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1877 kanidm.name_any()
1878 )
1879 );
1880
1881 let req_body = request.into_body().collect_bytes().await.unwrap();
1882 let json: serde_json::Value =
1883 serde_json::from_slice(&req_body).expect("patch object is json");
1884 let service: Service = serde_json::from_value(json).expect("valid service");
1885 let response = serde_json::to_vec(&service).unwrap();
1886 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1888 Ok(self)
1889 }
1890
1891 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1892 let (request, send) = self.0.next_request().await.expect("service not called");
1893 assert_eq!(request.method(), http::Method::PATCH);
1894 assert_eq!(
1895 request.uri().to_string(),
1896 format!(
1897 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1898 kanidm.name_any()
1899 )
1900 );
1901
1902 let req_body = request.into_body().collect_bytes().await.unwrap();
1903 let json: serde_json::Value =
1904 serde_json::from_slice(&req_body).expect("patch object is json");
1905 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1906 let response = serde_json::to_vec(&ingress).unwrap();
1907 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1909 Ok(self)
1910 }
1911 }
1912
1913 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1914 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1915 let mock_client = Client::new(mock_service, "default");
1916 let stores = Stores {
1917 stateful_set_store: Writer::default().as_reader(),
1918 service_store: Writer::default().as_reader(),
1919 ingress_store: Writer::default().as_reader(),
1920 secret_store: Writer::default().as_reader(),
1921 http_route_store: Some(Writer::default().as_reader()),
1922 backend_tls_policy_store: Some(Writer::default().as_reader()),
1923 deployment_store: Writer::default().as_reader(),
1924 config_map_store: Writer::default().as_reader(),
1925 };
1926 let controller_id = "test";
1927
1928 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
1930 let meter = provider.meter("test");
1931 let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
1932
1933 let state = State::new(
1934 metrics,
1935 Writer::default().as_reader(),
1936 Writer::default().as_reader(),
1937 None,
1938 );
1939 let ctx = Arc::new(Context::new(
1940 state.to_context(mock_client, controller_id),
1941 stores,
1942 ));
1943 (ctx, ApiServerVerifier(handle))
1944 }
1945
1946 #[tokio::test]
1947 async fn kanidm_create() {
1948 let (testctx, fakeserver) = get_test_context();
1949 let kanidm = Kanidm::test();
1950 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
1951 reconcile_kanidm(Arc::new(kanidm), testctx)
1952 .await
1953 .expect("reconciler");
1954 timeout_after_1s(mocksrv).await;
1955 }
1956
1957 #[tokio::test]
1958 async fn kanidm_create_with_two_replicas() {
1959 let (testctx, fakeserver) = get_test_context();
1960 let kanidm = Kanidm::test().with_replicas(2);
1961 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
1962 reconcile_kanidm(Arc::new(kanidm), testctx)
1963 .await
1964 .expect("reconciler");
1965 timeout_after_1s(mocksrv).await;
1966 }
1967
1968 #[tokio::test]
1969 async fn kanidm_create_with_ingress() {
1970 let (testctx, fakeserver) = get_test_context();
1971 let kanidm = Kanidm::test().with_ingress();
1972 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
1973 reconcile_kanidm(Arc::new(kanidm), testctx)
1974 .await
1975 .expect("reconciler");
1976 timeout_after_1s(mocksrv).await;
1977 }
1978
1979 #[tokio::test]
1980 async fn kanidm_create_with_ingress_with_two_replicas() {
1981 let (testctx, fakeserver) = get_test_context();
1982 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
1983 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
1984 reconcile_kanidm(Arc::new(kanidm), testctx)
1985 .await
1986 .expect("reconciler");
1987 timeout_after_1s(mocksrv).await;
1988 }
1989
1990 fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
1991 Arc::new(StatefulSet {
1992 metadata: ObjectMeta {
1993 name: Some(format!("{cluster}-default")),
1994 namespace: Some(namespace.to_string()),
1995 labels: Some(BTreeMap::from([(
1996 CLUSTER_LABEL.to_string(),
1997 cluster.to_string(),
1998 )])),
1999 ..Default::default()
2000 },
2001 spec: Some(StatefulSetSpec {
2002 template: PodTemplateSpec {
2003 metadata: Some(ObjectMeta {
2004 annotations: tls_hash.map(|h| {
2005 BTreeMap::from([(
2006 TLS_SECRET_HASH_ANNOTATION.to_string(),
2007 h.to_string(),
2008 )])
2009 }),
2010 ..Default::default()
2011 }),
2012 ..Default::default()
2013 },
2014 ..Default::default()
2015 }),
2016 ..Default::default()
2017 })
2018 }
2019
2020 #[test]
2021 fn test_previous_tls_secret_hash() {
2022 let kanidm = Kanidm::test();
2023
2024 let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2026 assert_eq!(
2027 previous_tls_secret_hash(&kanidm, stss),
2028 Some("abc123".to_string())
2029 );
2030
2031 let stss = vec![statefulset_of("test", "default", None)];
2033 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2034
2035 let stss = vec![
2037 statefulset_of("other", "default", Some("abc123")),
2038 statefulset_of("test", "other-ns", Some("abc123")),
2039 ];
2040 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2041 }
2042
2043 #[tokio::test]
2044 async fn kanidm_create_with_tls_secret() {
2045 let (testctx, fakeserver) = get_test_context();
2046 let kanidm = Kanidm::test();
2047 let secret = Secret {
2048 metadata: ObjectMeta {
2049 name: Some(kanidm.effective_tls_secret_name()),
2050 namespace: Some("default".to_string()),
2051 ..Default::default()
2052 },
2053 data: Some(BTreeMap::from([
2054 ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2055 ("tls.key".to_string(), ByteString(b"key".to_vec())),
2056 ])),
2057 ..Default::default()
2058 };
2059 let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2060 kanidm.clone(),
2061 Box::new(secret),
2062 ));
2063 reconcile_kanidm(Arc::new(kanidm), testctx)
2064 .await
2065 .expect("reconciler");
2066 timeout_after_1s(mocksrv).await;
2067 }
2068}