1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13 cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::{SECRET_TYPE_LABEL, SecretExt, SecretType};
20use self::service::ServiceExt;
21use self::statefulset::{
22 StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23 preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_backup_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::kanidm::KanidmUser;
31use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
32use crate::kanidm::crd::{
33 Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
34};
35use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
36use crate::telemetry;
37
38use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
39use kaniop_k8s_util::client::get_output;
40use kaniop_k8s_util::error::{Error, Result};
41use kaniop_k8s_util::parse::parse_semver;
42use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
43
44use std::collections::{BTreeMap, BTreeSet};
45use std::fmt::Debug;
46use std::sync::{Arc, LazyLock};
47
48use futures::future::{TryJoinAll, join_all, try_join_all};
49use futures::stream::{self, StreamExt};
50use futures::try_join;
51use k8s_openapi::NamespaceResourceScope;
52use k8s_openapi::api::apps::v1::StatefulSet;
53use k8s_openapi::api::core::v1::{Pod, Secret, Service};
54use kube::Resource;
55use kube::ResourceExt;
56use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
57use kube::runtime::controller::Action;
58use kube::runtime::events::{Event, EventType};
59use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
60use serde::{Deserialize, Serialize};
61use tokio::time::{Duration, sleep};
62use tracing::{Span, debug, field, info, instrument, trace, warn};
63
64const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
65const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
66const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
67const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
68const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
69const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
70const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
71const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
72const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
73
74pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
75const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
76pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
77
78const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
79
80static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
81 BTreeMap::from([
82 (NAME_LABEL.to_string(), "kanidm".to_string()),
83 (
84 MANAGED_BY_LABEL.to_string(),
85 format!("kaniop-{CONTROLLER_ID}"),
86 ),
87 ])
88});
89
90async fn restart_statefulsets_after_replica_secret_update(
91 ctx: Arc<Context>,
92 sts_api: &Api<StatefulSet>,
93 namespace: &str,
94 statefulset_names: &[String],
95 sequential: bool,
96) -> Result<()> {
97 let restart_futures = statefulset_names
100 .iter()
101 .map(|sts_name| sts_api.restart(sts_name));
102
103 if sequential {
104 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
105 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
106 return Err(Error::kube_error(
107 "restart",
108 "StatefulSet",
109 namespace,
110 "replica certificate secret update statefulsets",
111 first_err,
112 ));
113 }
114 } else {
115 let results: Vec<_> = join_all(restart_futures).await;
116 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
117 return Err(Error::kube_error(
118 "restart",
119 "StatefulSet",
120 namespace,
121 "replica certificate secret update statefulsets",
122 first_err,
123 ));
124 }
125 }
126
127 let rollout_futures = statefulset_names
128 .iter()
129 .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
130 try_join_all(rollout_futures).await?;
131
132 Ok(())
133}
134
135pub async fn reconcile_admins_secret(
136 kanidm: Arc<Kanidm>,
137 ctx: Arc<Context>,
138 status: &KanidmStatus,
139) -> Result<bool> {
140 if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
141 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
142 kanidm.patch(&ctx, admins_secret).await?;
143 return Ok(true);
144 }
145 Ok(false)
146}
147
148pub async fn reconcile_replication_secrets(
149 kanidm: Arc<Kanidm>,
150 ctx: Arc<Context>,
151 status: &KanidmStatus,
152) -> Result<bool> {
153 let mut changed = false;
154 let expected_secret_names = if kanidm.is_replication_enabled() {
155 status
156 .replica_statuses
157 .iter()
158 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
159 .collect::<Vec<_>>()
160 } else {
161 Vec::new()
162 };
163 let replica_cert_label =
164 serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
165 let deprecated_secrets = ctx
166 .stores
167 .secret_store
168 .state()
169 .into_iter()
170 .filter(|secret| {
171 secret.namespace() == kanidm.namespace()
172 && kanidm.admins_secret_name() != secret.name_any()
173 && !expected_secret_names.contains(&secret.name_any())
174 && secret.metadata.labels.as_ref().is_some_and(|l| {
175 l.get(SECRET_TYPE_LABEL) == Some(&replica_cert_label)
176 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
177 })
178 })
179 .collect::<Vec<_>>();
180 if !deprecated_secrets.is_empty() {
181 changed = true;
182 }
183 let secret_delete_future = deprecated_secrets
184 .iter()
185 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
186 .collect::<TryJoinAll<_>>();
187 try_join!(secret_delete_future)?;
188
189 if kanidm.is_replication_enabled() {
190 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
191 let statefulset_names = status
192 .replica_statuses
193 .iter()
194 .map(|rs| rs.statefulset_name.clone())
195 .collect::<BTreeSet<_>>()
196 .into_iter()
197 .collect::<Vec<_>>();
198
199 let has_pending = status
200 .replica_statuses
201 .iter()
202 .any(|rs| rs.state == KanidmReplicaState::Pending);
203
204 let sts_api =
205 Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
206
207 if has_pending {
208 changed = true;
209 stream::iter(
210 status
211 .replica_statuses
212 .iter()
213 .filter(|rs| rs.state == KanidmReplicaState::Pending),
214 )
215 .then(|rs| async {
216 let secret = kanidm
217 .generate_replica_secret(ctx.clone(), &rs.pod_name)
218 .await?;
219 kanidm.patch(&ctx, secret).await
220 })
221 .collect::<Vec<_>>()
222 .await
223 .into_iter()
224 .collect::<Result<Vec<_>>>()?;
225
226 let restart_futures = status
227 .replica_statuses
228 .iter()
229 .map(|rs| sts_api.restart(&rs.statefulset_name));
230
231 if has_single_replica {
232 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
233 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
234 return Err(Error::kube_error(
235 "restart",
236 "StatefulSet",
237 kanidm.get_namespace(),
238 "pending replica statefulsets",
239 first_err,
240 ));
241 }
242 } else {
243 let results: Vec<_> = join_all(restart_futures).await;
244 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
245 return Err(Error::kube_error(
246 "restart",
247 "StatefulSet",
248 kanidm.get_namespace(),
249 "pending replica statefulsets",
250 first_err,
251 ));
252 }
253 }
254 }
255
256 let has_certificate_host_invalid = status
257 .replica_statuses
258 .iter()
259 .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
260
261 let has_certificate_expiring = status
262 .replica_statuses
263 .iter()
264 .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
265
266 let has_non_ready_replicas = status
267 .replica_statuses
268 .iter()
269 .any(|rs| rs.state != KanidmReplicaState::Ready);
270
271 let host_invalid_replicas: Vec<_> = status
272 .replica_statuses
273 .iter()
274 .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
275 .collect();
276
277 let cert_expiring_replicas: Vec<_> = status
278 .replica_statuses
279 .iter()
280 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
281 .collect();
282
283 if !host_invalid_replicas.is_empty() {
284 changed = true;
285 let restart_futures = host_invalid_replicas
286 .iter()
287 .map(|rs| sts_api.restart(&rs.statefulset_name));
288
289 if has_single_replica {
290 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
291 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
292 return Err(Error::kube_error(
293 "restart",
294 "StatefulSet",
295 kanidm.get_namespace(),
296 "certificate host invalid statefulsets",
297 first_err,
298 ));
299 }
300 } else {
301 let results: Vec<_> = join_all(restart_futures).await;
302 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
303 return Err(Error::kube_error(
304 "restart",
305 "StatefulSet",
306 kanidm.get_namespace(),
307 "certificate host invalid statefulsets",
308 first_err,
309 ));
310 }
311 }
312
313 let restarted_sts_names: Vec<_> = host_invalid_replicas
314 .iter()
315 .map(|rs| rs.statefulset_name.clone())
316 .collect::<BTreeSet<_>>()
317 .into_iter()
318 .collect();
319
320 let namespace = kanidm.get_namespace();
321 let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
322 wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
323 });
324 try_join_all(rollout_futures).await?;
325
326 let renew_futures = host_invalid_replicas
327 .iter()
328 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
329 let results: Vec<_> = join_all(renew_futures).await;
330 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
331 return Err(first_err);
332 }
333
334 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
335
336 let cert_futures = host_invalid_replicas.iter().map(|rs| async {
337 let cert = show_replica_cert_with_retries(
338 &kanidm,
339 ctx.clone(),
340 &rs.pod_name,
341 CERT_SHOW_RETRY_ATTEMPTS,
342 )
343 .await?;
344 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
345 kanidm.patch(&ctx, secret.clone()).await
346 });
347 try_join_all(cert_futures).await?;
348
349 restart_statefulsets_after_replica_secret_update(
350 ctx.clone(),
351 &sts_api,
352 &kanidm.get_namespace(),
353 &statefulset_names,
354 has_single_replica,
355 )
356 .await?;
357
358 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
359 }
360
361 if !cert_expiring_replicas.is_empty() {
362 changed = true;
363 let renew_futures = cert_expiring_replicas
364 .iter()
365 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
366 let results: Vec<_> = join_all(renew_futures).await;
367 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
368 return Err(first_err);
369 }
370
371 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
372
373 let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
374 let cert = show_replica_cert_with_retries(
375 &kanidm,
376 ctx.clone(),
377 &rs.pod_name,
378 CERT_SHOW_RETRY_ATTEMPTS,
379 )
380 .await?;
381 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
382 kanidm.patch(&ctx, secret.clone()).await
383 });
384 try_join_all(cert_futures).await?;
385
386 restart_statefulsets_after_replica_secret_update(
387 ctx.clone(),
388 &sts_api,
389 &kanidm.get_namespace(),
390 &statefulset_names,
391 has_single_replica,
392 )
393 .await?;
394
395 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
396 }
397
398 if has_non_ready_replicas
399 && !has_pending
400 && !has_certificate_host_invalid
401 && !has_certificate_expiring
402 {
403 changed = true;
404 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
405
406 let restart_futures = status
407 .replica_statuses
408 .iter()
409 .map(|rs| sts_api.restart(&rs.statefulset_name));
410
411 if has_single_replica {
412 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
413 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
414 return Err(Error::kube_error(
415 "restart",
416 "StatefulSet",
417 kanidm.get_namespace(),
418 "replica statefulsets",
419 first_err,
420 ));
421 }
422 } else {
423 let results: Vec<_> = join_all(restart_futures).await;
424 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
425 return Err(Error::kube_error(
426 "restart",
427 "StatefulSet",
428 kanidm.get_namespace(),
429 "replica statefulsets",
430 first_err,
431 ));
432 }
433 }
434 }
435 }
436
437 Ok(changed)
438}
439
440#[instrument(skip(ctx, kanidm))]
441pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
442 let trace_id = telemetry::get_trace_id();
443 Span::current().record("trace_id", field::display(&trace_id));
444 let _timer = ctx
445 .kaniop_ctx
446 .metrics
447 .reconcile_count_and_measure(&trace_id);
448 info!("reconciling Kanidm");
449
450 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
451 debug!(%e, "failed to reconcile status");
452 ctx.kaniop_ctx.metrics.status_update_errors_inc();
453 e
454 })?;
455 let kanidm_api: Api<Kanidm> =
456 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
457 let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
458 let outcome_clone = outcome.clone();
459 let kanidm_for_finalizer = kanidm.clone();
460 let ctx_for_finalizer = ctx.clone();
461 let action = finalizer(
462 &kanidm_api,
463 KANIDM_FINALIZER,
464 kanidm_for_finalizer,
465 move |event| {
466 let outcome = outcome_clone.clone();
467 let ctx = ctx_for_finalizer.clone();
468 let status = status.clone();
469 async move {
470 match event {
471 Finalizer::Apply(kanidm) => {
472 let (action, changed) = reconcile(kanidm, ctx, status).await?;
473 outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
474 Ok(action)
475 }
476 Finalizer::Cleanup(kanidm) => {
477 let (action, changed) = cleanup(kanidm, ctx).await?;
478 outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
479 Ok(action)
480 }
481 }
482 }
483 },
484 )
485 .await
486 .or_else(|e| match e {
487 FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
488 debug!("resource already removed during finalizer cleanup");
489 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
490 }
491 _ => Err(Error::FinalizerError(
492 "failed on kanidm account finalizer".to_string(),
493 Box::new(e),
494 )),
495 });
496
497 if let Err(update_err) = kanidm.update_status(ctx.clone()).await {
500 debug!(%update_err, "failed to update status after reconciliation");
501 ctx.kaniop_ctx.metrics.status_update_errors_inc();
502 }
503
504 let action = action?;
505 let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
506 Ok((action, changed))
507}
508
509async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
513 let secret_name = kanidm.effective_tls_secret_name();
514 let secret_api =
515 Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
516 let secret = secret_api
517 .get_opt(&secret_name)
518 .await
519 .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
520 Ok(secret.as_ref().map(hash_secret_data))
521}
522
523fn previous_tls_secret_hash(
528 kanidm: &Kanidm,
529 stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
530) -> Option<String> {
531 stateful_sets
532 .into_iter()
533 .filter(|sts| {
534 sts.namespace() == kanidm.namespace()
535 && sts
536 .metadata
537 .labels
538 .as_ref()
539 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
540 })
541 .find_map(|sts| {
542 sts.spec
543 .as_ref()?
544 .template
545 .metadata
546 .as_ref()?
547 .annotations
548 .as_ref()?
549 .get(TLS_SECRET_HASH_ANNOTATION)
550 .cloned()
551 })
552}
553
554fn is_headless_service(service: &Service) -> bool {
555 service
556 .spec
557 .as_ref()
558 .and_then(|spec| spec.cluster_ip.as_deref())
559 == Some("None")
560}
561
562fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
563 if is_headless_service(desired) != is_headless_service(current) {
564 return;
565 }
566 let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
567 else {
568 return;
569 };
570 if desired_spec.cluster_ip.is_none() {
571 desired_spec.cluster_ip = current_spec.cluster_ip.clone();
572 }
573 if desired_spec.cluster_ips.is_none() {
574 desired_spec.cluster_ips = current_spec.cluster_ips.clone();
575 }
576}
577
578async fn wait_for_statefulset_deletion(
579 client: kube::Client,
580 namespace: &str,
581 name: &str,
582) -> Result<()> {
583 let api = Api::<StatefulSet>::namespaced(client, namespace);
584 let start = std::time::Instant::now();
585 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
586 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
587
588 loop {
589 match api.get_opt(name).await {
590 Ok(None) => return Ok(()),
591 Ok(Some(_)) => {}
592 Err(e) => {
593 return Err(Error::kube_error(
594 "wait for deletion of",
595 "StatefulSet",
596 namespace,
597 name,
598 e,
599 ));
600 }
601 }
602
603 if start.elapsed() >= timeout {
604 return Err(Error::ReceiveOutput(format!(
605 "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
606 )));
607 }
608 sleep(poll_interval).await;
609 }
610}
611
612async fn reconcile_service(
613 kanidm: &Kanidm,
614 ctx: &Context,
615 mut desired: Service,
616) -> Result<Service> {
617 let namespace = kanidm.get_namespace();
618 let name = desired.name_any();
619 let desired_headless = is_headless_service(&desired);
620
621 if let Some(cached) = ctx
622 .stores
623 .service_store
624 .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
625 {
626 if is_headless_service(cached.as_ref()) == desired_headless {
627 preserve_defaulted_service_fields(&mut desired, cached.as_ref());
628 } else {
629 let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
630 let current = service_api
631 .get_opt(&name)
632 .await
633 .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
634
635 if let Some(current) = current {
636 let current_headless = is_headless_service(¤t);
637 if current_headless != desired_headless {
638 info!(
639 resource.name = &name,
640 resource.namespace = &namespace,
641 current_headless,
642 desired_headless,
643 "recreating Service to change headless mode"
644 );
645 kanidm.delete(ctx, ¤t).await?;
646 ctx.kaniop_ctx
647 .metrics
648 .reconcile_deploy_delete_create_inc("Service", "headless_transition");
649 wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
650 .await?;
651 } else {
652 preserve_defaulted_service_fields(&mut desired, ¤t);
653 }
654 }
655 }
656 }
657
658 kanidm.patch(ctx, desired).await
659}
660
661async fn wait_for_service_deletion(
662 client: kube::Client,
663 namespace: &str,
664 name: &str,
665) -> Result<()> {
666 let api = Api::<Service>::namespaced(client, namespace);
667 let start = std::time::Instant::now();
668 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
669 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
670
671 loop {
672 match api.get_opt(name).await {
673 Ok(None) => return Ok(()),
674 Ok(Some(_)) => {}
675 Err(e) => {
676 return Err(Error::kube_error(
677 "wait for deletion of",
678 "Service",
679 namespace,
680 name,
681 e,
682 ));
683 }
684 }
685
686 if start.elapsed() >= timeout {
687 return Err(Error::ReceiveOutput(format!(
688 "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
689 )));
690 }
691 sleep(poll_interval).await;
692 }
693}
694
695async fn validate_statefulset_replacement(
696 api: &Api<StatefulSet>,
697 desired: &StatefulSet,
698 namespace: &str,
699 name: &str,
700) -> Result<()> {
701 let mut candidate = desired.clone();
702 candidate.metadata.name = None;
703 candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
704 candidate.metadata.namespace = None;
705 candidate.metadata.resource_version = None;
706 candidate.metadata.uid = None;
707 candidate.metadata.creation_timestamp = None;
708 candidate.metadata.generation = None;
709 candidate.metadata.managed_fields = None;
710 candidate.metadata.deletion_timestamp = None;
711 candidate.metadata.deletion_grace_period_seconds = None;
712 candidate.status = None;
713
714 api.create(
715 &PostParams {
716 dry_run: true,
717 ..Default::default()
718 },
719 &candidate,
720 )
721 .await
722 .map(|_| ())
723 .map_err(|e| {
724 Error::kube_error(
725 "validate replacement for",
726 "StatefulSet",
727 namespace,
728 name,
729 e,
730 )
731 })
732}
733
734async fn reconcile_statefulset(
735 kanidm: Arc<Kanidm>,
736 ctx: Arc<Context>,
737 desired: StatefulSet,
738) -> Result<StatefulSet> {
739 let namespace = kanidm.namespace().unwrap();
740 let name = desired.name_any();
741 let cached_current = ctx.stores.stateful_set_store.find(|current| {
742 current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
743 });
744
745 let cache_miss_422_error = if let Some(cached_current) = cached_current {
750 let mut cached_desired = desired.clone();
751 preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
752 if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
753 == StatefulSetApplyStrategy::Apply
754 {
755 return kanidm.apply(&ctx, cached_desired).await;
756 }
757 None
758 } else {
759 match kanidm.apply(&ctx, desired.clone()).await {
760 Ok(applied) => return Ok(applied),
761 Err(error) => {
762 if is_kube_422(&error) {
766 Some(error)
767 } else {
768 return Err(error);
769 }
770 }
771 }
772 };
773
774 let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
778 let Some(live_current) = api
779 .get_opt(&name)
780 .await
781 .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
782 else {
783 return match cache_miss_422_error {
784 Some(error) => Err(error),
785 None => kanidm.apply(&ctx, desired).await,
786 };
787 };
788
789 let mut live_desired = desired.clone();
792 preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
793 match classify_statefulset_change(&live_current, &live_desired) {
794 StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
795 Some(error) => Err(error),
796 None => kanidm.apply(&ctx, live_desired).await,
797 },
798 StatefulSetApplyStrategy::Recreate { .. } => {
799 validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
803
804 let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
807 Error::kube_error(
808 "re-read before deletion",
809 "StatefulSet",
810 &namespace,
811 &name,
812 e,
813 )
814 })?
815 else {
816 return kanidm.apply(&ctx, live_desired).await;
817 };
818 let mut latest_desired = live_desired.clone();
819 preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
820
821 match classify_statefulset_change(&latest_current, &latest_desired) {
822 StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
823 StatefulSetApplyStrategy::Recreate { immutable_fields } => {
824 validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
829 .await?;
830
831 info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
832 kanidm.delete(&ctx, &latest_current).await?;
833 ctx.kaniop_ctx
834 .metrics
835 .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
836 wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
837 .await?;
838 kanidm.apply(&ctx, latest_desired).await
839 }
840 }
841 }
842 }
843}
844
845fn is_kube_422(error: &Error) -> bool {
847 matches!(
848 error,
849 Error::KubeError(_, cause)
850 if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
851 )
852}
853
854#[cfg(test)]
855mod statefulset_recreate_recovery_tests {
856 use super::{Error, is_kube_422};
857
858 fn kube_api_error(code: u16) -> Error {
859 Error::KubeError(
860 "test error".to_string(),
861 Box::new(kube::Error::Api(Box::new(kube::error::Status {
862 code,
863 message: "test".to_string(),
864 reason: "test".to_string(),
865 ..Default::default()
866 }))),
867 )
868 }
869
870 #[test]
871 fn recreate_recovery_accepts_only_422() {
872 assert!(is_kube_422(&kube_api_error(422)));
873 assert!(!is_kube_422(&kube_api_error(403)));
874 assert!(!is_kube_422(&kube_api_error(429)));
875 assert!(!is_kube_422(&kube_api_error(500)));
876 }
877
878 #[test]
879 fn recreate_recovery_rejects_429_too_many_requests() {
880 assert!(!is_kube_422(&kube_api_error(429)));
881 }
882
883 #[test]
884 fn recreate_recovery_rejects_5xx_server_errors() {
885 assert!(!is_kube_422(&kube_api_error(500)));
886 assert!(!is_kube_422(&kube_api_error(502)));
887 assert!(!is_kube_422(&kube_api_error(503)));
888 assert!(!is_kube_422(&kube_api_error(504)));
889 }
890
891 #[test]
892 fn recreate_recovery_rejects_400_and_403() {
893 assert!(!is_kube_422(&kube_api_error(400)));
894 assert!(!is_kube_422(&kube_api_error(403)));
895 }
896}
897
898async fn reconcile(
899 kanidm: Arc<Kanidm>,
900 ctx: Arc<Context>,
901 status: KanidmStatus,
902) -> Result<(Action, bool)> {
903 if kanidm
904 .annotations()
905 .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
906 {
907 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
908 return Ok((Action::requeue(Duration::from_secs(5)), false));
909 }
910
911 let mut changed = false;
912 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
913 let replication_secret_futures =
914 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
915
916 let sts_to_delete = {
917 let expected_sts_names = kanidm
918 .spec
919 .replica_groups
920 .iter()
921 .map(|rg| kanidm.statefulset_name(&rg.name))
922 .collect::<Vec<_>>();
923 ctx.stores
924 .stateful_set_store
925 .state()
926 .into_iter()
927 .filter(|sts| {
928 sts.namespace() == kanidm.namespace()
929 && sts
930 .metadata
931 .labels
932 .as_ref()
933 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
934 && !expected_sts_names.contains(&sts.name_any())
935 })
936 .collect::<Vec<_>>()
937 };
938 if !sts_to_delete.is_empty() {
939 changed = true;
940 }
941 let sts_delete_futures = sts_to_delete
942 .iter()
943 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
944 .collect::<TryJoinAll<_>>();
945
946 let sts_futures = match kanidm.is_updatable(&status) {
947 true => {
948 let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
949 previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
950 });
951
952 let namespace = kanidm.get_namespace();
953 let schedule_api: Api<KanidmBackupSchedule> =
954 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
955 let schedules = schedule_api
956 .list(&Default::default())
957 .await
958 .map_err(|e| {
959 Error::KubeError(
960 "failed to list KanidmBackupSchedules for backup configuration resolution"
961 .to_string(),
962 Box::new(e),
963 )
964 })?
965 .items;
966
967 let repo_api: Api<KanidmBackupRepository> =
968 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
969 let repositories = repo_api
970 .list(&Default::default())
971 .await
972 .map_err(|e| {
973 Error::KubeError(
974 "failed to list KanidmBackupRepositories for backup configuration resolution"
975 .to_string(),
976 Box::new(e),
977 )
978 })?
979 .items;
980 let backup_config = resolve_backup_config(&kanidm, &schedules, &repositories);
981
982 kanidm
983 .spec
984 .replica_groups
985 .iter()
986 .map(|rg| {
987 let sts = kanidm.create_statefulset(
988 rg,
989 tls_secret_hash.as_deref(),
990 backup_config.as_ref(),
991 )?;
992 Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
993 })
994 .collect::<Result<TryJoinAll<_>, _>>()?
995 }
996 false => {
997 let note = match status.version.as_ref() {
998 Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
999 format!(
1000 "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
1001 v.image_tag,
1002 crate::version::KANIDM_CLIENT_VERSION
1003 )
1004 }
1005 _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
1006 };
1007 let _ignore_error = ctx
1008 .kaniop_ctx
1009 .recorder
1010 .publish(
1011 &Event {
1012 type_: EventType::Warning,
1013 reason: "UpgradeBlocked".to_string(),
1014 note: Some(note),
1015 action: "ReconcileStatefulSet".to_string(),
1016 secondary: None,
1017 },
1018 &kanidm.object_ref(&()),
1019 )
1020 .await
1021 .map_err(|e| {
1022 warn!(%e, "failed to publish KanidmError event");
1023 Error::KubeError("failed to publish event".to_string(), Box::new(e))
1024 });
1025 try_join_all([])
1026 }
1027 };
1028 let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1029
1030 let deprecated_rg_svcs = {
1031 let expected_rg_svcs_names = kanidm
1032 .spec
1033 .replica_groups
1034 .iter()
1035 .filter(|rg| rg.services.is_some())
1036 .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1037 .collect::<Vec<_>>();
1038 ctx.stores
1039 .service_store
1040 .state()
1041 .into_iter()
1042 .filter(|svc| {
1043 let labels = svc.metadata.labels.as_ref();
1044 svc.namespace() == kanidm.namespace()
1045 && !expected_rg_svcs_names.contains(&svc.name_any())
1046 && labels.is_some_and(|l| {
1047 l.contains_key(REPLICA_LABEL)
1048 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1049 })
1050 })
1051 .collect::<Vec<_>>()
1052 };
1053 if !deprecated_rg_svcs.is_empty() {
1054 changed = true;
1055 }
1056 let rg_svcs_delete_futures = deprecated_rg_svcs
1057 .iter()
1058 .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1059 .collect::<TryJoinAll<_>>();
1060
1061 let rg_services_futures = kanidm
1062 .spec
1063 .replica_groups
1064 .iter()
1065 .filter(|rg| rg.services.is_some())
1066 .flat_map(|rg| {
1067 (0..rg.replicas).map(|i| {
1068 let mut svc = kanidm.create_replica_group_service(rg, i);
1069 if let Some(current) = ctx.stores.service_store.find(|current| {
1070 current.namespace() == kanidm.namespace()
1071 && current.name_any() == svc.name_any()
1072 }) {
1073 preserve_defaulted_service_fields(&mut svc, current.as_ref());
1074 }
1075 kanidm.patch(&ctx, svc)
1076 })
1077 })
1078 .collect::<TryJoinAll<_>>();
1079
1080 let deprecated_ingresses = {
1081 let names = [
1082 kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1083 kanidm.generate_region_ingress_name(),
1084 ]
1085 .into_iter()
1086 .flatten()
1087 .collect::<Vec<_>>();
1088 ctx.stores
1089 .ingress_store
1090 .state()
1091 .into_iter()
1092 .filter(|ing| {
1093 ing.namespace() == kanidm.namespace()
1094 && !names.contains(&ing.name_any())
1095 && ing.metadata.labels == Some(kanidm.generate_labels())
1096 })
1097 .collect::<Vec<_>>()
1098 };
1099 if !deprecated_ingresses.is_empty() {
1100 changed = true;
1101 }
1102 let ingress_delete_futures = deprecated_ingresses
1103 .iter()
1104 .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1105 .collect::<TryJoinAll<_>>();
1106
1107 let ingress_futures = kanidm
1108 .create_ingress()
1109 .into_iter()
1110 .chain(kanidm.create_region_ingress())
1111 .map(|ingress| kanidm.patch(&ctx, ingress))
1112 .collect::<TryJoinAll<_>>();
1113
1114 let deprecated_http_routes = match &ctx.stores.http_route_store {
1115 Some(store) => {
1116 let expected_names = kanidm
1117 .spec
1118 .gateway
1119 .as_ref()
1120 .map(|_| vec![kanidm.name_any()])
1121 .unwrap_or_default();
1122 let routes: Vec<_> = store
1123 .state()
1124 .into_iter()
1125 .filter(|route| {
1126 route.namespace() == kanidm.namespace()
1127 && !expected_names.contains(&route.name_any())
1128 && route.metadata.labels == Some(kanidm.generate_labels())
1129 })
1130 .collect();
1131 if !routes.is_empty() {
1132 changed = true;
1133 }
1134 routes
1135 }
1136 None => Vec::new(),
1137 };
1138 let http_route_delete_futures = deprecated_http_routes
1139 .iter()
1140 .map(|route| kanidm.delete(&ctx, route.as_ref()))
1141 .collect::<TryJoinAll<_>>();
1142
1143 let http_route_futures = if ctx.stores.http_route_store.is_some() {
1144 kanidm
1145 .create_http_route()
1146 .into_iter()
1147 .map(|route| kanidm.patch(&ctx, route))
1148 .collect::<TryJoinAll<_>>()
1149 } else {
1150 try_join_all(Vec::new())
1151 };
1152
1153 let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1154 Some(store) => {
1155 let expected_names = kanidm
1156 .spec
1157 .gateway
1158 .as_ref()
1159 .and_then(|g| g.backend_tls_policy.as_ref())
1160 .map(|_| vec![kanidm.name_any()])
1161 .unwrap_or_default();
1162 let policies: Vec<_> = store
1163 .state()
1164 .into_iter()
1165 .filter(|policy| {
1166 policy.namespace() == kanidm.namespace()
1167 && !expected_names.contains(&policy.name_any())
1168 && policy.metadata.labels == Some(kanidm.generate_labels())
1169 })
1170 .collect();
1171 if !policies.is_empty() {
1172 changed = true;
1173 }
1174 policies
1175 }
1176 None => Vec::new(),
1177 };
1178 let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1179 .iter()
1180 .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1181 .collect::<TryJoinAll<_>>();
1182
1183 let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1184 kanidm
1185 .create_backend_tls_policy()
1186 .into_iter()
1187 .map(|policy| kanidm.patch(&ctx, policy))
1188 .collect::<TryJoinAll<_>>()
1189 } else {
1190 try_join_all(Vec::new())
1191 };
1192
1193 let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1194 sts_delete_futures,
1195 admin_secret_future,
1196 replication_secret_futures,
1197 sts_futures,
1198 service_future,
1199 rg_svcs_delete_futures,
1200 rg_services_futures,
1201 ingress_delete_futures,
1202 ingress_futures,
1203 http_route_delete_futures,
1204 http_route_futures,
1205 backend_tls_policy_delete_futures,
1206 backend_tls_policy_futures
1207 )?;
1208
1209 if admin_secret_changed || replication_secrets_changed {
1210 changed = true;
1211 }
1212
1213 if admin_secret_changed {
1214 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1216 }
1217
1218 if is_kanidm_available(status.clone()) {
1219 let namespace = kanidm.namespace().unwrap();
1220 let name = kanidm.name_any();
1221 match ctx
1222 .kaniop_ctx
1223 .get_kanidm_cluster_client(&kanidm, KanidmUser::Admin)
1224 .await
1225 {
1226 Ok(system_client) => {
1227 if let Err(e) =
1228 reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1229 {
1230 warn!(%e, "failed to reconcile domain appearance");
1231 }
1232 }
1233 Err(e) => {
1234 warn!(%e, "failed to create admin client for domain appearance reconciliation");
1235 }
1236 }
1237
1238 match ctx
1239 .kaniop_ctx
1240 .get_kanidm_cluster_client(&kanidm, KanidmUser::IdmAdmin)
1241 .await
1242 {
1243 Ok(kanidm_client) => {
1244 let (mail_sender_status, mail_sender_mutated) =
1245 reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1246 .await
1247 .unwrap_or_else(|e| {
1248 warn!(%e, "failed to reconcile mail sender");
1249 (None, false)
1250 });
1251 if mail_sender_mutated {
1252 changed = true;
1253 }
1254 let current_mail_sender_status =
1255 kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1256 if mail_sender_status != current_mail_sender_status {
1257 let kanidm_api: Api<Kanidm> =
1258 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1259 let status_patch = serde_json::json!({
1260 "status": {
1261 "mailSender": mail_sender_status
1262 }
1263 });
1264 if let Err(e) = kanidm_api
1265 .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1266 .await
1267 {
1268 warn!(%e, "failed to patch Kanidm/status for mail sender");
1269 }
1270 }
1271 }
1272 Err(e) => {
1273 warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1274 }
1275 }
1276 }
1277
1278 Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1279}
1280
1281async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1282 debug!("cleanup");
1283
1284 let mut changed = false;
1285 let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1286 if mail_resources_cleaned {
1287 changed = true;
1288 }
1289
1290 let name = kanidm.name_any();
1291
1292 if let Ok(kanidm_client) = ctx
1293 .kaniop_ctx
1294 .get_kanidm_cluster_client(&kanidm, KanidmUser::IdmAdmin)
1295 .await
1296 {
1297 let kanidm_cleanup_done =
1298 cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1299 if kanidm_cleanup_done {
1300 changed = true;
1301 }
1302 }
1303
1304 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1305 Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1306}
1307
1308async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1309 let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1310 let start = std::time::Instant::now();
1311 let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1312 let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1313
1314 loop {
1315 let sts = sts_api.get(sts_name).await.map_err(|e| {
1316 Error::KubeError(
1317 format!("failed to get StatefulSet {namespace}/{sts_name}"),
1318 Box::new(e),
1319 )
1320 })?;
1321
1322 if sts.metadata.deletion_timestamp.is_some() {
1323 return Err(Error::ReceiveOutput(format!(
1324 "StatefulSet {namespace}/{sts_name} is being deleted"
1325 )));
1326 }
1327
1328 let spec = sts.spec.as_ref().ok_or_else(|| {
1329 Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1330 })?;
1331
1332 let status = match sts.status {
1333 Some(ref s) => s,
1334 None => {
1335 if start.elapsed() >= timeout {
1336 return Err(Error::ReceiveOutput(format!(
1337 "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1338 )));
1339 }
1340 sleep(poll_interval).await;
1341 continue;
1342 }
1343 };
1344
1345 let replicas = spec.replicas.unwrap_or(0);
1346
1347 if replicas == 0 {
1348 return Ok(());
1349 }
1350
1351 let updated_replicas = status.updated_replicas.unwrap_or(0);
1352 let current_revision = status.current_revision.as_deref().unwrap_or("");
1353 let update_revision = status.update_revision.as_deref().unwrap_or("");
1354 let ready_replicas = status.ready_replicas.unwrap_or(0);
1355
1356 if updated_replicas == replicas
1357 && current_revision == update_revision
1358 && ready_replicas == replicas
1359 {
1360 return Ok(());
1361 }
1362
1363 if start.elapsed() >= timeout {
1364 return Err(Error::ReceiveOutput(format!(
1365 "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1366 )));
1367 }
1368
1369 sleep(poll_interval).await;
1370 }
1371}
1372
1373async fn show_replica_cert_with_retries(
1374 kanidm: &Kanidm,
1375 ctx: Arc<Context>,
1376 pod_name: &str,
1377 max_attempts: u32,
1378) -> std::result::Result<String, Error> {
1379 let mut last_error = None;
1380 for attempt in 0..max_attempts {
1381 if attempt > 0 {
1382 sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1383 }
1384 match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1385 Ok(cert) => return Ok(cert),
1386 Err(e) => {
1387 info!(
1388 "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1389 attempt + 1,
1390 max_attempts
1391 );
1392 last_error = Some(e);
1393 }
1394 }
1395 }
1396 Err(last_error.unwrap_or_else(|| {
1397 Error::ReceiveOutput(format!(
1398 "failed to get certificate for {pod_name} after {max_attempts} attempts"
1399 ))
1400 }))
1401}
1402
1403impl Kanidm {
1404 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1406 where
1407 K: Resource<Scope = NamespaceResourceScope>
1408 + Serialize
1409 + Clone
1410 + std::fmt::Debug
1411 + for<'de> Deserialize<'de>,
1412 <K as kube::Resource>::DynamicType: Default,
1413 <K as Resource>::Scope: std::marker::Sized,
1414 {
1415 self.kube_delete(
1416 ctx.kaniop_ctx.client.clone(),
1417 &ctx.kaniop_ctx.metrics,
1418 resource,
1419 )
1420 .await
1421 }
1422
1423 pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1426 where
1427 K: Resource<Scope = NamespaceResourceScope>
1428 + Serialize
1429 + Clone
1430 + std::fmt::Debug
1431 + for<'de> Deserialize<'de>,
1432 <K as kube::Resource>::DynamicType: Default,
1433 <K as Resource>::Scope: std::marker::Sized,
1434 {
1435 self.kube_apply(
1436 ctx.kaniop_ctx.client.clone(),
1437 resource,
1438 KANIDM_OPERATOR_NAME,
1439 )
1440 .await
1441 }
1442
1443 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1444 where
1445 K: Resource<Scope = NamespaceResourceScope>
1446 + Serialize
1447 + Clone
1448 + std::fmt::Debug
1449 + for<'de> Deserialize<'de>,
1450 <K as kube::Resource>::DynamicType: Default,
1451 <K as Resource>::Scope: std::marker::Sized,
1452 {
1453 self.kube_patch(
1454 ctx.kaniop_ctx.client.clone(),
1455 &ctx.kaniop_ctx.metrics,
1456 resource,
1457 KANIDM_OPERATOR_NAME,
1458 )
1459 .await
1460 }
1461
1462 #[inline]
1463 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1464 LABELS
1465 .clone()
1466 .into_iter()
1467 .chain([
1468 (INSTANCE_LABEL.to_string(), self.name_any()),
1469 (CLUSTER_LABEL.to_string(), self.name_any()),
1470 ])
1471 .collect()
1472 }
1473
1474 #[inline]
1475 fn generate_labels(&self) -> BTreeMap<String, String> {
1476 self.generate_resource_labels()
1477 }
1478
1479 #[inline]
1480 fn get_tls_secret_name(&self) -> String {
1481 format!("{}-tls", self.name_any())
1482 }
1483
1484 pub(crate) fn effective_tls_secret_name(&self) -> String {
1487 self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1488 self.spec
1489 .ingress
1490 .as_ref()
1491 .and_then(|i| i.tls_secret_name.clone())
1492 .unwrap_or_else(|| self.get_tls_secret_name())
1493 })
1494 }
1495
1496 #[inline]
1497 fn get_namespace(&self) -> String {
1498 self.namespace().unwrap()
1500 }
1501
1502 #[inline]
1503 fn is_replication_enabled(&self) -> bool {
1504 self.spec.replica_groups.len() > 1
1505 || self
1506 .spec
1507 .replica_groups
1508 .first()
1509 .is_some_and(|rg| rg.replicas > 1)
1510 || !self.spec.external_replication_nodes.is_empty()
1511 }
1512
1513 #[inline]
1514 fn is_updatable(&self, status: &KanidmStatus) -> bool {
1515 if self.spec.disable_upgrade_checks {
1518 return true;
1519 }
1520 status
1521 .version
1522 .as_ref()
1523 .map(|v| {
1524 if matches!(
1525 v.compatibility_result,
1526 VersionCompatibilityResult::Incompatible
1527 | VersionCompatibilityResult::NotChecked
1528 ) {
1529 return false;
1530 }
1531 let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1532 let versions = (parse_semver(¤t_tag), parse_semver(&v.image_tag));
1533 match v.upgrade_check_result {
1534 KanidmUpgradeCheckResult::Passed => {
1535 if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1536 minor <= v_minor + 1
1537 } else {
1538 true
1539 }
1540 }
1541 KanidmUpgradeCheckResult::Failed => {
1542 if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1543 versions
1544 {
1545 major == v_major && minor == v_minor && patch >= v_patch
1546 } else {
1547 v.image_tag == current_tag
1548 }
1549 }
1550 KanidmUpgradeCheckResult::Skipped => false,
1552 }
1553 })
1554 .unwrap_or(true)
1555 }
1556
1557 async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1558 let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1559 let pod = pod_api.get(pod_name).await.map_err(|e| {
1560 Error::KubeError(
1561 format!("failed to get pod {namespace}/{pod_name}"),
1562 Box::new(e),
1563 )
1564 })?;
1565 Ok(pod
1566 .status
1567 .as_ref()
1568 .and_then(|s| s.conditions.as_ref())
1569 .is_some_and(|conditions| {
1570 conditions
1571 .iter()
1572 .any(|c| c.type_ == "Ready" && c.status == "True")
1573 }))
1574 }
1575
1576 async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1577 let start = std::time::Instant::now();
1578 let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1579 let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1580
1581 loop {
1582 match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1583 Ok(true) => return Ok(()),
1584 Ok(false) => {}
1585 Err(e) => {
1586 if !e.is_retryable() {
1587 return Err(e);
1588 }
1589 trace!(%e, "pod readiness check failed, retrying");
1590 }
1591 }
1592
1593 if start.elapsed() >= timeout {
1594 return Err(Error::ReceiveOutput(format!(
1595 "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1596 )));
1597 }
1598
1599 sleep(poll_interval).await;
1600 }
1601 }
1602
1603 async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1604 where
1605 I: IntoIterator<Item = T> + Debug,
1606 T: Into<String>,
1607 {
1608 let namespace = &self.get_namespace();
1609 trace!(
1610 resource.name = &pod_name,
1611 resource.namespace = &namespace,
1612 ?command,
1613 "pod exec"
1614 );
1615 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1616 let attached = pod
1617 .exec(
1618 pod_name,
1619 command,
1620 &AttachParams::default().container("kanidm"),
1621 )
1622 .await
1623 .map_err(|e| {
1624 Error::KubeError(
1625 format!("failed to exec pod {namespace}/{pod_name}"),
1626 Box::new(e),
1627 )
1628 })?;
1629 get_output(attached).await
1630 }
1631
1632 async fn exec_with_wait<I, T>(
1633 &self,
1634 ctx: Arc<Context>,
1635 pod_name: &str,
1636 command: I,
1637 ) -> Result<String>
1638 where
1639 I: IntoIterator<Item = T> + Debug,
1640 T: Into<String>,
1641 {
1642 let namespace = &self.get_namespace();
1643 Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1644 self.exec(ctx, pod_name, command).await
1645 }
1646
1647 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1648 where
1649 I: IntoIterator<Item = T> + Debug,
1650 T: Into<String>,
1651 {
1652 let first_replica_group = self
1654 .spec
1655 .replica_groups
1656 .first()
1657 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1658 let sts_name = self.statefulset_name(&first_replica_group.name);
1659 let pod_name = format!("{sts_name}-0");
1660 self.exec(ctx, &pod_name, command).await
1661 }
1662
1663 async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1664 where
1665 I: IntoIterator<Item = T> + Debug,
1666 T: Into<String>,
1667 {
1668 let first_replica_group = self
1669 .spec
1670 .replica_groups
1671 .first()
1672 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1673 let sts_name = self.statefulset_name(&first_replica_group.name);
1674 let pod_name = format!("{sts_name}-0");
1675 self.exec_with_wait(ctx, &pod_name, command).await
1676 }
1677}
1678
1679#[cfg(test)]
1680mod test {
1681 use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1682 use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1683
1684 use crate::controller::State;
1685 use crate::kanidm::controller::context::{Context, Stores};
1686 use crate::kanidm::crd::KanidmStatus;
1687
1688 use kaniop_k8s_util::error::Result;
1689 use kaniop_k8s_util::resources::hash_secret_data;
1690
1691 use std::collections::BTreeMap;
1692 use std::sync::Arc;
1693
1694 use http::{Request, Response};
1695 use k8s_openapi::ByteString;
1696 use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1697 use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1698 use k8s_openapi::api::networking::v1::Ingress;
1699 use kube::api::ObjectMeta;
1700 use kube::runtime::reflector::store::Writer;
1701 use kube::{Client, Resource, ResourceExt, client::Body};
1702 use opentelemetry::metrics::MeterProvider;
1703 use serde_json::json;
1704
1705 impl Kanidm {
1706 pub fn test() -> Self {
1708 let mut e = Kanidm::new(
1709 "test",
1710 serde_json::from_value(json!({
1711 "domain": "idm.example.com",
1712 "replicaGroups": [
1713 {
1714 "name": "default",
1715 "replicas": 1
1716 }
1717 ]
1718 }))
1719 .unwrap(),
1720 );
1721 e.meta_mut().namespace = Some("default".into());
1722 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1725 e
1726 }
1727
1728 pub fn with_ingress(mut self) -> Self {
1729 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1730 self
1731 }
1732
1733 pub fn with_replicas(mut self, replicas: i32) -> Self {
1735 self.spec.replica_groups[0].replicas = replicas;
1736 self
1737 }
1738
1739 pub fn needs_delete(mut self) -> Self {
1741 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1742 use k8s_openapi::jiff::Timestamp;
1743 let now = Timestamp::from_second(1491138632).unwrap(); self.meta_mut().deletion_timestamp = Some(Time(now));
1745 self
1746 }
1747
1748 pub fn with_status(mut self, status: KanidmStatus) -> Self {
1750 self.status = Some(status);
1751 self
1752 }
1753 }
1754
1755 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1757 pub struct ApiServerVerifier(pub ApiServerHandle);
1758
1759 pub enum Scenario {
1761 Create(Kanidm),
1762 CreateWithTwoReplicas(Kanidm),
1763 CreateWithIngress(Kanidm),
1764 CreateWithIngressWithTwoReplicas(Kanidm),
1765 CreateWithTlsSecret(Kanidm, Box<Secret>),
1766 }
1767
1768 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1769 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1770 .await
1771 .expect("timeout on mock apiserver")
1772 .expect("scenario succeeded")
1773 }
1774
1775 impl ApiServerVerifier {
1776 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1787 tokio::spawn(async move {
1788 match scenario {
1790 Scenario::Create(kanidm) => {
1791 self.handle_kanidm_get(kanidm.clone())
1792 .await
1793 .unwrap()
1794 .handle_kanidm_status_patch(kanidm.clone())
1795 .await
1796 .unwrap()
1797 .handle_tls_secret_get_not_found(kanidm.clone())
1798 .await
1799 .unwrap()
1800 .handle_backup_schedule_list()
1801 .await
1802 .unwrap()
1803 .handle_backup_repository_list()
1804 .await
1805 .unwrap()
1806 .handle_statefulset_patch(kanidm.clone(), None)
1807 .await
1808 .unwrap()
1809 .handle_service_patch(kanidm.clone())
1810 .await
1811 }
1812 Scenario::CreateWithTwoReplicas(kanidm) => {
1813 self.handle_kanidm_get(kanidm.clone())
1814 .await
1815 .unwrap()
1816 .handle_kanidm_status_patch(kanidm.clone())
1817 .await
1818 .unwrap()
1819 .handle_tls_secret_get_not_found(kanidm.clone())
1820 .await
1821 .unwrap()
1822 .handle_backup_schedule_list()
1823 .await
1824 .unwrap()
1825 .handle_backup_repository_list()
1826 .await
1827 .unwrap()
1828 .handle_statefulset_patch(kanidm.clone(), None)
1829 .await
1830 .unwrap()
1831 .handle_service_patch(kanidm.clone())
1832 .await
1833 }
1834 Scenario::CreateWithIngress(kanidm) => {
1835 self.handle_kanidm_get(kanidm.clone())
1836 .await
1837 .unwrap()
1838 .handle_kanidm_status_patch(kanidm.clone())
1839 .await
1840 .unwrap()
1841 .handle_tls_secret_get_not_found(kanidm.clone())
1842 .await
1843 .unwrap()
1844 .handle_backup_schedule_list()
1845 .await
1846 .unwrap()
1847 .handle_backup_repository_list()
1848 .await
1849 .unwrap()
1850 .handle_statefulset_patch(kanidm.clone(), None)
1851 .await
1852 .unwrap()
1853 .handle_service_patch(kanidm.clone())
1854 .await
1855 .unwrap()
1856 .handle_ingress_patch(kanidm.clone())
1857 .await
1858 }
1859 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1860 self.handle_kanidm_get(kanidm.clone())
1861 .await
1862 .unwrap()
1863 .handle_kanidm_status_patch(kanidm.clone())
1864 .await
1865 .unwrap()
1866 .handle_tls_secret_get_not_found(kanidm.clone())
1867 .await
1868 .unwrap()
1869 .handle_backup_schedule_list()
1870 .await
1871 .unwrap()
1872 .handle_backup_repository_list()
1873 .await
1874 .unwrap()
1875 .handle_statefulset_patch(kanidm.clone(), None)
1876 .await
1877 .unwrap()
1878 .handle_service_patch(kanidm.clone())
1879 .await
1880 .unwrap()
1881 .handle_ingress_patch(kanidm.clone())
1882 .await
1883 }
1884 Scenario::CreateWithTlsSecret(kanidm, secret) => {
1885 let expected_hash = hash_secret_data(&secret);
1886 self.handle_kanidm_get(kanidm.clone())
1887 .await
1888 .unwrap()
1889 .handle_kanidm_status_patch(kanidm.clone())
1890 .await
1891 .unwrap()
1892 .handle_tls_secret_get(kanidm.clone(), *secret)
1893 .await
1894 .unwrap()
1895 .handle_backup_schedule_list()
1896 .await
1897 .unwrap()
1898 .handle_backup_repository_list()
1899 .await
1900 .unwrap()
1901 .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1902 .await
1903 .unwrap()
1904 .handle_service_patch(kanidm.clone())
1905 .await
1906 }
1907 }
1908 .expect("scenario completed without errors");
1909 })
1910 }
1911
1912 async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1913 let (request, send) = self.0.next_request().await.expect("service not called");
1914 assert_eq!(request.method(), http::Method::GET);
1915 let expected_uri_prefix = format!(
1916 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1917 kanidm.name_any()
1918 );
1919 let uri = request.uri().to_string();
1920 assert!(
1921 uri.starts_with(&expected_uri_prefix),
1922 "expected uri to start with {}, got {}",
1923 expected_uri_prefix,
1924 uri
1925 );
1926 let response = serde_json::to_vec(&kanidm).unwrap();
1927 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1928 Ok(self)
1929 }
1930
1931 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1932 let (request, send) = self.0.next_request().await.expect("service not called");
1933 assert_eq!(request.method(), http::Method::PATCH);
1934 assert_eq!(
1935 request.uri().to_string(),
1936 format!(
1937 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1938 kanidm.name_any()
1939 )
1940 );
1941
1942 let req_body = request.into_body().collect_bytes().await.unwrap();
1943 let json: serde_json::Value =
1944 serde_json::from_slice(&req_body).expect("patch object is json");
1945 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1946 .expect("valid kanidm status");
1947 let mut kanidm = Kanidm::test();
1948 kanidm.status = Some(status.clone());
1949 let response = serde_json::to_vec(&kanidm).unwrap();
1950 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1952 Ok(self)
1953 }
1954
1955 async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1956 let (request, send) = self.0.next_request().await.expect("service not called");
1957 assert_eq!(request.method(), http::Method::GET);
1958 let expected_uri_prefix = format!(
1959 "/api/v1/namespaces/default/secrets/{}",
1960 kanidm.effective_tls_secret_name()
1961 );
1962 let uri = request.uri().to_string();
1963 assert!(
1964 uri.starts_with(&expected_uri_prefix),
1965 "expected uri to start with {expected_uri_prefix}, got {uri}"
1966 );
1967 let status = serde_json::json!({
1968 "kind": "Status",
1969 "apiVersion": "v1",
1970 "metadata": {},
1971 "status": "Failure",
1972 "message": "secret not found",
1973 "reason": "NotFound",
1974 "code": 404
1975 });
1976 send.send_response(
1977 Response::builder()
1978 .status(404)
1979 .body(Body::from(serde_json::to_vec(&status).unwrap()))
1980 .unwrap(),
1981 );
1982 Ok(self)
1983 }
1984
1985 async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1986 let (request, send) = self.0.next_request().await.expect("service not called");
1987 assert_eq!(request.method(), http::Method::GET);
1988 let expected_uri_prefix = format!(
1989 "/api/v1/namespaces/default/secrets/{}",
1990 kanidm.effective_tls_secret_name()
1991 );
1992 let uri = request.uri().to_string();
1993 assert!(
1994 uri.starts_with(&expected_uri_prefix),
1995 "expected uri to start with {expected_uri_prefix}, got {uri}"
1996 );
1997 let response = serde_json::to_vec(&secret).unwrap();
1998 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1999 Ok(self)
2000 }
2001
2002 async fn handle_backup_schedule_list(mut self) -> Result<Self> {
2003 let (request, send) = self.0.next_request().await.expect("service not called");
2004 assert_eq!(request.method(), http::Method::GET);
2005 assert!(
2006 request
2007 .uri()
2008 .to_string()
2009 .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
2010 "expected backup schedule list request, got {}",
2011 request.uri()
2012 );
2013 let list = json!({
2014 "apiVersion": "kaniop.rs/v1alpha1",
2015 "kind": "KanidmBackupScheduleList",
2016 "metadata": {},
2017 "items": []
2018 });
2019 let response = serde_json::to_vec(&list).unwrap();
2020 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2021 Ok(self)
2022 }
2023
2024 async fn handle_backup_repository_list(mut self) -> Result<Self> {
2025 let (request, send) = self.0.next_request().await.expect("service not called");
2026 assert_eq!(request.method(), http::Method::GET);
2027 assert!(
2028 request.uri().to_string().contains(
2029 "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2030 ),
2031 "expected backup repository list request, got {}",
2032 request.uri()
2033 );
2034 let list = json!({
2035 "apiVersion": "kaniop.rs/v1alpha1",
2036 "kind": "KanidmBackupRepositoryList",
2037 "metadata": {},
2038 "items": []
2039 });
2040 let response = serde_json::to_vec(&list).unwrap();
2041 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2042 Ok(self)
2043 }
2044
2045 async fn handle_statefulset_patch(
2046 mut self,
2047 kanidm: Kanidm,
2048 expected_tls_hash: Option<String>,
2049 ) -> Result<Self> {
2050 for rg in kanidm.spec.replica_groups.iter() {
2051 let (request, send) = self.0.next_request().await.expect("service not called");
2052 assert_eq!(request.method(), http::Method::PATCH);
2053 assert_eq!(
2054 request.uri().to_string(),
2055 format!(
2056 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2057 kanidm.statefulset_name(&rg.name)
2058 )
2059 );
2060 let req_body = request.into_body().collect_bytes().await.unwrap();
2061 let json: serde_json::Value =
2062 serde_json::from_slice(&req_body).expect("patch object is json");
2063 let statefulset: StatefulSet =
2064 serde_json::from_value(json).expect("valid statefulset");
2065 assert_eq!(
2066 statefulset.clone().spec.unwrap().replicas.unwrap(),
2067 rg.replicas
2068 );
2069 let tls_hash_annotation = statefulset
2070 .spec
2071 .as_ref()
2072 .unwrap()
2073 .template
2074 .metadata
2075 .as_ref()
2076 .unwrap()
2077 .annotations
2078 .as_ref()
2079 .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2080 .cloned();
2081 assert_eq!(tls_hash_annotation, expected_tls_hash);
2082 let response = serde_json::to_vec(&statefulset).unwrap();
2083 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2085 }
2086
2087 Ok(self)
2088 }
2089
2090 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2091 let (request, send) = self.0.next_request().await.expect("service not called");
2092 assert_eq!(request.method(), http::Method::PATCH);
2093 assert_eq!(
2094 request.uri().to_string(),
2095 format!(
2096 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2097 kanidm.name_any()
2098 )
2099 );
2100
2101 let req_body = request.into_body().collect_bytes().await.unwrap();
2102 let json: serde_json::Value =
2103 serde_json::from_slice(&req_body).expect("patch object is json");
2104 let service: Service = serde_json::from_value(json).expect("valid service");
2105 let response = serde_json::to_vec(&service).unwrap();
2106 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2108 Ok(self)
2109 }
2110
2111 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2112 let (request, send) = self.0.next_request().await.expect("service not called");
2113 assert_eq!(request.method(), http::Method::PATCH);
2114 assert_eq!(
2115 request.uri().to_string(),
2116 format!(
2117 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2118 kanidm.name_any()
2119 )
2120 );
2121
2122 let req_body = request.into_body().collect_bytes().await.unwrap();
2123 let json: serde_json::Value =
2124 serde_json::from_slice(&req_body).expect("patch object is json");
2125 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2126 let response = serde_json::to_vec(&ingress).unwrap();
2127 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2129 Ok(self)
2130 }
2131 }
2132
2133 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2134 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2135 let mock_client = Client::new(mock_service, "default");
2136 let stores = Stores {
2137 stateful_set_store: Writer::default().as_reader(),
2138 service_store: Writer::default().as_reader(),
2139 ingress_store: Writer::default().as_reader(),
2140 secret_store: Writer::default().as_reader(),
2141 http_route_store: Some(Writer::default().as_reader()),
2142 backend_tls_policy_store: Some(Writer::default().as_reader()),
2143 deployment_store: Writer::default().as_reader(),
2144 config_map_store: Writer::default().as_reader(),
2145 };
2146 let controller_id = "test";
2147
2148 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2150 let meter = provider.meter("test");
2151 let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2152
2153 let state = State::new(
2154 metrics,
2155 Writer::default().as_reader(),
2156 Writer::default().as_reader(),
2157 None,
2158 );
2159 let ctx = Arc::new(Context::new(
2160 state.to_context(mock_client, controller_id),
2161 stores,
2162 ));
2163 (ctx, ApiServerVerifier(handle))
2164 }
2165
2166 #[tokio::test]
2167 async fn kanidm_create() {
2168 let (testctx, fakeserver) = get_test_context();
2169 let kanidm = Kanidm::test();
2170 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2171 reconcile_kanidm(Arc::new(kanidm), testctx)
2172 .await
2173 .expect("reconciler");
2174 timeout_after_1s(mocksrv).await;
2175 }
2176
2177 #[tokio::test]
2178 async fn kanidm_create_with_two_replicas() {
2179 let (testctx, fakeserver) = get_test_context();
2180 let kanidm = Kanidm::test().with_replicas(2);
2181 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2182 reconcile_kanidm(Arc::new(kanidm), testctx)
2183 .await
2184 .expect("reconciler");
2185 timeout_after_1s(mocksrv).await;
2186 }
2187
2188 #[tokio::test]
2189 async fn kanidm_create_with_ingress() {
2190 let (testctx, fakeserver) = get_test_context();
2191 let kanidm = Kanidm::test().with_ingress();
2192 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2193 reconcile_kanidm(Arc::new(kanidm), testctx)
2194 .await
2195 .expect("reconciler");
2196 timeout_after_1s(mocksrv).await;
2197 }
2198
2199 #[tokio::test]
2200 async fn kanidm_create_with_ingress_with_two_replicas() {
2201 let (testctx, fakeserver) = get_test_context();
2202 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2203 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2204 reconcile_kanidm(Arc::new(kanidm), testctx)
2205 .await
2206 .expect("reconciler");
2207 timeout_after_1s(mocksrv).await;
2208 }
2209
2210 fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2211 Arc::new(StatefulSet {
2212 metadata: ObjectMeta {
2213 name: Some(format!("{cluster}-default")),
2214 namespace: Some(namespace.to_string()),
2215 labels: Some(BTreeMap::from([(
2216 CLUSTER_LABEL.to_string(),
2217 cluster.to_string(),
2218 )])),
2219 ..Default::default()
2220 },
2221 spec: Some(StatefulSetSpec {
2222 template: PodTemplateSpec {
2223 metadata: Some(ObjectMeta {
2224 annotations: tls_hash.map(|h| {
2225 BTreeMap::from([(
2226 TLS_SECRET_HASH_ANNOTATION.to_string(),
2227 h.to_string(),
2228 )])
2229 }),
2230 ..Default::default()
2231 }),
2232 ..Default::default()
2233 },
2234 ..Default::default()
2235 }),
2236 ..Default::default()
2237 })
2238 }
2239
2240 #[test]
2241 fn test_previous_tls_secret_hash() {
2242 let kanidm = Kanidm::test();
2243
2244 let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2246 assert_eq!(
2247 previous_tls_secret_hash(&kanidm, stss),
2248 Some("abc123".to_string())
2249 );
2250
2251 let stss = vec![statefulset_of("test", "default", None)];
2253 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2254
2255 let stss = vec![
2257 statefulset_of("other", "default", Some("abc123")),
2258 statefulset_of("test", "other-ns", Some("abc123")),
2259 ];
2260 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2261 }
2262
2263 #[tokio::test]
2264 async fn kanidm_create_with_tls_secret() {
2265 let (testctx, fakeserver) = get_test_context();
2266 let kanidm = Kanidm::test();
2267 let secret = Secret {
2268 metadata: ObjectMeta {
2269 name: Some(kanidm.effective_tls_secret_name()),
2270 namespace: Some("default".to_string()),
2271 ..Default::default()
2272 },
2273 data: Some(BTreeMap::from([
2274 ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2275 ("tls.key".to_string(), ByteString(b"key".to_vec())),
2276 ])),
2277 ..Default::default()
2278 };
2279 let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2280 kanidm.clone(),
2281 Box::new(secret),
2282 ));
2283 reconcile_kanidm(Arc::new(kanidm), testctx)
2284 .await
2285 .expect("reconciler");
2286 timeout_after_1s(mocksrv).await;
2287 }
2288
2289 #[tokio::test]
2290 async fn replication_secret_gc_skips_non_replica_secrets() {
2291 use super::reconcile_replication_secrets;
2292 use super::secret::{SECRET_TYPE_LABEL, SecretType};
2293
2294 let kanidm = Kanidm::test();
2295 let status = kanidm.status.clone().unwrap_or_default();
2296
2297 let replica_cert_label =
2298 serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
2299
2300 let replica_cert_secret = Secret {
2301 metadata: ObjectMeta {
2302 name: Some(format!("{}-default-0-cert", kanidm.name_any())),
2303 namespace: Some("default".into()),
2304 labels: Some(BTreeMap::from([
2305 (CLUSTER_LABEL.to_string(), kanidm.name_any()),
2306 (SECRET_TYPE_LABEL.to_string(), replica_cert_label.clone()),
2307 ])),
2308 ..Default::default()
2309 },
2310 ..Default::default()
2311 };
2312
2313 let mail_sender_secret = Secret {
2314 metadata: ObjectMeta {
2315 name: Some(format!("{}-mail-sender-config", kanidm.name_any())),
2316 namespace: Some("default".into()),
2317 labels: Some(BTreeMap::from([(
2318 CLUSTER_LABEL.to_string(),
2319 kanidm.name_any(),
2320 )])),
2321 ..Default::default()
2322 },
2323 ..Default::default()
2324 };
2325
2326 let (mock_service, mut handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2327 let mock_client = Client::new(mock_service, "default");
2328
2329 let mut secret_writer = Writer::default();
2330 secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2331 replica_cert_secret.clone(),
2332 ));
2333 secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2334 mail_sender_secret.clone(),
2335 ));
2336
2337 let stores = Stores {
2338 stateful_set_store: Writer::default().as_reader(),
2339 service_store: Writer::default().as_reader(),
2340 ingress_store: Writer::default().as_reader(),
2341 secret_store: secret_writer.as_reader(),
2342 http_route_store: Some(Writer::default().as_reader()),
2343 backend_tls_policy_store: Some(Writer::default().as_reader()),
2344 deployment_store: Writer::default().as_reader(),
2345 config_map_store: Writer::default().as_reader(),
2346 };
2347
2348 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2349 let meter = provider.meter("test");
2350 let metrics = crate::metrics::Metrics::new(&meter, &["test"]);
2351 let state = State::new(
2352 metrics,
2353 Writer::default().as_reader(),
2354 Writer::default().as_reader(),
2355 None,
2356 );
2357 let ctx = Arc::new(Context::new(state.to_context(mock_client, "test"), stores));
2358
2359 let mock_srv = tokio::spawn(async move {
2360 let (request, send) = handle.next_request().await.expect("expected a request");
2361 assert_eq!(request.method(), http::Method::DELETE);
2362 let uri = request.uri().to_string();
2363 assert!(
2364 uri.contains("-default-0-cert"),
2365 "expected delete for replica cert secret, got {uri}"
2366 );
2367 let not_found = serde_json::json!({
2368 "kind": "Status",
2369 "apiVersion": "v1",
2370 "metadata": {},
2371 "status": "Failure",
2372 "message": "not found",
2373 "reason": "NotFound",
2374 "code": 404
2375 });
2376 send.send_response(
2377 Response::builder()
2378 .status(404)
2379 .body(Body::from(serde_json::to_vec(¬_found).unwrap()))
2380 .unwrap(),
2381 );
2382
2383 if let Ok(Some((req, _))) =
2384 tokio::time::timeout(std::time::Duration::from_millis(200), handle.next_request())
2385 .await
2386 {
2387 panic!("unexpected second request: {} {}", req.method(), req.uri());
2388 }
2389 });
2390
2391 let changed = reconcile_replication_secrets(Arc::new(kanidm), ctx, &status)
2392 .await
2393 .expect("reconcile succeeds despite 404 on delete");
2394 assert!(
2395 changed,
2396 "expected changed=true since a deprecated secret was found"
2397 );
2398
2399 tokio::time::timeout(std::time::Duration::from_secs(1), mock_srv)
2400 .await
2401 .expect("timeout waiting for mock server")
2402 .expect("mock server task succeeded");
2403 }
2404
2405 #[test]
2406 fn disable_upgrade_checks_allows_unchecked_version_status() {
2407 let mut kanidm = Kanidm::test();
2408 kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2409 kanidm.spec.disable_upgrade_checks = true;
2410 let status = KanidmStatus {
2411 version: Some(crate::kanidm::crd::KanidmVersionStatus {
2412 image_tag: "1.11.2".to_string(),
2413 upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Skipped,
2414 compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::NotChecked,
2415 }),
2416 ..Default::default()
2417 };
2418 assert!(kanidm.is_updatable(&status));
2419 }
2420
2421 #[test]
2422 fn disable_upgrade_checks_allows_previous_incompatible_status() {
2423 let mut kanidm = Kanidm::test();
2424 kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2425 kanidm.spec.disable_upgrade_checks = true;
2426 let status = KanidmStatus {
2427 version: Some(crate::kanidm::crd::KanidmVersionStatus {
2428 image_tag: "1.11.2".to_string(),
2429 upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Passed,
2430 compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::Incompatible,
2431 }),
2432 ..Default::default()
2433 };
2434 assert!(kanidm.is_updatable(&status));
2435 }
2436
2437 #[test]
2438 fn upgrade_checks_still_block_incompatible_versions_when_enabled() {
2439 let mut kanidm = Kanidm::test();
2440 kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2441 kanidm.spec.disable_upgrade_checks = false;
2442 let status = KanidmStatus {
2443 version: Some(crate::kanidm::crd::KanidmVersionStatus {
2444 image_tag: "1.11.2".to_string(),
2445 upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Passed,
2446 compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::Incompatible,
2447 }),
2448 ..Default::default()
2449 };
2450 assert!(!kanidm.is_updatable(&status));
2451 }
2452
2453 #[test]
2454 fn upgrade_checks_do_not_treat_skipped_checks_as_passed() {
2455 let mut kanidm = Kanidm::test();
2456 kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2457 kanidm.spec.disable_upgrade_checks = false;
2458 let status = KanidmStatus {
2459 version: Some(crate::kanidm::crd::KanidmVersionStatus {
2460 image_tag: "1.11.2".to_string(),
2461 upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Skipped,
2462 compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::NotChecked,
2463 }),
2464 ..Default::default()
2465 };
2466 assert!(!kanidm.is_updatable(&status));
2467 }
2468}