1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13 cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::SecretExt;
20use self::service::ServiceExt;
21use self::statefulset::{
22 StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23 preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_backup_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
31use crate::kanidm::crd::{
32 Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
33};
34use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
35use crate::telemetry;
36
37use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
38use kaniop_k8s_util::client::get_output;
39use kaniop_k8s_util::error::{Error, Result};
40use kaniop_k8s_util::parse::parse_semver;
41use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::fmt::Debug;
45use std::sync::{Arc, LazyLock};
46
47use futures::future::{TryJoinAll, join_all, try_join_all};
48use futures::stream::{self, StreamExt};
49use futures::try_join;
50use k8s_openapi::NamespaceResourceScope;
51use k8s_openapi::api::apps::v1::StatefulSet;
52use k8s_openapi::api::core::v1::{Pod, Secret, Service};
53use kube::Resource;
54use kube::ResourceExt;
55use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
56use kube::runtime::controller::Action;
57use kube::runtime::events::{Event, EventType};
58use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
59use serde::{Deserialize, Serialize};
60use tokio::time::{Duration, sleep};
61use tracing::{Span, debug, field, info, instrument, trace, warn};
62
63const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
64const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
65const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
66const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
67const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
68const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
69const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
70const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
71const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
72
73pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
74const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
75pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
76
77const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
78
79static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
80 BTreeMap::from([
81 (NAME_LABEL.to_string(), "kanidm".to_string()),
82 (
83 MANAGED_BY_LABEL.to_string(),
84 format!("kaniop-{CONTROLLER_ID}"),
85 ),
86 ])
87});
88
89async fn restart_statefulsets_after_replica_secret_update(
90 ctx: Arc<Context>,
91 sts_api: &Api<StatefulSet>,
92 namespace: &str,
93 statefulset_names: &[String],
94 sequential: bool,
95) -> Result<()> {
96 let restart_futures = statefulset_names
99 .iter()
100 .map(|sts_name| sts_api.restart(sts_name));
101
102 if sequential {
103 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
104 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
105 return Err(Error::kube_error(
106 "restart",
107 "StatefulSet",
108 namespace,
109 "replica certificate secret update statefulsets",
110 first_err,
111 ));
112 }
113 } else {
114 let results: Vec<_> = join_all(restart_futures).await;
115 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
116 return Err(Error::kube_error(
117 "restart",
118 "StatefulSet",
119 namespace,
120 "replica certificate secret update statefulsets",
121 first_err,
122 ));
123 }
124 }
125
126 let rollout_futures = statefulset_names
127 .iter()
128 .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
129 try_join_all(rollout_futures).await?;
130
131 Ok(())
132}
133
134pub async fn reconcile_admins_secret(
135 kanidm: Arc<Kanidm>,
136 ctx: Arc<Context>,
137 status: &KanidmStatus,
138) -> Result<bool> {
139 if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
140 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
141 kanidm.patch(&ctx, admins_secret).await?;
142 return Ok(true);
143 }
144 Ok(false)
145}
146
147pub async fn reconcile_replication_secrets(
148 kanidm: Arc<Kanidm>,
149 ctx: Arc<Context>,
150 status: &KanidmStatus,
151) -> Result<bool> {
152 let mut changed = false;
153 let expected_secret_names = if kanidm.is_replication_enabled() {
154 status
155 .replica_statuses
156 .iter()
157 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
158 .collect::<Vec<_>>()
159 } else {
160 Vec::new()
161 };
162 let deprecated_secrets = ctx
163 .stores
164 .secret_store
165 .state()
166 .into_iter()
167 .filter(|secret| {
168 secret.namespace() == kanidm.namespace()
169 && kanidm.admins_secret_name() != secret.name_any()
170 && !expected_secret_names.contains(&secret.name_any())
171 && secret
172 .metadata
173 .labels
174 .as_ref()
175 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
176 })
177 .collect::<Vec<_>>();
178 if !deprecated_secrets.is_empty() {
179 changed = true;
180 }
181 let secret_delete_future = deprecated_secrets
182 .iter()
183 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
184 .collect::<TryJoinAll<_>>();
185 try_join!(secret_delete_future)?;
186
187 if kanidm.is_replication_enabled() {
188 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
189 let statefulset_names = status
190 .replica_statuses
191 .iter()
192 .map(|rs| rs.statefulset_name.clone())
193 .collect::<BTreeSet<_>>()
194 .into_iter()
195 .collect::<Vec<_>>();
196
197 let has_pending = status
198 .replica_statuses
199 .iter()
200 .any(|rs| rs.state == KanidmReplicaState::Pending);
201
202 let sts_api =
203 Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
204
205 if has_pending {
206 changed = true;
207 stream::iter(
208 status
209 .replica_statuses
210 .iter()
211 .filter(|rs| rs.state == KanidmReplicaState::Pending),
212 )
213 .then(|rs| async {
214 let secret = kanidm
215 .generate_replica_secret(ctx.clone(), &rs.pod_name)
216 .await?;
217 kanidm.patch(&ctx, secret).await
218 })
219 .collect::<Vec<_>>()
220 .await
221 .into_iter()
222 .collect::<Result<Vec<_>>>()?;
223
224 let restart_futures = status
225 .replica_statuses
226 .iter()
227 .map(|rs| sts_api.restart(&rs.statefulset_name));
228
229 if has_single_replica {
230 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
231 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
232 return Err(Error::kube_error(
233 "restart",
234 "StatefulSet",
235 kanidm.get_namespace(),
236 "pending replica statefulsets",
237 first_err,
238 ));
239 }
240 } else {
241 let results: Vec<_> = join_all(restart_futures).await;
242 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
243 return Err(Error::kube_error(
244 "restart",
245 "StatefulSet",
246 kanidm.get_namespace(),
247 "pending replica statefulsets",
248 first_err,
249 ));
250 }
251 }
252 }
253
254 let has_certificate_host_invalid = status
255 .replica_statuses
256 .iter()
257 .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
258
259 let has_certificate_expiring = status
260 .replica_statuses
261 .iter()
262 .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
263
264 let has_non_ready_replicas = status
265 .replica_statuses
266 .iter()
267 .any(|rs| rs.state != KanidmReplicaState::Ready);
268
269 let host_invalid_replicas: Vec<_> = status
270 .replica_statuses
271 .iter()
272 .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
273 .collect();
274
275 let cert_expiring_replicas: Vec<_> = status
276 .replica_statuses
277 .iter()
278 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
279 .collect();
280
281 if !host_invalid_replicas.is_empty() {
282 changed = true;
283 let restart_futures = host_invalid_replicas
284 .iter()
285 .map(|rs| sts_api.restart(&rs.statefulset_name));
286
287 if has_single_replica {
288 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
289 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
290 return Err(Error::kube_error(
291 "restart",
292 "StatefulSet",
293 kanidm.get_namespace(),
294 "certificate host invalid statefulsets",
295 first_err,
296 ));
297 }
298 } else {
299 let results: Vec<_> = join_all(restart_futures).await;
300 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
301 return Err(Error::kube_error(
302 "restart",
303 "StatefulSet",
304 kanidm.get_namespace(),
305 "certificate host invalid statefulsets",
306 first_err,
307 ));
308 }
309 }
310
311 let restarted_sts_names: Vec<_> = host_invalid_replicas
312 .iter()
313 .map(|rs| rs.statefulset_name.clone())
314 .collect::<BTreeSet<_>>()
315 .into_iter()
316 .collect();
317
318 let namespace = kanidm.get_namespace();
319 let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
320 wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
321 });
322 try_join_all(rollout_futures).await?;
323
324 let renew_futures = host_invalid_replicas
325 .iter()
326 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
327 let results: Vec<_> = join_all(renew_futures).await;
328 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
329 return Err(first_err);
330 }
331
332 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
333
334 let cert_futures = host_invalid_replicas.iter().map(|rs| async {
335 let cert = show_replica_cert_with_retries(
336 &kanidm,
337 ctx.clone(),
338 &rs.pod_name,
339 CERT_SHOW_RETRY_ATTEMPTS,
340 )
341 .await?;
342 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
343 kanidm.patch(&ctx, secret.clone()).await
344 });
345 try_join_all(cert_futures).await?;
346
347 restart_statefulsets_after_replica_secret_update(
348 ctx.clone(),
349 &sts_api,
350 &kanidm.get_namespace(),
351 &statefulset_names,
352 has_single_replica,
353 )
354 .await?;
355
356 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
357 }
358
359 if !cert_expiring_replicas.is_empty() {
360 changed = true;
361 let renew_futures = cert_expiring_replicas
362 .iter()
363 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
364 let results: Vec<_> = join_all(renew_futures).await;
365 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
366 return Err(first_err);
367 }
368
369 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
370
371 let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
372 let cert = show_replica_cert_with_retries(
373 &kanidm,
374 ctx.clone(),
375 &rs.pod_name,
376 CERT_SHOW_RETRY_ATTEMPTS,
377 )
378 .await?;
379 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
380 kanidm.patch(&ctx, secret.clone()).await
381 });
382 try_join_all(cert_futures).await?;
383
384 restart_statefulsets_after_replica_secret_update(
385 ctx.clone(),
386 &sts_api,
387 &kanidm.get_namespace(),
388 &statefulset_names,
389 has_single_replica,
390 )
391 .await?;
392
393 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
394 }
395
396 if has_non_ready_replicas
397 && !has_pending
398 && !has_certificate_host_invalid
399 && !has_certificate_expiring
400 {
401 changed = true;
402 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
403
404 let restart_futures = status
405 .replica_statuses
406 .iter()
407 .map(|rs| sts_api.restart(&rs.statefulset_name));
408
409 if has_single_replica {
410 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
411 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
412 return Err(Error::kube_error(
413 "restart",
414 "StatefulSet",
415 kanidm.get_namespace(),
416 "replica statefulsets",
417 first_err,
418 ));
419 }
420 } else {
421 let results: Vec<_> = join_all(restart_futures).await;
422 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
423 return Err(Error::kube_error(
424 "restart",
425 "StatefulSet",
426 kanidm.get_namespace(),
427 "replica statefulsets",
428 first_err,
429 ));
430 }
431 }
432 }
433 }
434
435 Ok(changed)
436}
437
438#[instrument(skip(ctx, kanidm))]
439pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
440 let trace_id = telemetry::get_trace_id();
441 Span::current().record("trace_id", field::display(&trace_id));
442 let _timer = ctx
443 .kaniop_ctx
444 .metrics
445 .reconcile_count_and_measure(&trace_id);
446 info!("reconciling Kanidm");
447
448 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
449 debug!(%e, "failed to reconcile status");
450 ctx.kaniop_ctx.metrics.status_update_errors_inc();
451 e
452 })?;
453 let kanidm_api: Api<Kanidm> =
454 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
455 let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
456 let outcome_clone = outcome.clone();
457 let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
458 let outcome = outcome_clone.clone();
459 let ctx = ctx.clone();
460 let status = status.clone();
461 async move {
462 match event {
463 Finalizer::Apply(kanidm) => {
464 let (action, changed) = reconcile(kanidm, ctx, status).await?;
465 outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
466 Ok(action)
467 }
468 Finalizer::Cleanup(kanidm) => {
469 let (action, changed) = cleanup(kanidm, ctx).await?;
470 outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
471 Ok(action)
472 }
473 }
474 }
475 })
476 .await
477 .or_else(|e| match e {
478 FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
479 debug!("resource already removed during finalizer cleanup");
480 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
481 }
482 _ => Err(Error::FinalizerError(
483 "failed on kanidm account finalizer".to_string(),
484 Box::new(e),
485 )),
486 })?;
487 let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
488 Ok((action, changed))
489}
490
491async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
495 let secret_name = kanidm.effective_tls_secret_name();
496 let secret_api =
497 Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
498 let secret = secret_api
499 .get_opt(&secret_name)
500 .await
501 .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
502 Ok(secret.as_ref().map(hash_secret_data))
503}
504
505fn previous_tls_secret_hash(
510 kanidm: &Kanidm,
511 stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
512) -> Option<String> {
513 stateful_sets
514 .into_iter()
515 .filter(|sts| {
516 sts.namespace() == kanidm.namespace()
517 && sts
518 .metadata
519 .labels
520 .as_ref()
521 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
522 })
523 .find_map(|sts| {
524 sts.spec
525 .as_ref()?
526 .template
527 .metadata
528 .as_ref()?
529 .annotations
530 .as_ref()?
531 .get(TLS_SECRET_HASH_ANNOTATION)
532 .cloned()
533 })
534}
535
536fn is_headless_service(service: &Service) -> bool {
537 service
538 .spec
539 .as_ref()
540 .and_then(|spec| spec.cluster_ip.as_deref())
541 == Some("None")
542}
543
544fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
545 if is_headless_service(desired) != is_headless_service(current) {
546 return;
547 }
548 let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
549 else {
550 return;
551 };
552 if desired_spec.cluster_ip.is_none() {
553 desired_spec.cluster_ip = current_spec.cluster_ip.clone();
554 }
555 if desired_spec.cluster_ips.is_none() {
556 desired_spec.cluster_ips = current_spec.cluster_ips.clone();
557 }
558}
559
560async fn wait_for_statefulset_deletion(
561 client: kube::Client,
562 namespace: &str,
563 name: &str,
564) -> Result<()> {
565 let api = Api::<StatefulSet>::namespaced(client, namespace);
566 let start = std::time::Instant::now();
567 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
568 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
569
570 loop {
571 match api.get_opt(name).await {
572 Ok(None) => return Ok(()),
573 Ok(Some(_)) => {}
574 Err(e) => {
575 return Err(Error::kube_error(
576 "wait for deletion of",
577 "StatefulSet",
578 namespace,
579 name,
580 e,
581 ));
582 }
583 }
584
585 if start.elapsed() >= timeout {
586 return Err(Error::ReceiveOutput(format!(
587 "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
588 )));
589 }
590 sleep(poll_interval).await;
591 }
592}
593
594async fn reconcile_service(
595 kanidm: &Kanidm,
596 ctx: &Context,
597 mut desired: Service,
598) -> Result<Service> {
599 let namespace = kanidm.get_namespace();
600 let name = desired.name_any();
601 let desired_headless = is_headless_service(&desired);
602
603 if let Some(cached) = ctx
604 .stores
605 .service_store
606 .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
607 {
608 if is_headless_service(cached.as_ref()) == desired_headless {
609 preserve_defaulted_service_fields(&mut desired, cached.as_ref());
610 } else {
611 let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
612 let current = service_api
613 .get_opt(&name)
614 .await
615 .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
616
617 if let Some(current) = current {
618 let current_headless = is_headless_service(¤t);
619 if current_headless != desired_headless {
620 info!(
621 resource.name = &name,
622 resource.namespace = &namespace,
623 current_headless,
624 desired_headless,
625 "recreating Service to change headless mode"
626 );
627 kanidm.delete(ctx, ¤t).await?;
628 ctx.kaniop_ctx
629 .metrics
630 .reconcile_deploy_delete_create_inc("Service", "headless_transition");
631 wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
632 .await?;
633 } else {
634 preserve_defaulted_service_fields(&mut desired, ¤t);
635 }
636 }
637 }
638 }
639
640 kanidm.patch(ctx, desired).await
641}
642
643async fn wait_for_service_deletion(
644 client: kube::Client,
645 namespace: &str,
646 name: &str,
647) -> Result<()> {
648 let api = Api::<Service>::namespaced(client, namespace);
649 let start = std::time::Instant::now();
650 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
651 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
652
653 loop {
654 match api.get_opt(name).await {
655 Ok(None) => return Ok(()),
656 Ok(Some(_)) => {}
657 Err(e) => {
658 return Err(Error::kube_error(
659 "wait for deletion of",
660 "Service",
661 namespace,
662 name,
663 e,
664 ));
665 }
666 }
667
668 if start.elapsed() >= timeout {
669 return Err(Error::ReceiveOutput(format!(
670 "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
671 )));
672 }
673 sleep(poll_interval).await;
674 }
675}
676
677async fn validate_statefulset_replacement(
678 api: &Api<StatefulSet>,
679 desired: &StatefulSet,
680 namespace: &str,
681 name: &str,
682) -> Result<()> {
683 let mut candidate = desired.clone();
684 candidate.metadata.name = None;
685 candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
686 candidate.metadata.namespace = None;
687 candidate.metadata.resource_version = None;
688 candidate.metadata.uid = None;
689 candidate.metadata.creation_timestamp = None;
690 candidate.metadata.generation = None;
691 candidate.metadata.managed_fields = None;
692 candidate.metadata.deletion_timestamp = None;
693 candidate.metadata.deletion_grace_period_seconds = None;
694 candidate.status = None;
695
696 api.create(
697 &PostParams {
698 dry_run: true,
699 ..Default::default()
700 },
701 &candidate,
702 )
703 .await
704 .map(|_| ())
705 .map_err(|e| {
706 Error::kube_error(
707 "validate replacement for",
708 "StatefulSet",
709 namespace,
710 name,
711 e,
712 )
713 })
714}
715
716async fn reconcile_statefulset(
717 kanidm: Arc<Kanidm>,
718 ctx: Arc<Context>,
719 desired: StatefulSet,
720) -> Result<StatefulSet> {
721 let namespace = kanidm.namespace().unwrap();
722 let name = desired.name_any();
723 let cached_current = ctx.stores.stateful_set_store.find(|current| {
724 current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
725 });
726
727 let cache_miss_422_error = if let Some(cached_current) = cached_current {
732 let mut cached_desired = desired.clone();
733 preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
734 if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
735 == StatefulSetApplyStrategy::Apply
736 {
737 return kanidm.apply(&ctx, cached_desired).await;
738 }
739 None
740 } else {
741 match kanidm.apply(&ctx, desired.clone()).await {
742 Ok(applied) => return Ok(applied),
743 Err(error) => {
744 if is_kube_422(&error) {
748 Some(error)
749 } else {
750 return Err(error);
751 }
752 }
753 }
754 };
755
756 let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
760 let Some(live_current) = api
761 .get_opt(&name)
762 .await
763 .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
764 else {
765 return match cache_miss_422_error {
766 Some(error) => Err(error),
767 None => kanidm.apply(&ctx, desired).await,
768 };
769 };
770
771 let mut live_desired = desired.clone();
774 preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
775 match classify_statefulset_change(&live_current, &live_desired) {
776 StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
777 Some(error) => Err(error),
778 None => kanidm.apply(&ctx, live_desired).await,
779 },
780 StatefulSetApplyStrategy::Recreate { .. } => {
781 validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
785
786 let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
789 Error::kube_error(
790 "re-read before deletion",
791 "StatefulSet",
792 &namespace,
793 &name,
794 e,
795 )
796 })?
797 else {
798 return kanidm.apply(&ctx, live_desired).await;
799 };
800 let mut latest_desired = live_desired.clone();
801 preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
802
803 match classify_statefulset_change(&latest_current, &latest_desired) {
804 StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
805 StatefulSetApplyStrategy::Recreate { immutable_fields } => {
806 validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
811 .await?;
812
813 info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
814 kanidm.delete(&ctx, &latest_current).await?;
815 ctx.kaniop_ctx
816 .metrics
817 .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
818 wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
819 .await?;
820 kanidm.apply(&ctx, latest_desired).await
821 }
822 }
823 }
824 }
825}
826
827fn is_kube_422(error: &Error) -> bool {
829 matches!(
830 error,
831 Error::KubeError(_, cause)
832 if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
833 )
834}
835
836#[cfg(test)]
837mod statefulset_recreate_recovery_tests {
838 use super::{Error, is_kube_422};
839
840 fn kube_api_error(code: u16) -> Error {
841 Error::KubeError(
842 "test error".to_string(),
843 Box::new(kube::Error::Api(Box::new(kube::error::Status {
844 code,
845 message: "test".to_string(),
846 reason: "test".to_string(),
847 ..Default::default()
848 }))),
849 )
850 }
851
852 #[test]
853 fn recreate_recovery_accepts_only_422() {
854 assert!(is_kube_422(&kube_api_error(422)));
855 assert!(!is_kube_422(&kube_api_error(403)));
856 assert!(!is_kube_422(&kube_api_error(429)));
857 assert!(!is_kube_422(&kube_api_error(500)));
858 }
859
860 #[test]
861 fn recreate_recovery_rejects_429_too_many_requests() {
862 assert!(!is_kube_422(&kube_api_error(429)));
863 }
864
865 #[test]
866 fn recreate_recovery_rejects_5xx_server_errors() {
867 assert!(!is_kube_422(&kube_api_error(500)));
868 assert!(!is_kube_422(&kube_api_error(502)));
869 assert!(!is_kube_422(&kube_api_error(503)));
870 assert!(!is_kube_422(&kube_api_error(504)));
871 }
872
873 #[test]
874 fn recreate_recovery_rejects_400_and_403() {
875 assert!(!is_kube_422(&kube_api_error(400)));
876 assert!(!is_kube_422(&kube_api_error(403)));
877 }
878}
879
880async fn reconcile(
881 kanidm: Arc<Kanidm>,
882 ctx: Arc<Context>,
883 status: KanidmStatus,
884) -> Result<(Action, bool)> {
885 if kanidm
886 .annotations()
887 .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
888 {
889 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
890 return Ok((Action::requeue(Duration::from_secs(5)), false));
891 }
892
893 let mut changed = false;
894 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
895 let replication_secret_futures =
896 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
897
898 let sts_to_delete = {
899 let expected_sts_names = kanidm
900 .spec
901 .replica_groups
902 .iter()
903 .map(|rg| kanidm.statefulset_name(&rg.name))
904 .collect::<Vec<_>>();
905 ctx.stores
906 .stateful_set_store
907 .state()
908 .into_iter()
909 .filter(|sts| {
910 sts.namespace() == kanidm.namespace()
911 && sts
912 .metadata
913 .labels
914 .as_ref()
915 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
916 && !expected_sts_names.contains(&sts.name_any())
917 })
918 .collect::<Vec<_>>()
919 };
920 if !sts_to_delete.is_empty() {
921 changed = true;
922 }
923 let sts_delete_futures = sts_to_delete
924 .iter()
925 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
926 .collect::<TryJoinAll<_>>();
927
928 let sts_futures = match kanidm.is_updatable(&status) {
929 true => {
930 let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
931 previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
932 });
933
934 let namespace = kanidm.get_namespace();
935 let schedule_api: Api<KanidmBackupSchedule> =
936 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
937 let schedules = schedule_api
938 .list(&Default::default())
939 .await
940 .map_err(|e| {
941 Error::KubeError(
942 "failed to list KanidmBackupSchedules for backup configuration resolution"
943 .to_string(),
944 Box::new(e),
945 )
946 })?
947 .items;
948
949 let repo_api: Api<KanidmBackupRepository> =
950 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
951 let repositories = repo_api
952 .list(&Default::default())
953 .await
954 .map_err(|e| {
955 Error::KubeError(
956 "failed to list KanidmBackupRepositories for backup configuration resolution"
957 .to_string(),
958 Box::new(e),
959 )
960 })?
961 .items;
962 let backup_config = resolve_backup_config(&kanidm, &schedules, &repositories);
963
964 kanidm
965 .spec
966 .replica_groups
967 .iter()
968 .map(|rg| {
969 let sts = kanidm.create_statefulset(
970 rg,
971 tls_secret_hash.as_deref(),
972 backup_config.as_ref(),
973 )?;
974 Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
975 })
976 .collect::<Result<TryJoinAll<_>, _>>()?
977 }
978 false => {
979 let note = match status.version.as_ref() {
980 Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
981 format!(
982 "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
983 v.image_tag,
984 crate::version::KANIDM_CLIENT_VERSION
985 )
986 }
987 _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
988 };
989 let _ignore_error = ctx
990 .kaniop_ctx
991 .recorder
992 .publish(
993 &Event {
994 type_: EventType::Warning,
995 reason: "UpgradeBlocked".to_string(),
996 note: Some(note),
997 action: "ReconcileStatefulSet".to_string(),
998 secondary: None,
999 },
1000 &kanidm.object_ref(&()),
1001 )
1002 .await
1003 .map_err(|e| {
1004 warn!(%e, "failed to publish KanidmError event");
1005 Error::KubeError("failed to publish event".to_string(), Box::new(e))
1006 });
1007 try_join_all([])
1008 }
1009 };
1010 let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1011
1012 let deprecated_rg_svcs = {
1013 let expected_rg_svcs_names = kanidm
1014 .spec
1015 .replica_groups
1016 .iter()
1017 .filter(|rg| rg.services.is_some())
1018 .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1019 .collect::<Vec<_>>();
1020 ctx.stores
1021 .service_store
1022 .state()
1023 .into_iter()
1024 .filter(|svc| {
1025 let labels = svc.metadata.labels.as_ref();
1026 svc.namespace() == kanidm.namespace()
1027 && !expected_rg_svcs_names.contains(&svc.name_any())
1028 && labels.is_some_and(|l| {
1029 l.contains_key(REPLICA_LABEL)
1030 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1031 })
1032 })
1033 .collect::<Vec<_>>()
1034 };
1035 if !deprecated_rg_svcs.is_empty() {
1036 changed = true;
1037 }
1038 let rg_svcs_delete_futures = deprecated_rg_svcs
1039 .iter()
1040 .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1041 .collect::<TryJoinAll<_>>();
1042
1043 let rg_services_futures = kanidm
1044 .spec
1045 .replica_groups
1046 .iter()
1047 .filter(|rg| rg.services.is_some())
1048 .flat_map(|rg| {
1049 (0..rg.replicas).map(|i| {
1050 let mut svc = kanidm.create_replica_group_service(rg, i);
1051 if let Some(current) = ctx.stores.service_store.find(|current| {
1052 current.namespace() == kanidm.namespace()
1053 && current.name_any() == svc.name_any()
1054 }) {
1055 preserve_defaulted_service_fields(&mut svc, current.as_ref());
1056 }
1057 kanidm.patch(&ctx, svc)
1058 })
1059 })
1060 .collect::<TryJoinAll<_>>();
1061
1062 let deprecated_ingresses = {
1063 let names = [
1064 kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1065 kanidm.generate_region_ingress_name(),
1066 ]
1067 .into_iter()
1068 .flatten()
1069 .collect::<Vec<_>>();
1070 ctx.stores
1071 .ingress_store
1072 .state()
1073 .into_iter()
1074 .filter(|ing| {
1075 ing.namespace() == kanidm.namespace()
1076 && !names.contains(&ing.name_any())
1077 && ing.metadata.labels == Some(kanidm.generate_labels())
1078 })
1079 .collect::<Vec<_>>()
1080 };
1081 if !deprecated_ingresses.is_empty() {
1082 changed = true;
1083 }
1084 let ingress_delete_futures = deprecated_ingresses
1085 .iter()
1086 .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1087 .collect::<TryJoinAll<_>>();
1088
1089 let ingress_futures = kanidm
1090 .create_ingress()
1091 .into_iter()
1092 .chain(kanidm.create_region_ingress())
1093 .map(|ingress| kanidm.patch(&ctx, ingress))
1094 .collect::<TryJoinAll<_>>();
1095
1096 let deprecated_http_routes = match &ctx.stores.http_route_store {
1097 Some(store) => {
1098 let expected_names = kanidm
1099 .spec
1100 .gateway
1101 .as_ref()
1102 .map(|_| vec![kanidm.name_any()])
1103 .unwrap_or_default();
1104 let routes: Vec<_> = store
1105 .state()
1106 .into_iter()
1107 .filter(|route| {
1108 route.namespace() == kanidm.namespace()
1109 && !expected_names.contains(&route.name_any())
1110 && route.metadata.labels == Some(kanidm.generate_labels())
1111 })
1112 .collect();
1113 if !routes.is_empty() {
1114 changed = true;
1115 }
1116 routes
1117 }
1118 None => Vec::new(),
1119 };
1120 let http_route_delete_futures = deprecated_http_routes
1121 .iter()
1122 .map(|route| kanidm.delete(&ctx, route.as_ref()))
1123 .collect::<TryJoinAll<_>>();
1124
1125 let http_route_futures = if ctx.stores.http_route_store.is_some() {
1126 kanidm
1127 .create_http_route()
1128 .into_iter()
1129 .map(|route| kanidm.patch(&ctx, route))
1130 .collect::<TryJoinAll<_>>()
1131 } else {
1132 try_join_all(Vec::new())
1133 };
1134
1135 let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1136 Some(store) => {
1137 let expected_names = kanidm
1138 .spec
1139 .gateway
1140 .as_ref()
1141 .and_then(|g| g.backend_tls_policy.as_ref())
1142 .map(|_| vec![kanidm.name_any()])
1143 .unwrap_or_default();
1144 let policies: Vec<_> = store
1145 .state()
1146 .into_iter()
1147 .filter(|policy| {
1148 policy.namespace() == kanidm.namespace()
1149 && !expected_names.contains(&policy.name_any())
1150 && policy.metadata.labels == Some(kanidm.generate_labels())
1151 })
1152 .collect();
1153 if !policies.is_empty() {
1154 changed = true;
1155 }
1156 policies
1157 }
1158 None => Vec::new(),
1159 };
1160 let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1161 .iter()
1162 .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1163 .collect::<TryJoinAll<_>>();
1164
1165 let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1166 kanidm
1167 .create_backend_tls_policy()
1168 .into_iter()
1169 .map(|policy| kanidm.patch(&ctx, policy))
1170 .collect::<TryJoinAll<_>>()
1171 } else {
1172 try_join_all(Vec::new())
1173 };
1174
1175 let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1176 sts_delete_futures,
1177 admin_secret_future,
1178 replication_secret_futures,
1179 sts_futures,
1180 service_future,
1181 rg_svcs_delete_futures,
1182 rg_services_futures,
1183 ingress_delete_futures,
1184 ingress_futures,
1185 http_route_delete_futures,
1186 http_route_futures,
1187 backend_tls_policy_delete_futures,
1188 backend_tls_policy_futures
1189 )?;
1190
1191 if admin_secret_changed || replication_secrets_changed {
1192 changed = true;
1193 }
1194
1195 if is_kanidm_available(status.clone()) {
1196 let namespace = kanidm.namespace().unwrap();
1197 let name = kanidm.name_any();
1198 match crate::controller::kanidm::KanidmClients::create_client(
1199 &namespace,
1200 &name,
1201 crate::controller::kanidm::KanidmUser::Admin,
1202 ctx.kaniop_ctx.client.clone(),
1203 )
1204 .await
1205 {
1206 Ok(system_client) => {
1207 if let Err(e) =
1208 reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1209 {
1210 warn!(%e, "failed to reconcile domain appearance");
1211 }
1212 }
1213 Err(e) => {
1214 warn!(%e, "failed to create admin client for domain appearance reconciliation");
1215 }
1216 }
1217
1218 match crate::controller::kanidm::KanidmClients::create_client(
1219 &namespace,
1220 &name,
1221 crate::controller::kanidm::KanidmUser::IdmAdmin,
1222 ctx.kaniop_ctx.client.clone(),
1223 )
1224 .await
1225 {
1226 Ok(kanidm_client) => {
1227 let (mail_sender_status, mail_sender_mutated) =
1228 reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1229 .await
1230 .unwrap_or_else(|e| {
1231 warn!(%e, "failed to reconcile mail sender");
1232 (None, false)
1233 });
1234 if mail_sender_mutated {
1235 changed = true;
1236 }
1237 let current_mail_sender_status =
1238 kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1239 if mail_sender_status != current_mail_sender_status {
1240 let kanidm_api: Api<Kanidm> =
1241 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1242 let status_patch = serde_json::json!({
1243 "status": {
1244 "mailSender": mail_sender_status
1245 }
1246 });
1247 if let Err(e) = kanidm_api
1248 .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1249 .await
1250 {
1251 warn!(%e, "failed to patch Kanidm/status for mail sender");
1252 }
1253 }
1254 }
1255 Err(e) => {
1256 warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1257 }
1258 }
1259 }
1260
1261 Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1262}
1263
1264async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1265 debug!("cleanup");
1266
1267 let mut changed = false;
1268 let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1269 if mail_resources_cleaned {
1270 changed = true;
1271 }
1272
1273 let namespace = kanidm.namespace().unwrap();
1274 let name = kanidm.name_any();
1275
1276 if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1277 &namespace,
1278 &name,
1279 crate::controller::kanidm::KanidmUser::IdmAdmin,
1280 ctx.kaniop_ctx.client.clone(),
1281 )
1282 .await
1283 {
1284 let kanidm_cleanup_done =
1285 cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1286 if kanidm_cleanup_done {
1287 changed = true;
1288 }
1289 }
1290
1291 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1292 Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1293}
1294
1295async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1296 let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1297 let start = std::time::Instant::now();
1298 let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1299 let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1300
1301 loop {
1302 let sts = sts_api.get(sts_name).await.map_err(|e| {
1303 Error::KubeError(
1304 format!("failed to get StatefulSet {namespace}/{sts_name}"),
1305 Box::new(e),
1306 )
1307 })?;
1308
1309 if sts.metadata.deletion_timestamp.is_some() {
1310 return Err(Error::ReceiveOutput(format!(
1311 "StatefulSet {namespace}/{sts_name} is being deleted"
1312 )));
1313 }
1314
1315 let spec = sts.spec.as_ref().ok_or_else(|| {
1316 Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1317 })?;
1318
1319 let status = match sts.status {
1320 Some(ref s) => s,
1321 None => {
1322 if start.elapsed() >= timeout {
1323 return Err(Error::ReceiveOutput(format!(
1324 "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1325 )));
1326 }
1327 sleep(poll_interval).await;
1328 continue;
1329 }
1330 };
1331
1332 let replicas = spec.replicas.unwrap_or(0);
1333
1334 if replicas == 0 {
1335 return Ok(());
1336 }
1337
1338 let updated_replicas = status.updated_replicas.unwrap_or(0);
1339 let current_revision = status.current_revision.as_deref().unwrap_or("");
1340 let update_revision = status.update_revision.as_deref().unwrap_or("");
1341 let ready_replicas = status.ready_replicas.unwrap_or(0);
1342
1343 if updated_replicas == replicas
1344 && current_revision == update_revision
1345 && ready_replicas == replicas
1346 {
1347 return Ok(());
1348 }
1349
1350 if start.elapsed() >= timeout {
1351 return Err(Error::ReceiveOutput(format!(
1352 "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1353 )));
1354 }
1355
1356 sleep(poll_interval).await;
1357 }
1358}
1359
1360async fn show_replica_cert_with_retries(
1361 kanidm: &Kanidm,
1362 ctx: Arc<Context>,
1363 pod_name: &str,
1364 max_attempts: u32,
1365) -> std::result::Result<String, Error> {
1366 let mut last_error = None;
1367 for attempt in 0..max_attempts {
1368 if attempt > 0 {
1369 sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1370 }
1371 match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1372 Ok(cert) => return Ok(cert),
1373 Err(e) => {
1374 info!(
1375 "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1376 attempt + 1,
1377 max_attempts
1378 );
1379 last_error = Some(e);
1380 }
1381 }
1382 }
1383 Err(last_error.unwrap_or_else(|| {
1384 Error::ReceiveOutput(format!(
1385 "failed to get certificate for {pod_name} after {max_attempts} attempts"
1386 ))
1387 }))
1388}
1389
1390impl Kanidm {
1391 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1393 where
1394 K: Resource<Scope = NamespaceResourceScope>
1395 + Serialize
1396 + Clone
1397 + std::fmt::Debug
1398 + for<'de> Deserialize<'de>,
1399 <K as kube::Resource>::DynamicType: Default,
1400 <K as Resource>::Scope: std::marker::Sized,
1401 {
1402 self.kube_delete(
1403 ctx.kaniop_ctx.client.clone(),
1404 &ctx.kaniop_ctx.metrics,
1405 resource,
1406 )
1407 .await
1408 }
1409
1410 pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1413 where
1414 K: Resource<Scope = NamespaceResourceScope>
1415 + Serialize
1416 + Clone
1417 + std::fmt::Debug
1418 + for<'de> Deserialize<'de>,
1419 <K as kube::Resource>::DynamicType: Default,
1420 <K as Resource>::Scope: std::marker::Sized,
1421 {
1422 self.kube_apply(
1423 ctx.kaniop_ctx.client.clone(),
1424 resource,
1425 KANIDM_OPERATOR_NAME,
1426 )
1427 .await
1428 }
1429
1430 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1431 where
1432 K: Resource<Scope = NamespaceResourceScope>
1433 + Serialize
1434 + Clone
1435 + std::fmt::Debug
1436 + for<'de> Deserialize<'de>,
1437 <K as kube::Resource>::DynamicType: Default,
1438 <K as Resource>::Scope: std::marker::Sized,
1439 {
1440 self.kube_patch(
1441 ctx.kaniop_ctx.client.clone(),
1442 &ctx.kaniop_ctx.metrics,
1443 resource,
1444 KANIDM_OPERATOR_NAME,
1445 )
1446 .await
1447 }
1448
1449 #[inline]
1450 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1451 LABELS
1452 .clone()
1453 .into_iter()
1454 .chain([
1455 (INSTANCE_LABEL.to_string(), self.name_any()),
1456 (CLUSTER_LABEL.to_string(), self.name_any()),
1457 ])
1458 .collect()
1459 }
1460
1461 #[inline]
1462 fn generate_labels(&self) -> BTreeMap<String, String> {
1463 self.generate_resource_labels()
1464 }
1465
1466 #[inline]
1467 fn get_tls_secret_name(&self) -> String {
1468 format!("{}-tls", self.name_any())
1469 }
1470
1471 pub(crate) fn effective_tls_secret_name(&self) -> String {
1474 self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1475 self.spec
1476 .ingress
1477 .as_ref()
1478 .and_then(|i| i.tls_secret_name.clone())
1479 .unwrap_or_else(|| self.get_tls_secret_name())
1480 })
1481 }
1482
1483 #[inline]
1484 fn get_namespace(&self) -> String {
1485 self.namespace().unwrap()
1487 }
1488
1489 #[inline]
1490 fn is_replication_enabled(&self) -> bool {
1491 self.spec.replica_groups.len() > 1
1492 || self
1493 .spec
1494 .replica_groups
1495 .first()
1496 .is_some_and(|rg| rg.replicas > 1)
1497 || !self.spec.external_replication_nodes.is_empty()
1498 }
1499
1500 #[inline]
1501 fn is_updatable(&self, status: &KanidmStatus) -> bool {
1502 status
1503 .version
1504 .as_ref()
1505 .map(|v| {
1506 if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1507 return false;
1508 }
1509 let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1510 let versions = (parse_semver(¤t_tag), parse_semver(&v.image_tag));
1511 match v.upgrade_check_result {
1512 KanidmUpgradeCheckResult::Passed => {
1513 if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1514 minor <= v_minor + 1
1515 } else {
1516 true
1517 }
1518 }
1519 KanidmUpgradeCheckResult::Failed => {
1520 if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1521 versions
1522 {
1523 major == v_major && minor == v_minor && patch >= v_patch
1524 } else {
1525 v.image_tag == current_tag
1526 }
1527 }
1528 }
1529 })
1530 .unwrap_or(true)
1531 }
1532
1533 async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1534 let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1535 let pod = pod_api.get(pod_name).await.map_err(|e| {
1536 Error::KubeError(
1537 format!("failed to get pod {namespace}/{pod_name}"),
1538 Box::new(e),
1539 )
1540 })?;
1541 Ok(pod
1542 .status
1543 .as_ref()
1544 .and_then(|s| s.conditions.as_ref())
1545 .is_some_and(|conditions| {
1546 conditions
1547 .iter()
1548 .any(|c| c.type_ == "Ready" && c.status == "True")
1549 }))
1550 }
1551
1552 async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1553 let start = std::time::Instant::now();
1554 let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1555 let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1556
1557 loop {
1558 match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1559 Ok(true) => return Ok(()),
1560 Ok(false) => {}
1561 Err(e) => {
1562 if !e.is_retryable() {
1563 return Err(e);
1564 }
1565 trace!(%e, "pod readiness check failed, retrying");
1566 }
1567 }
1568
1569 if start.elapsed() >= timeout {
1570 return Err(Error::ReceiveOutput(format!(
1571 "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1572 )));
1573 }
1574
1575 sleep(poll_interval).await;
1576 }
1577 }
1578
1579 async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1580 where
1581 I: IntoIterator<Item = T> + Debug,
1582 T: Into<String>,
1583 {
1584 let namespace = &self.get_namespace();
1585 trace!(
1586 resource.name = &pod_name,
1587 resource.namespace = &namespace,
1588 ?command,
1589 "pod exec"
1590 );
1591 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1592 let attached = pod
1593 .exec(
1594 pod_name,
1595 command,
1596 &AttachParams::default().container("kanidm"),
1597 )
1598 .await
1599 .map_err(|e| {
1600 Error::KubeError(
1601 format!("failed to exec pod {namespace}/{pod_name}"),
1602 Box::new(e),
1603 )
1604 })?;
1605 get_output(attached).await
1606 }
1607
1608 async fn exec_with_wait<I, T>(
1609 &self,
1610 ctx: Arc<Context>,
1611 pod_name: &str,
1612 command: I,
1613 ) -> Result<String>
1614 where
1615 I: IntoIterator<Item = T> + Debug,
1616 T: Into<String>,
1617 {
1618 let namespace = &self.get_namespace();
1619 Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1620 self.exec(ctx, pod_name, command).await
1621 }
1622
1623 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1624 where
1625 I: IntoIterator<Item = T> + Debug,
1626 T: Into<String>,
1627 {
1628 let first_replica_group = self
1630 .spec
1631 .replica_groups
1632 .first()
1633 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1634 let sts_name = self.statefulset_name(&first_replica_group.name);
1635 let pod_name = format!("{sts_name}-0");
1636 self.exec(ctx, &pod_name, command).await
1637 }
1638
1639 async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1640 where
1641 I: IntoIterator<Item = T> + Debug,
1642 T: Into<String>,
1643 {
1644 let first_replica_group = self
1645 .spec
1646 .replica_groups
1647 .first()
1648 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1649 let sts_name = self.statefulset_name(&first_replica_group.name);
1650 let pod_name = format!("{sts_name}-0");
1651 self.exec_with_wait(ctx, &pod_name, command).await
1652 }
1653}
1654
1655#[cfg(test)]
1656mod test {
1657 use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1658 use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1659
1660 use crate::controller::State;
1661 use crate::kanidm::controller::context::{Context, Stores};
1662 use crate::kanidm::crd::KanidmStatus;
1663
1664 use kaniop_k8s_util::error::Result;
1665 use kaniop_k8s_util::resources::hash_secret_data;
1666
1667 use std::collections::BTreeMap;
1668 use std::sync::Arc;
1669
1670 use http::{Request, Response};
1671 use k8s_openapi::ByteString;
1672 use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1673 use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1674 use k8s_openapi::api::networking::v1::Ingress;
1675 use kube::api::ObjectMeta;
1676 use kube::runtime::reflector::store::Writer;
1677 use kube::{Client, Resource, ResourceExt, client::Body};
1678 use opentelemetry::metrics::MeterProvider;
1679 use serde_json::json;
1680
1681 impl Kanidm {
1682 pub fn test() -> Self {
1684 let mut e = Kanidm::new(
1685 "test",
1686 serde_json::from_value(json!({
1687 "domain": "idm.example.com",
1688 "replicaGroups": [
1689 {
1690 "name": "default",
1691 "replicas": 1
1692 }
1693 ]
1694 }))
1695 .unwrap(),
1696 );
1697 e.meta_mut().namespace = Some("default".into());
1698 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1701 e
1702 }
1703
1704 pub fn with_ingress(mut self) -> Self {
1705 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1706 self
1707 }
1708
1709 pub fn with_replicas(mut self, replicas: i32) -> Self {
1711 self.spec.replica_groups[0].replicas = replicas;
1712 self
1713 }
1714
1715 pub fn needs_delete(mut self) -> Self {
1717 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1718 use k8s_openapi::jiff::Timestamp;
1719 let now = Timestamp::from_second(1491138632).unwrap(); self.meta_mut().deletion_timestamp = Some(Time(now));
1721 self
1722 }
1723
1724 pub fn with_status(mut self, status: KanidmStatus) -> Self {
1726 self.status = Some(status);
1727 self
1728 }
1729 }
1730
1731 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1733 pub struct ApiServerVerifier(ApiServerHandle);
1734
1735 pub enum Scenario {
1737 Create(Kanidm),
1738 CreateWithTwoReplicas(Kanidm),
1739 CreateWithIngress(Kanidm),
1740 CreateWithIngressWithTwoReplicas(Kanidm),
1741 CreateWithTlsSecret(Kanidm, Box<Secret>),
1742 }
1743
1744 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1745 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1746 .await
1747 .expect("timeout on mock apiserver")
1748 .expect("scenario succeeded")
1749 }
1750
1751 impl ApiServerVerifier {
1752 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1763 tokio::spawn(async move {
1764 match scenario {
1766 Scenario::Create(kanidm) => {
1767 self.handle_kanidm_get(kanidm.clone())
1768 .await
1769 .unwrap()
1770 .handle_kanidm_status_patch(kanidm.clone())
1771 .await
1772 .unwrap()
1773 .handle_tls_secret_get_not_found(kanidm.clone())
1774 .await
1775 .unwrap()
1776 .handle_backup_schedule_list()
1777 .await
1778 .unwrap()
1779 .handle_backup_repository_list()
1780 .await
1781 .unwrap()
1782 .handle_statefulset_patch(kanidm.clone(), None)
1783 .await
1784 .unwrap()
1785 .handle_service_patch(kanidm.clone())
1786 .await
1787 }
1788 Scenario::CreateWithTwoReplicas(kanidm) => {
1789 self.handle_kanidm_get(kanidm.clone())
1790 .await
1791 .unwrap()
1792 .handle_kanidm_status_patch(kanidm.clone())
1793 .await
1794 .unwrap()
1795 .handle_tls_secret_get_not_found(kanidm.clone())
1796 .await
1797 .unwrap()
1798 .handle_backup_schedule_list()
1799 .await
1800 .unwrap()
1801 .handle_backup_repository_list()
1802 .await
1803 .unwrap()
1804 .handle_statefulset_patch(kanidm.clone(), None)
1805 .await
1806 .unwrap()
1807 .handle_service_patch(kanidm.clone())
1808 .await
1809 }
1810 Scenario::CreateWithIngress(kanidm) => {
1811 self.handle_kanidm_get(kanidm.clone())
1812 .await
1813 .unwrap()
1814 .handle_kanidm_status_patch(kanidm.clone())
1815 .await
1816 .unwrap()
1817 .handle_tls_secret_get_not_found(kanidm.clone())
1818 .await
1819 .unwrap()
1820 .handle_backup_schedule_list()
1821 .await
1822 .unwrap()
1823 .handle_backup_repository_list()
1824 .await
1825 .unwrap()
1826 .handle_statefulset_patch(kanidm.clone(), None)
1827 .await
1828 .unwrap()
1829 .handle_service_patch(kanidm.clone())
1830 .await
1831 .unwrap()
1832 .handle_ingress_patch(kanidm.clone())
1833 .await
1834 }
1835 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1836 self.handle_kanidm_get(kanidm.clone())
1837 .await
1838 .unwrap()
1839 .handle_kanidm_status_patch(kanidm.clone())
1840 .await
1841 .unwrap()
1842 .handle_tls_secret_get_not_found(kanidm.clone())
1843 .await
1844 .unwrap()
1845 .handle_backup_schedule_list()
1846 .await
1847 .unwrap()
1848 .handle_backup_repository_list()
1849 .await
1850 .unwrap()
1851 .handle_statefulset_patch(kanidm.clone(), None)
1852 .await
1853 .unwrap()
1854 .handle_service_patch(kanidm.clone())
1855 .await
1856 .unwrap()
1857 .handle_ingress_patch(kanidm.clone())
1858 .await
1859 }
1860 Scenario::CreateWithTlsSecret(kanidm, secret) => {
1861 let expected_hash = hash_secret_data(&secret);
1862 self.handle_kanidm_get(kanidm.clone())
1863 .await
1864 .unwrap()
1865 .handle_kanidm_status_patch(kanidm.clone())
1866 .await
1867 .unwrap()
1868 .handle_tls_secret_get(kanidm.clone(), *secret)
1869 .await
1870 .unwrap()
1871 .handle_backup_schedule_list()
1872 .await
1873 .unwrap()
1874 .handle_backup_repository_list()
1875 .await
1876 .unwrap()
1877 .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1878 .await
1879 .unwrap()
1880 .handle_service_patch(kanidm.clone())
1881 .await
1882 }
1883 }
1884 .expect("scenario completed without errors");
1885 })
1886 }
1887
1888 async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1889 let (request, send) = self.0.next_request().await.expect("service not called");
1890 assert_eq!(request.method(), http::Method::GET);
1891 let expected_uri_prefix = format!(
1892 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1893 kanidm.name_any()
1894 );
1895 let uri = request.uri().to_string();
1896 assert!(
1897 uri.starts_with(&expected_uri_prefix),
1898 "expected uri to start with {}, got {}",
1899 expected_uri_prefix,
1900 uri
1901 );
1902 let response = serde_json::to_vec(&kanidm).unwrap();
1903 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1904 Ok(self)
1905 }
1906
1907 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1908 let (request, send) = self.0.next_request().await.expect("service not called");
1909 assert_eq!(request.method(), http::Method::PATCH);
1910 assert_eq!(
1911 request.uri().to_string(),
1912 format!(
1913 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1914 kanidm.name_any()
1915 )
1916 );
1917
1918 let req_body = request.into_body().collect_bytes().await.unwrap();
1919 let json: serde_json::Value =
1920 serde_json::from_slice(&req_body).expect("patch object is json");
1921 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1922 .expect("valid kanidm status");
1923 let mut kanidm = Kanidm::test();
1924 kanidm.status = Some(status.clone());
1925 let response = serde_json::to_vec(&kanidm).unwrap();
1926 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1928 Ok(self)
1929 }
1930
1931 async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1932 let (request, send) = self.0.next_request().await.expect("service not called");
1933 assert_eq!(request.method(), http::Method::GET);
1934 let expected_uri_prefix = format!(
1935 "/api/v1/namespaces/default/secrets/{}",
1936 kanidm.effective_tls_secret_name()
1937 );
1938 let uri = request.uri().to_string();
1939 assert!(
1940 uri.starts_with(&expected_uri_prefix),
1941 "expected uri to start with {expected_uri_prefix}, got {uri}"
1942 );
1943 let status = serde_json::json!({
1944 "kind": "Status",
1945 "apiVersion": "v1",
1946 "metadata": {},
1947 "status": "Failure",
1948 "message": "secret not found",
1949 "reason": "NotFound",
1950 "code": 404
1951 });
1952 send.send_response(
1953 Response::builder()
1954 .status(404)
1955 .body(Body::from(serde_json::to_vec(&status).unwrap()))
1956 .unwrap(),
1957 );
1958 Ok(self)
1959 }
1960
1961 async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1962 let (request, send) = self.0.next_request().await.expect("service not called");
1963 assert_eq!(request.method(), http::Method::GET);
1964 let expected_uri_prefix = format!(
1965 "/api/v1/namespaces/default/secrets/{}",
1966 kanidm.effective_tls_secret_name()
1967 );
1968 let uri = request.uri().to_string();
1969 assert!(
1970 uri.starts_with(&expected_uri_prefix),
1971 "expected uri to start with {expected_uri_prefix}, got {uri}"
1972 );
1973 let response = serde_json::to_vec(&secret).unwrap();
1974 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1975 Ok(self)
1976 }
1977
1978 async fn handle_backup_schedule_list(mut self) -> Result<Self> {
1979 let (request, send) = self.0.next_request().await.expect("service not called");
1980 assert_eq!(request.method(), http::Method::GET);
1981 assert!(
1982 request
1983 .uri()
1984 .to_string()
1985 .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
1986 "expected backup schedule list request, got {}",
1987 request.uri()
1988 );
1989 let list = json!({
1990 "apiVersion": "kaniop.rs/v1alpha1",
1991 "kind": "KanidmBackupScheduleList",
1992 "metadata": {},
1993 "items": []
1994 });
1995 let response = serde_json::to_vec(&list).unwrap();
1996 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1997 Ok(self)
1998 }
1999
2000 async fn handle_backup_repository_list(mut self) -> Result<Self> {
2001 let (request, send) = self.0.next_request().await.expect("service not called");
2002 assert_eq!(request.method(), http::Method::GET);
2003 assert!(
2004 request.uri().to_string().contains(
2005 "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2006 ),
2007 "expected backup repository list request, got {}",
2008 request.uri()
2009 );
2010 let list = json!({
2011 "apiVersion": "kaniop.rs/v1alpha1",
2012 "kind": "KanidmBackupRepositoryList",
2013 "metadata": {},
2014 "items": []
2015 });
2016 let response = serde_json::to_vec(&list).unwrap();
2017 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2018 Ok(self)
2019 }
2020
2021 async fn handle_statefulset_patch(
2022 mut self,
2023 kanidm: Kanidm,
2024 expected_tls_hash: Option<String>,
2025 ) -> Result<Self> {
2026 for rg in kanidm.spec.replica_groups.iter() {
2027 let (request, send) = self.0.next_request().await.expect("service not called");
2028 assert_eq!(request.method(), http::Method::PATCH);
2029 assert_eq!(
2030 request.uri().to_string(),
2031 format!(
2032 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2033 kanidm.statefulset_name(&rg.name)
2034 )
2035 );
2036 let req_body = request.into_body().collect_bytes().await.unwrap();
2037 let json: serde_json::Value =
2038 serde_json::from_slice(&req_body).expect("patch object is json");
2039 let statefulset: StatefulSet =
2040 serde_json::from_value(json).expect("valid statefulset");
2041 assert_eq!(
2042 statefulset.clone().spec.unwrap().replicas.unwrap(),
2043 rg.replicas
2044 );
2045 let tls_hash_annotation = statefulset
2046 .spec
2047 .as_ref()
2048 .unwrap()
2049 .template
2050 .metadata
2051 .as_ref()
2052 .unwrap()
2053 .annotations
2054 .as_ref()
2055 .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2056 .cloned();
2057 assert_eq!(tls_hash_annotation, expected_tls_hash);
2058 let response = serde_json::to_vec(&statefulset).unwrap();
2059 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2061 }
2062
2063 Ok(self)
2064 }
2065
2066 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2067 let (request, send) = self.0.next_request().await.expect("service not called");
2068 assert_eq!(request.method(), http::Method::PATCH);
2069 assert_eq!(
2070 request.uri().to_string(),
2071 format!(
2072 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2073 kanidm.name_any()
2074 )
2075 );
2076
2077 let req_body = request.into_body().collect_bytes().await.unwrap();
2078 let json: serde_json::Value =
2079 serde_json::from_slice(&req_body).expect("patch object is json");
2080 let service: Service = serde_json::from_value(json).expect("valid service");
2081 let response = serde_json::to_vec(&service).unwrap();
2082 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2084 Ok(self)
2085 }
2086
2087 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2088 let (request, send) = self.0.next_request().await.expect("service not called");
2089 assert_eq!(request.method(), http::Method::PATCH);
2090 assert_eq!(
2091 request.uri().to_string(),
2092 format!(
2093 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2094 kanidm.name_any()
2095 )
2096 );
2097
2098 let req_body = request.into_body().collect_bytes().await.unwrap();
2099 let json: serde_json::Value =
2100 serde_json::from_slice(&req_body).expect("patch object is json");
2101 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2102 let response = serde_json::to_vec(&ingress).unwrap();
2103 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2105 Ok(self)
2106 }
2107 }
2108
2109 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2110 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2111 let mock_client = Client::new(mock_service, "default");
2112 let stores = Stores {
2113 stateful_set_store: Writer::default().as_reader(),
2114 service_store: Writer::default().as_reader(),
2115 ingress_store: Writer::default().as_reader(),
2116 secret_store: Writer::default().as_reader(),
2117 http_route_store: Some(Writer::default().as_reader()),
2118 backend_tls_policy_store: Some(Writer::default().as_reader()),
2119 deployment_store: Writer::default().as_reader(),
2120 config_map_store: Writer::default().as_reader(),
2121 };
2122 let controller_id = "test";
2123
2124 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2126 let meter = provider.meter("test");
2127 let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2128
2129 let state = State::new(
2130 metrics,
2131 Writer::default().as_reader(),
2132 Writer::default().as_reader(),
2133 None,
2134 );
2135 let ctx = Arc::new(Context::new(
2136 state.to_context(mock_client, controller_id),
2137 stores,
2138 ));
2139 (ctx, ApiServerVerifier(handle))
2140 }
2141
2142 #[tokio::test]
2143 async fn kanidm_create() {
2144 let (testctx, fakeserver) = get_test_context();
2145 let kanidm = Kanidm::test();
2146 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2147 reconcile_kanidm(Arc::new(kanidm), testctx)
2148 .await
2149 .expect("reconciler");
2150 timeout_after_1s(mocksrv).await;
2151 }
2152
2153 #[tokio::test]
2154 async fn kanidm_create_with_two_replicas() {
2155 let (testctx, fakeserver) = get_test_context();
2156 let kanidm = Kanidm::test().with_replicas(2);
2157 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2158 reconcile_kanidm(Arc::new(kanidm), testctx)
2159 .await
2160 .expect("reconciler");
2161 timeout_after_1s(mocksrv).await;
2162 }
2163
2164 #[tokio::test]
2165 async fn kanidm_create_with_ingress() {
2166 let (testctx, fakeserver) = get_test_context();
2167 let kanidm = Kanidm::test().with_ingress();
2168 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2169 reconcile_kanidm(Arc::new(kanidm), testctx)
2170 .await
2171 .expect("reconciler");
2172 timeout_after_1s(mocksrv).await;
2173 }
2174
2175 #[tokio::test]
2176 async fn kanidm_create_with_ingress_with_two_replicas() {
2177 let (testctx, fakeserver) = get_test_context();
2178 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2179 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2180 reconcile_kanidm(Arc::new(kanidm), testctx)
2181 .await
2182 .expect("reconciler");
2183 timeout_after_1s(mocksrv).await;
2184 }
2185
2186 fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2187 Arc::new(StatefulSet {
2188 metadata: ObjectMeta {
2189 name: Some(format!("{cluster}-default")),
2190 namespace: Some(namespace.to_string()),
2191 labels: Some(BTreeMap::from([(
2192 CLUSTER_LABEL.to_string(),
2193 cluster.to_string(),
2194 )])),
2195 ..Default::default()
2196 },
2197 spec: Some(StatefulSetSpec {
2198 template: PodTemplateSpec {
2199 metadata: Some(ObjectMeta {
2200 annotations: tls_hash.map(|h| {
2201 BTreeMap::from([(
2202 TLS_SECRET_HASH_ANNOTATION.to_string(),
2203 h.to_string(),
2204 )])
2205 }),
2206 ..Default::default()
2207 }),
2208 ..Default::default()
2209 },
2210 ..Default::default()
2211 }),
2212 ..Default::default()
2213 })
2214 }
2215
2216 #[test]
2217 fn test_previous_tls_secret_hash() {
2218 let kanidm = Kanidm::test();
2219
2220 let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2222 assert_eq!(
2223 previous_tls_secret_hash(&kanidm, stss),
2224 Some("abc123".to_string())
2225 );
2226
2227 let stss = vec![statefulset_of("test", "default", None)];
2229 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2230
2231 let stss = vec![
2233 statefulset_of("other", "default", Some("abc123")),
2234 statefulset_of("test", "other-ns", Some("abc123")),
2235 ];
2236 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2237 }
2238
2239 #[tokio::test]
2240 async fn kanidm_create_with_tls_secret() {
2241 let (testctx, fakeserver) = get_test_context();
2242 let kanidm = Kanidm::test();
2243 let secret = Secret {
2244 metadata: ObjectMeta {
2245 name: Some(kanidm.effective_tls_secret_name()),
2246 namespace: Some("default".to_string()),
2247 ..Default::default()
2248 },
2249 data: Some(BTreeMap::from([
2250 ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2251 ("tls.key".to_string(), ByteString(b"key".to_vec())),
2252 ])),
2253 ..Default::default()
2254 };
2255 let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2256 kanidm.clone(),
2257 Box::new(secret),
2258 ));
2259 reconcile_kanidm(Arc::new(kanidm), testctx)
2260 .await
2261 .expect("reconciler");
2262 timeout_after_1s(mocksrv).await;
2263 }
2264}