1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12 cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::StatefulSetExt;
21use self::status::StatusExt;
22use self::status::{is_kanidm_available, is_kanidm_initialized};
23
24use crate::controller::context::KubeOperations;
25use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
26use crate::kanidm::crd::{
27 Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
28};
29use crate::kanidm::reconcile::statefulset::REPLICA_LABEL;
30use crate::telemetry;
31
32use kaniop_k8s_util::client::get_output;
33use kaniop_k8s_util::error::{Error, Result};
34use kaniop_k8s_util::parse::parse_semver;
35use kaniop_k8s_util::resources::get_image_tag;
36
37use std::collections::{BTreeMap, BTreeSet};
38use std::fmt::Debug;
39use std::sync::{Arc, LazyLock};
40
41use futures::future::{TryJoinAll, join_all, try_join_all};
42use futures::stream::{self, StreamExt};
43use futures::try_join;
44use k8s_openapi::NamespaceResourceScope;
45use k8s_openapi::api::apps::v1::StatefulSet;
46use k8s_openapi::api::core::v1::Pod;
47use kube::Resource;
48use kube::ResourceExt;
49use kube::api::{Api, AttachParams, Patch, PatchParams};
50use kube::runtime::controller::Action;
51use kube::runtime::events::{Event, EventType};
52use kube::runtime::finalizer::{Event as Finalizer, finalizer};
53use serde::{Deserialize, Serialize};
54use tokio::time::{Duration, sleep};
55use tracing::{Span, debug, field, info, instrument, trace, warn};
56
57const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
58const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
59const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
60const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
61const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
62const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
63const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
64
65pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
66const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
67pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
68
69const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
70
71static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
72 BTreeMap::from([
73 (NAME_LABEL.to_string(), "kanidm".to_string()),
74 (
75 MANAGED_BY_LABEL.to_string(),
76 format!("kaniop-{CONTROLLER_ID}"),
77 ),
78 ])
79});
80
81async fn restart_statefulsets_after_replica_secret_update(
82 ctx: Arc<Context>,
83 sts_api: &Api<StatefulSet>,
84 namespace: &str,
85 statefulset_names: &[String],
86 sequential: bool,
87) -> Result<()> {
88 let restart_futures = statefulset_names
91 .iter()
92 .map(|sts_name| sts_api.restart(sts_name));
93
94 if sequential {
95 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
96 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
97 return Err(Error::kube_error(
98 "restart",
99 "StatefulSet",
100 namespace,
101 "replica certificate secret update statefulsets",
102 first_err,
103 ));
104 }
105 } else {
106 let results: Vec<_> = join_all(restart_futures).await;
107 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
108 return Err(Error::kube_error(
109 "restart",
110 "StatefulSet",
111 namespace,
112 "replica certificate secret update statefulsets",
113 first_err,
114 ));
115 }
116 }
117
118 let rollout_futures = statefulset_names
119 .iter()
120 .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
121 try_join_all(rollout_futures).await?;
122
123 Ok(())
124}
125
126pub async fn reconcile_admins_secret(
127 kanidm: Arc<Kanidm>,
128 ctx: Arc<Context>,
129 status: &KanidmStatus,
130) -> Result<()> {
131 if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
132 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
133 kanidm.patch(&ctx, admins_secret).await?;
134 }
135 Ok(())
136}
137
138pub async fn reconcile_replication_secrets(
139 kanidm: Arc<Kanidm>,
140 ctx: Arc<Context>,
141 status: &KanidmStatus,
142) -> Result<()> {
143 let expected_secret_names = if kanidm.is_replication_enabled() {
144 status
145 .replica_statuses
146 .iter()
147 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
148 .collect::<Vec<_>>()
149 } else {
150 Vec::new()
151 };
152 let deprecated_secrets = ctx
153 .stores
154 .secret_store
155 .state()
156 .into_iter()
157 .filter(|secret| {
158 secret.namespace() == kanidm.namespace()
159 && kanidm.admins_secret_name() != secret.name_any()
160 && !expected_secret_names.contains(&secret.name_any())
161 && secret
162 .metadata
163 .labels
164 .as_ref()
165 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
166 })
167 .collect::<Vec<_>>();
168 let secret_delete_future = deprecated_secrets
169 .iter()
170 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
171 .collect::<TryJoinAll<_>>();
172 try_join!(secret_delete_future)?;
173
174 if kanidm.is_replication_enabled() {
175 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
176 let statefulset_names = status
177 .replica_statuses
178 .iter()
179 .map(|rs| rs.statefulset_name.clone())
180 .collect::<BTreeSet<_>>()
181 .into_iter()
182 .collect::<Vec<_>>();
183
184 let has_pending = status
185 .replica_statuses
186 .iter()
187 .any(|rs| rs.state == KanidmReplicaState::Pending);
188
189 let sts_api =
190 Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
191
192 if has_pending {
193 stream::iter(
194 status
195 .replica_statuses
196 .iter()
197 .filter(|rs| rs.state == KanidmReplicaState::Pending),
198 )
199 .then(|rs| async {
200 let secret = kanidm
201 .generate_replica_secret(ctx.clone(), &rs.pod_name)
202 .await?;
203 kanidm.patch(&ctx, secret).await
204 })
205 .collect::<Vec<_>>()
206 .await
207 .into_iter()
208 .collect::<Result<Vec<_>>>()?;
209
210 let restart_futures = status
211 .replica_statuses
212 .iter()
213 .map(|rs| sts_api.restart(&rs.statefulset_name));
214
215 if has_single_replica {
216 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
217 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
218 return Err(Error::kube_error(
219 "restart",
220 "StatefulSet",
221 kanidm.get_namespace(),
222 "pending replica statefulsets",
223 first_err,
224 ));
225 }
226 } else {
227 let results: Vec<_> = join_all(restart_futures).await;
228 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
229 return Err(Error::kube_error(
230 "restart",
231 "StatefulSet",
232 kanidm.get_namespace(),
233 "pending replica statefulsets",
234 first_err,
235 ));
236 }
237 }
238 }
239
240 let has_certificate_host_invalid = status
241 .replica_statuses
242 .iter()
243 .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
244
245 let has_certificate_expiring = status
246 .replica_statuses
247 .iter()
248 .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
249
250 let has_non_ready_replicas = status
251 .replica_statuses
252 .iter()
253 .any(|rs| rs.state != KanidmReplicaState::Ready);
254
255 let host_invalid_replicas: Vec<_> = status
256 .replica_statuses
257 .iter()
258 .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
259 .collect();
260
261 let cert_expiring_replicas: Vec<_> = status
262 .replica_statuses
263 .iter()
264 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
265 .collect();
266
267 if !host_invalid_replicas.is_empty() {
268 let restart_futures = host_invalid_replicas
269 .iter()
270 .map(|rs| sts_api.restart(&rs.statefulset_name));
271
272 if has_single_replica {
273 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
274 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
275 return Err(Error::kube_error(
276 "restart",
277 "StatefulSet",
278 kanidm.get_namespace(),
279 "certificate host invalid statefulsets",
280 first_err,
281 ));
282 }
283 } else {
284 let results: Vec<_> = join_all(restart_futures).await;
285 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
286 return Err(Error::kube_error(
287 "restart",
288 "StatefulSet",
289 kanidm.get_namespace(),
290 "certificate host invalid statefulsets",
291 first_err,
292 ));
293 }
294 }
295
296 let restarted_sts_names: Vec<_> = host_invalid_replicas
297 .iter()
298 .map(|rs| rs.statefulset_name.clone())
299 .collect::<BTreeSet<_>>()
300 .into_iter()
301 .collect();
302
303 let namespace = kanidm.get_namespace();
304 let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
305 wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
306 });
307 try_join_all(rollout_futures).await?;
308
309 let renew_futures = host_invalid_replicas
310 .iter()
311 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
312 let results: Vec<_> = join_all(renew_futures).await;
313 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
314 return Err(first_err);
315 }
316
317 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
318
319 let cert_futures = host_invalid_replicas.iter().map(|rs| async {
320 let cert = show_replica_cert_with_retries(
321 &kanidm,
322 ctx.clone(),
323 &rs.pod_name,
324 CERT_SHOW_RETRY_ATTEMPTS,
325 )
326 .await?;
327 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
328 kanidm.patch(&ctx, secret.clone()).await
329 });
330 try_join_all(cert_futures).await?;
331
332 restart_statefulsets_after_replica_secret_update(
333 ctx.clone(),
334 &sts_api,
335 &kanidm.get_namespace(),
336 &statefulset_names,
337 has_single_replica,
338 )
339 .await?;
340
341 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
342 }
343
344 if !cert_expiring_replicas.is_empty() {
345 let renew_futures = cert_expiring_replicas
346 .iter()
347 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
348 let results: Vec<_> = join_all(renew_futures).await;
349 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
350 return Err(first_err);
351 }
352
353 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
354
355 let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
356 let cert = show_replica_cert_with_retries(
357 &kanidm,
358 ctx.clone(),
359 &rs.pod_name,
360 CERT_SHOW_RETRY_ATTEMPTS,
361 )
362 .await?;
363 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
364 kanidm.patch(&ctx, secret.clone()).await
365 });
366 try_join_all(cert_futures).await?;
367
368 restart_statefulsets_after_replica_secret_update(
369 ctx.clone(),
370 &sts_api,
371 &kanidm.get_namespace(),
372 &statefulset_names,
373 has_single_replica,
374 )
375 .await?;
376
377 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
378 }
379
380 if has_non_ready_replicas
381 && !has_pending
382 && !has_certificate_host_invalid
383 && !has_certificate_expiring
384 {
385 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
386
387 let restart_futures = status
388 .replica_statuses
389 .iter()
390 .map(|rs| sts_api.restart(&rs.statefulset_name));
391
392 if has_single_replica {
393 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
394 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
395 return Err(Error::kube_error(
396 "restart",
397 "StatefulSet",
398 kanidm.get_namespace(),
399 "replica statefulsets",
400 first_err,
401 ));
402 }
403 } else {
404 let results: Vec<_> = join_all(restart_futures).await;
405 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
406 return Err(Error::kube_error(
407 "restart",
408 "StatefulSet",
409 kanidm.get_namespace(),
410 "replica statefulsets",
411 first_err,
412 ));
413 }
414 }
415 }
416 }
417
418 Ok(())
419}
420
421#[instrument(skip(ctx, kanidm))]
422pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
423 let trace_id = telemetry::get_trace_id();
424 Span::current().record("trace_id", field::display(&trace_id));
425 let _timer = ctx
426 .kaniop_ctx
427 .metrics
428 .reconcile_count_and_measure(&trace_id);
429 info!(msg = "reconciling Kanidm");
430
431 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
432 debug!(msg = "failed to reconcile status", %e);
433 ctx.kaniop_ctx.metrics.status_update_errors_inc();
434 e
435 })?;
436 let kanidm_api: Api<Kanidm> =
437 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
438 finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
439 match event {
440 Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
441 Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
442 }
443 })
444 .await
445 .map_err(|e| {
446 Error::FinalizerError(
447 "failed on kanidm account finalizer".to_string(),
448 Box::new(e),
449 )
450 })
451}
452
453async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
454 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
455 let replication_secret_futures =
456 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
457
458 let sts_to_delete = {
459 let expected_sts_names = kanidm
460 .spec
461 .replica_groups
462 .iter()
463 .map(|rg| kanidm.statefulset_name(&rg.name))
464 .collect::<Vec<_>>();
465 ctx.stores
466 .stateful_set_store
467 .state()
468 .into_iter()
469 .filter(|sts| {
470 sts.namespace() == kanidm.namespace()
471 && sts
472 .metadata
473 .labels
474 .as_ref()
475 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
476 && !expected_sts_names.contains(&sts.name_any())
477 })
478 .collect::<Vec<_>>()
479 };
480 let sts_delete_futures = sts_to_delete
481 .iter()
482 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
483 .collect::<TryJoinAll<_>>();
484
485 let sts_futures = match kanidm.is_updatable(&status) {
486 true => kanidm
487 .spec
488 .replica_groups
489 .iter()
490 .map(|rg| {
491 let sts = kanidm.create_statefulset(rg)?;
492 Ok(kanidm.patch(&ctx, sts))
493 })
494 .collect::<Result<TryJoinAll<_>, _>>()?,
495 false => {
496 let note = match status.version.as_ref() {
497 Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
498 format!(
499 "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
500 v.image_tag,
501 crate::version::KANIDM_CLIENT_VERSION
502 )
503 }
504 _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
505 };
506 let _ignore_error = ctx
507 .kaniop_ctx
508 .recorder
509 .publish(
510 &Event {
511 type_: EventType::Warning,
512 reason: "UpgradeBlocked".to_string(),
513 note: Some(note),
514 action: "ReconcileStatefulSet".to_string(),
515 secondary: None,
516 },
517 &kanidm.object_ref(&()),
518 )
519 .await
520 .map_err(|e| {
521 warn!(msg = "failed to publish KanidmError event", %e);
522 Error::KubeError("failed to publish event".to_string(), Box::new(e))
523 });
524 try_join_all([])
525 }
526 };
527 let service_future = kanidm.patch(&ctx, kanidm.create_service());
528
529 let deprecated_rg_svcs = {
530 let expected_rg_svcs_names = kanidm
531 .spec
532 .replica_groups
533 .iter()
534 .filter(|rg| rg.services.is_some())
535 .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
536 .collect::<Vec<_>>();
537 ctx.stores
538 .service_store
539 .state()
540 .into_iter()
541 .filter(|svc| {
542 let labels = svc.metadata.labels.as_ref();
543 svc.namespace() == kanidm.namespace()
544 && !expected_rg_svcs_names.contains(&svc.name_any())
545 && labels.is_some_and(|l| {
546 l.contains_key(REPLICA_LABEL)
547 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
548 })
549 })
550 .collect::<Vec<_>>()
551 };
552 let rg_svcs_delete_futures = deprecated_rg_svcs
553 .iter()
554 .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
555 .collect::<TryJoinAll<_>>();
556
557 let rg_services_futures = kanidm
558 .spec
559 .replica_groups
560 .iter()
561 .filter(|rg| rg.services.is_some())
562 .flat_map(|rg| {
563 (0..rg.replicas).map(|i| kanidm.patch(&ctx, kanidm.create_replica_group_service(rg, i)))
564 })
565 .collect::<TryJoinAll<_>>();
566
567 let deprecated_ingresses = {
568 let names = [
569 kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
570 kanidm.generate_region_ingress_name(),
571 ]
572 .into_iter()
573 .flatten()
574 .collect::<Vec<_>>();
575 ctx.stores
576 .ingress_store
577 .state()
578 .into_iter()
579 .filter(|ing| {
580 ing.namespace() == kanidm.namespace()
581 && !names.contains(&ing.name_any())
582 && ing.metadata.labels == Some(kanidm.generate_labels())
583 })
584 .collect::<Vec<_>>()
585 };
586 let ingress_delete_futures = deprecated_ingresses
587 .iter()
588 .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
589 .collect::<TryJoinAll<_>>();
590
591 let ingress_futures = kanidm
592 .create_ingress()
593 .into_iter()
594 .chain(kanidm.create_region_ingress())
595 .map(|ingress| kanidm.patch(&ctx, ingress))
596 .collect::<TryJoinAll<_>>();
597
598 let deprecated_http_routes = match &ctx.stores.http_route_store {
599 Some(store) => {
600 let expected_names = kanidm
601 .spec
602 .gateway
603 .as_ref()
604 .map(|_| vec![kanidm.name_any()])
605 .unwrap_or_default();
606 store
607 .state()
608 .into_iter()
609 .filter(|route| {
610 route.namespace() == kanidm.namespace()
611 && !expected_names.contains(&route.name_any())
612 && route.metadata.labels == Some(kanidm.generate_labels())
613 })
614 .collect::<Vec<_>>()
615 }
616 None => Vec::new(),
617 };
618 let http_route_delete_futures = deprecated_http_routes
619 .iter()
620 .map(|route| kanidm.delete(&ctx, route.as_ref()))
621 .collect::<TryJoinAll<_>>();
622
623 let http_route_futures = if ctx.stores.http_route_store.is_some() {
624 kanidm
625 .create_http_route()
626 .into_iter()
627 .map(|route| kanidm.patch(&ctx, route))
628 .collect::<TryJoinAll<_>>()
629 } else {
630 try_join_all(Vec::new())
631 };
632
633 try_join!(
634 sts_delete_futures,
635 admin_secret_future,
636 replication_secret_futures,
637 sts_futures,
638 service_future,
639 rg_svcs_delete_futures,
640 rg_services_futures,
641 ingress_delete_futures,
642 ingress_futures,
643 http_route_delete_futures,
644 http_route_futures
645 )?;
646
647 if is_kanidm_available(status.clone()) {
648 let namespace = kanidm.namespace().unwrap();
649 let name = kanidm.name_any();
650 let system_client = crate::controller::kanidm::KanidmClients::create_client(
651 &namespace,
652 &name,
653 crate::controller::kanidm::KanidmUser::Admin,
654 ctx.kaniop_ctx.client.clone(),
655 )
656 .await?;
657 reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
658
659 let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
660 &namespace,
661 &name,
662 crate::controller::kanidm::KanidmUser::IdmAdmin,
663 ctx.kaniop_ctx.client.clone(),
664 )
665 .await?;
666 let mail_sender_status =
667 reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
668 let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
669 if mail_sender_status != current_mail_sender_status {
670 let kanidm_api: Api<Kanidm> =
671 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
672 let status_patch = serde_json::json!({
673 "status": {
674 "mailSender": mail_sender_status
675 }
676 });
677 kanidm_api
678 .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
679 .await
680 .map_err(|e| {
681 Error::KubeError(
682 format!("failed to patch Kanidm/status {namespace}/{name}"),
683 Box::new(e),
684 )
685 })?;
686 }
687 }
688
689 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
690}
691
692async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
693 debug!(msg = "cleanup");
694
695 cleanup_mail_sender_resources(&kanidm, &ctx).await?;
696
697 let namespace = kanidm.namespace().unwrap();
698 let name = kanidm.name_any();
699
700 if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
701 &namespace,
702 &name,
703 crate::controller::kanidm::KanidmUser::IdmAdmin,
704 ctx.kaniop_ctx.client.clone(),
705 )
706 .await
707 {
708 cleanup_mail_sender_in_kanidm(&kanidm_client, &name).await?;
709 }
710
711 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
712 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
713}
714
715async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
716 let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
717 let start = std::time::Instant::now();
718 let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
719 let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
720
721 loop {
722 let sts = sts_api.get(sts_name).await.map_err(|e| {
723 Error::KubeError(
724 format!("failed to get StatefulSet {namespace}/{sts_name}"),
725 Box::new(e),
726 )
727 })?;
728
729 if sts.metadata.deletion_timestamp.is_some() {
730 return Err(Error::ReceiveOutput(format!(
731 "StatefulSet {namespace}/{sts_name} is being deleted"
732 )));
733 }
734
735 let spec = sts.spec.as_ref().ok_or_else(|| {
736 Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
737 })?;
738
739 let status = match sts.status {
740 Some(ref s) => s,
741 None => {
742 if start.elapsed() >= timeout {
743 return Err(Error::ReceiveOutput(format!(
744 "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
745 )));
746 }
747 sleep(poll_interval).await;
748 continue;
749 }
750 };
751
752 let replicas = spec.replicas.unwrap_or(0);
753
754 if replicas == 0 {
755 return Ok(());
756 }
757
758 let updated_replicas = status.updated_replicas.unwrap_or(0);
759 let current_revision = status.current_revision.as_deref().unwrap_or("");
760 let update_revision = status.update_revision.as_deref().unwrap_or("");
761 let ready_replicas = status.ready_replicas.unwrap_or(0);
762
763 if updated_replicas == replicas
764 && current_revision == update_revision
765 && ready_replicas == replicas
766 {
767 return Ok(());
768 }
769
770 if start.elapsed() >= timeout {
771 return Err(Error::ReceiveOutput(format!(
772 "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
773 )));
774 }
775
776 sleep(poll_interval).await;
777 }
778}
779
780async fn show_replica_cert_with_retries(
781 kanidm: &Kanidm,
782 ctx: Arc<Context>,
783 pod_name: &str,
784 max_attempts: u32,
785) -> std::result::Result<String, Error> {
786 let mut last_error = None;
787 for attempt in 0..max_attempts {
788 if attempt > 0 {
789 sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
790 }
791 match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
792 Ok(cert) => return Ok(cert),
793 Err(e) => {
794 info!(
795 msg = format!(
796 "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
797 attempt + 1,
798 max_attempts
799 )
800 );
801 last_error = Some(e);
802 }
803 }
804 }
805 Err(last_error.unwrap_or_else(|| {
806 Error::ReceiveOutput(format!(
807 "failed to get certificate for {pod_name} after {max_attempts} attempts"
808 ))
809 }))
810}
811
812impl Kanidm {
813 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
815 where
816 K: Resource<Scope = NamespaceResourceScope>
817 + Serialize
818 + Clone
819 + std::fmt::Debug
820 + for<'de> Deserialize<'de>,
821 <K as kube::Resource>::DynamicType: Default,
822 <K as Resource>::Scope: std::marker::Sized,
823 {
824 self.kube_delete(
825 ctx.kaniop_ctx.client.clone(),
826 &ctx.kaniop_ctx.metrics,
827 resource,
828 )
829 .await
830 }
831
832 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
833 where
834 K: Resource<Scope = NamespaceResourceScope>
835 + Serialize
836 + Clone
837 + std::fmt::Debug
838 + for<'de> Deserialize<'de>,
839 <K as kube::Resource>::DynamicType: Default,
840 <K as Resource>::Scope: std::marker::Sized,
841 {
842 self.kube_patch(
843 ctx.kaniop_ctx.client.clone(),
844 &ctx.kaniop_ctx.metrics,
845 resource,
846 KANIDM_OPERATOR_NAME,
847 )
848 .await
849 }
850
851 #[inline]
852 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
853 LABELS
854 .clone()
855 .into_iter()
856 .chain([
857 (INSTANCE_LABEL.to_string(), self.name_any()),
858 (CLUSTER_LABEL.to_string(), self.name_any()),
859 ])
860 .collect()
861 }
862
863 #[inline]
864 fn generate_labels(&self) -> BTreeMap<String, String> {
865 self.generate_resource_labels()
866 }
867
868 #[inline]
869 fn get_tls_secret_name(&self) -> String {
870 format!("{}-tls", self.name_any())
871 }
872
873 #[inline]
874 fn get_namespace(&self) -> String {
875 self.namespace().unwrap()
877 }
878
879 #[inline]
880 fn is_replication_enabled(&self) -> bool {
881 self.spec.replica_groups.len() > 1
882 || self
883 .spec
884 .replica_groups
885 .first()
886 .is_some_and(|rg| rg.replicas > 1)
887 || !self.spec.external_replication_nodes.is_empty()
888 }
889
890 #[inline]
891 fn is_updatable(&self, status: &KanidmStatus) -> bool {
892 status
893 .version
894 .as_ref()
895 .map(|v| {
896 if v.compatibility_result == VersionCompatibilityResult::Incompatible {
897 return false;
898 }
899 let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
900 let versions = (parse_semver(¤t_tag), parse_semver(&v.image_tag));
901 match v.upgrade_check_result {
902 KanidmUpgradeCheckResult::Passed => {
903 if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
904 minor <= v_minor + 1
905 } else {
906 true
907 }
908 }
909 KanidmUpgradeCheckResult::Failed => {
910 if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
911 versions
912 {
913 major == v_major && minor == v_minor && patch >= v_patch
914 } else {
915 v.image_tag == current_tag
916 }
917 }
918 }
919 })
920 .unwrap_or(true)
921 }
922
923 async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
924 let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
925 let pod = pod_api.get(pod_name).await.map_err(|e| {
926 Error::KubeError(
927 format!("failed to get pod {namespace}/{pod_name}"),
928 Box::new(e),
929 )
930 })?;
931 Ok(pod
932 .status
933 .as_ref()
934 .and_then(|s| s.conditions.as_ref())
935 .is_some_and(|conditions| {
936 conditions
937 .iter()
938 .any(|c| c.type_ == "Ready" && c.status == "True")
939 }))
940 }
941
942 async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
943 let start = std::time::Instant::now();
944 let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
945 let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
946
947 loop {
948 match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
949 Ok(true) => return Ok(()),
950 Ok(false) => {}
951 Err(e) => {
952 if !e.is_retryable() {
953 return Err(e);
954 }
955 trace!(msg = "pod readiness check failed, retrying", %e);
956 }
957 }
958
959 if start.elapsed() >= timeout {
960 return Err(Error::ReceiveOutput(format!(
961 "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
962 )));
963 }
964
965 sleep(poll_interval).await;
966 }
967 }
968
969 async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
970 where
971 I: IntoIterator<Item = T> + Debug,
972 T: Into<String>,
973 {
974 let namespace = &self.get_namespace();
975 trace!(
976 msg = "pod exec",
977 resource.name = &pod_name,
978 resource.namespace = &namespace,
979 ?command
980 );
981 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
982 let attached = pod
983 .exec(
984 pod_name,
985 command,
986 &AttachParams::default().container("kanidm"),
987 )
988 .await
989 .map_err(|e| {
990 Error::KubeError(
991 format!("failed to exec pod {namespace}/{pod_name}"),
992 Box::new(e),
993 )
994 })?;
995 get_output(attached).await
996 }
997
998 async fn exec_with_wait<I, T>(
999 &self,
1000 ctx: Arc<Context>,
1001 pod_name: &str,
1002 command: I,
1003 ) -> Result<String>
1004 where
1005 I: IntoIterator<Item = T> + Debug,
1006 T: Into<String>,
1007 {
1008 let namespace = &self.get_namespace();
1009 Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1010 self.exec(ctx, pod_name, command).await
1011 }
1012
1013 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1014 where
1015 I: IntoIterator<Item = T> + Debug,
1016 T: Into<String>,
1017 {
1018 let first_replica_group = self
1020 .spec
1021 .replica_groups
1022 .first()
1023 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1024 let sts_name = self.statefulset_name(&first_replica_group.name);
1025 let pod_name = format!("{sts_name}-0");
1026 self.exec(ctx, &pod_name, command).await
1027 }
1028
1029 async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1030 where
1031 I: IntoIterator<Item = T> + Debug,
1032 T: Into<String>,
1033 {
1034 let first_replica_group = self
1035 .spec
1036 .replica_groups
1037 .first()
1038 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1039 let sts_name = self.statefulset_name(&first_replica_group.name);
1040 let pod_name = format!("{sts_name}-0");
1041 self.exec_with_wait(ctx, &pod_name, command).await
1042 }
1043}
1044
1045#[cfg(test)]
1046mod test {
1047 use super::statefulset::StatefulSetExt;
1048 use super::{Kanidm, reconcile_kanidm};
1049
1050 use crate::controller::State;
1051 use crate::kanidm::controller::context::{Context, Stores};
1052 use crate::kanidm::crd::KanidmStatus;
1053
1054 use kaniop_k8s_util::error::Result;
1055
1056 use std::sync::Arc;
1057
1058 use http::{Request, Response};
1059 use k8s_openapi::api::apps::v1::StatefulSet;
1060 use k8s_openapi::api::core::v1::Service;
1061 use k8s_openapi::api::networking::v1::Ingress;
1062 use kube::runtime::reflector::store::Writer;
1063 use kube::{Client, Resource, ResourceExt, client::Body};
1064 use opentelemetry::metrics::MeterProvider;
1065 use serde_json::json;
1066
1067 impl Kanidm {
1068 pub fn test() -> Self {
1070 let mut e = Kanidm::new(
1071 "test",
1072 serde_json::from_value(json!({
1073 "domain": "idm.example.com",
1074 "replicaGroups": [
1075 {
1076 "name": "default",
1077 "replicas": 1
1078 }
1079 ]
1080 }))
1081 .unwrap(),
1082 );
1083 e.meta_mut().namespace = Some("default".into());
1084 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1087 e
1088 }
1089
1090 pub fn with_ingress(mut self) -> Self {
1091 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1092 self
1093 }
1094
1095 pub fn with_replicas(mut self, replicas: i32) -> Self {
1097 self.spec.replica_groups[0].replicas = replicas;
1098 self
1099 }
1100
1101 pub fn needs_delete(mut self) -> Self {
1103 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1104 use k8s_openapi::jiff::Timestamp;
1105 let now = Timestamp::from_second(1491138632).unwrap(); self.meta_mut().deletion_timestamp = Some(Time(now));
1107 self
1108 }
1109
1110 pub fn with_status(mut self, status: KanidmStatus) -> Self {
1112 self.status = Some(status);
1113 self
1114 }
1115 }
1116
1117 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1119 pub struct ApiServerVerifier(ApiServerHandle);
1120
1121 pub enum Scenario {
1123 Create(Kanidm),
1124 CreateWithTwoReplicas(Kanidm),
1125 CreateWithIngress(Kanidm),
1126 CreateWithIngressWithTwoReplicas(Kanidm),
1127 }
1128
1129 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1130 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1131 .await
1132 .expect("timeout on mock apiserver")
1133 .expect("scenario succeeded")
1134 }
1135
1136 impl ApiServerVerifier {
1137 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1148 tokio::spawn(async move {
1149 match scenario {
1151 Scenario::Create(kanidm) => {
1152 self.handle_kanidm_get(kanidm.clone())
1153 .await
1154 .unwrap()
1155 .handle_kanidm_status_patch(kanidm.clone())
1156 .await
1157 .unwrap()
1158 .handle_statefulset_patch(kanidm.clone())
1159 .await
1160 .unwrap()
1161 .handle_service_patch(kanidm.clone())
1162 .await
1163 }
1164 Scenario::CreateWithTwoReplicas(kanidm) => {
1165 self.handle_kanidm_get(kanidm.clone())
1166 .await
1167 .unwrap()
1168 .handle_kanidm_status_patch(kanidm.clone())
1169 .await
1170 .unwrap()
1171 .handle_statefulset_patch(kanidm.clone())
1172 .await
1173 .unwrap()
1174 .handle_service_patch(kanidm.clone())
1175 .await
1176 }
1177 Scenario::CreateWithIngress(kanidm) => {
1178 self.handle_kanidm_get(kanidm.clone())
1179 .await
1180 .unwrap()
1181 .handle_kanidm_status_patch(kanidm.clone())
1182 .await
1183 .unwrap()
1184 .handle_statefulset_patch(kanidm.clone())
1185 .await
1186 .unwrap()
1187 .handle_service_patch(kanidm.clone())
1188 .await
1189 .unwrap()
1190 .handle_ingress_patch(kanidm.clone())
1191 .await
1192 }
1193 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1194 self.handle_kanidm_get(kanidm.clone())
1195 .await
1196 .unwrap()
1197 .handle_kanidm_status_patch(kanidm.clone())
1198 .await
1199 .unwrap()
1200 .handle_statefulset_patch(kanidm.clone())
1201 .await
1202 .unwrap()
1203 .handle_service_patch(kanidm.clone())
1204 .await
1205 .unwrap()
1206 .handle_ingress_patch(kanidm.clone())
1207 .await
1208 }
1209 }
1210 .expect("scenario completed without errors");
1211 })
1212 }
1213
1214 async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1215 let (request, send) = self.0.next_request().await.expect("service not called");
1216 assert_eq!(request.method(), http::Method::GET);
1217 let expected_uri_prefix = format!(
1218 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1219 kanidm.name_any()
1220 );
1221 let uri = request.uri().to_string();
1222 assert!(
1223 uri.starts_with(&expected_uri_prefix),
1224 "expected uri to start with {}, got {}",
1225 expected_uri_prefix,
1226 uri
1227 );
1228 let response = serde_json::to_vec(&kanidm).unwrap();
1229 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1230 Ok(self)
1231 }
1232
1233 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1234 let (request, send) = self.0.next_request().await.expect("service not called");
1235 assert_eq!(request.method(), http::Method::PATCH);
1236 assert_eq!(
1237 request.uri().to_string(),
1238 format!(
1239 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1240 kanidm.name_any()
1241 )
1242 );
1243
1244 let req_body = request.into_body().collect_bytes().await.unwrap();
1245 let json: serde_json::Value =
1246 serde_json::from_slice(&req_body).expect("patch object is json");
1247 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1248 .expect("valid kanidm status");
1249 let mut kanidm = Kanidm::test();
1250 kanidm.status = Some(status.clone());
1251 let response = serde_json::to_vec(&kanidm).unwrap();
1252 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1254 Ok(self)
1255 }
1256
1257 async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1258 for rg in kanidm.spec.replica_groups.iter() {
1259 let (request, send) = self.0.next_request().await.expect("service not called");
1260 assert_eq!(request.method(), http::Method::PATCH);
1261 assert_eq!(
1262 request.uri().to_string(),
1263 format!(
1264 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1265 kanidm.statefulset_name(&rg.name)
1266 )
1267 );
1268 let req_body = request.into_body().collect_bytes().await.unwrap();
1269 let json: serde_json::Value =
1270 serde_json::from_slice(&req_body).expect("patch object is json");
1271 let statefulset: StatefulSet =
1272 serde_json::from_value(json).expect("valid statefulset");
1273 assert_eq!(
1274 statefulset.clone().spec.unwrap().replicas.unwrap(),
1275 rg.replicas
1276 );
1277 let response = serde_json::to_vec(&statefulset).unwrap();
1278 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1280 }
1281
1282 Ok(self)
1283 }
1284
1285 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1286 let (request, send) = self.0.next_request().await.expect("service not called");
1287 assert_eq!(request.method(), http::Method::PATCH);
1288 assert_eq!(
1289 request.uri().to_string(),
1290 format!(
1291 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1292 kanidm.name_any()
1293 )
1294 );
1295
1296 let req_body = request.into_body().collect_bytes().await.unwrap();
1297 let json: serde_json::Value =
1298 serde_json::from_slice(&req_body).expect("patch object is json");
1299 let service: Service = serde_json::from_value(json).expect("valid service");
1300 let response = serde_json::to_vec(&service).unwrap();
1301 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1303 Ok(self)
1304 }
1305
1306 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1307 let (request, send) = self.0.next_request().await.expect("service not called");
1308 assert_eq!(request.method(), http::Method::PATCH);
1309 assert_eq!(
1310 request.uri().to_string(),
1311 format!(
1312 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1313 kanidm.name_any()
1314 )
1315 );
1316
1317 let req_body = request.into_body().collect_bytes().await.unwrap();
1318 let json: serde_json::Value =
1319 serde_json::from_slice(&req_body).expect("patch object is json");
1320 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1321 let response = serde_json::to_vec(&ingress).unwrap();
1322 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1324 Ok(self)
1325 }
1326 }
1327
1328 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1329 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1330 let mock_client = Client::new(mock_service, "default");
1331 let stores = Stores {
1332 stateful_set_store: Writer::default().as_reader(),
1333 service_store: Writer::default().as_reader(),
1334 ingress_store: Writer::default().as_reader(),
1335 secret_store: Writer::default().as_reader(),
1336 http_route_store: Some(Writer::default().as_reader()),
1337 deployment_store: Writer::default().as_reader(),
1338 config_map_store: Writer::default().as_reader(),
1339 };
1340 let controller_id = "test";
1341
1342 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
1344 let meter = provider.meter("test");
1345 let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
1346
1347 let state = State::new(
1348 metrics,
1349 Writer::default().as_reader(),
1350 Writer::default().as_reader(),
1351 None,
1352 );
1353 let ctx = Arc::new(Context::new(
1354 state.to_context(mock_client, controller_id),
1355 stores,
1356 ));
1357 (ctx, ApiServerVerifier(handle))
1358 }
1359
1360 #[tokio::test]
1361 async fn kanidm_create() {
1362 let (testctx, fakeserver) = get_test_context();
1363 let kanidm = Kanidm::test();
1364 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
1365 reconcile_kanidm(Arc::new(kanidm), testctx)
1366 .await
1367 .expect("reconciler");
1368 timeout_after_1s(mocksrv).await;
1369 }
1370
1371 #[tokio::test]
1372 async fn kanidm_create_with_two_replicas() {
1373 let (testctx, fakeserver) = get_test_context();
1374 let kanidm = Kanidm::test().with_replicas(2);
1375 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
1376 reconcile_kanidm(Arc::new(kanidm), testctx)
1377 .await
1378 .expect("reconciler");
1379 timeout_after_1s(mocksrv).await;
1380 }
1381
1382 #[tokio::test]
1383 async fn kanidm_create_with_ingress() {
1384 let (testctx, fakeserver) = get_test_context();
1385 let kanidm = Kanidm::test().with_ingress();
1386 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
1387 reconcile_kanidm(Arc::new(kanidm), testctx)
1388 .await
1389 .expect("reconciler");
1390 timeout_after_1s(mocksrv).await;
1391 }
1392
1393 #[tokio::test]
1394 async fn kanidm_create_with_ingress_with_two_replicas() {
1395 let (testctx, fakeserver) = get_test_context();
1396 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
1397 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
1398 reconcile_kanidm(Arc::new(kanidm), testctx)
1399 .await
1400 .expect("reconciler");
1401 timeout_after_1s(mocksrv).await;
1402 }
1403}