1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12 cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::StatefulSetExt;
21use self::status::StatusExt;
22use self::status::{is_kanidm_available, is_kanidm_initialized};
23
24use crate::controller::context::KubeOperations;
25use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
26use crate::kanidm::crd::{
27 Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
28};
29use crate::kanidm::reconcile::statefulset::REPLICA_LABEL;
30use crate::telemetry;
31
32use kaniop_k8s_util::client::get_output;
33use kaniop_k8s_util::error::{Error, Result};
34use kaniop_k8s_util::parse::parse_semver;
35use kaniop_k8s_util::resources::get_image_tag;
36
37use std::collections::{BTreeMap, BTreeSet};
38use std::fmt::Debug;
39use std::sync::{Arc, LazyLock};
40
41use futures::future::{TryJoinAll, join_all, try_join_all};
42use futures::stream::{self, StreamExt};
43use futures::try_join;
44use k8s_openapi::NamespaceResourceScope;
45use k8s_openapi::api::apps::v1::StatefulSet;
46use k8s_openapi::api::core::v1::Pod;
47use kube::Resource;
48use kube::ResourceExt;
49use kube::api::{Api, AttachParams, Patch, PatchParams};
50use kube::runtime::controller::Action;
51use kube::runtime::events::{Event, EventType};
52use kube::runtime::finalizer::{Event as Finalizer, finalizer};
53use serde::{Deserialize, Serialize};
54use tokio::time::{Duration, sleep};
55use tracing::{Span, debug, field, info, instrument, trace, warn};
56
57const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
58const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
59const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
60const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
61const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
62const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
63const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
64
65pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
66const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
67pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
68
69const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
70
71static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
72 BTreeMap::from([
73 (NAME_LABEL.to_string(), "kanidm".to_string()),
74 (
75 MANAGED_BY_LABEL.to_string(),
76 format!("kaniop-{CONTROLLER_ID}"),
77 ),
78 ])
79});
80
81async fn restart_statefulsets_after_replica_secret_update(
82 ctx: Arc<Context>,
83 sts_api: &Api<StatefulSet>,
84 namespace: &str,
85 statefulset_names: &[String],
86 sequential: bool,
87) -> Result<()> {
88 let restart_futures = statefulset_names
91 .iter()
92 .map(|sts_name| sts_api.restart(sts_name));
93
94 if sequential {
95 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
96 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
97 return Err(Error::kube_error(
98 "restart",
99 "StatefulSet",
100 namespace,
101 "replica certificate secret update statefulsets",
102 first_err,
103 ));
104 }
105 } else {
106 let results: Vec<_> = join_all(restart_futures).await;
107 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
108 return Err(Error::kube_error(
109 "restart",
110 "StatefulSet",
111 namespace,
112 "replica certificate secret update statefulsets",
113 first_err,
114 ));
115 }
116 }
117
118 let rollout_futures = statefulset_names
119 .iter()
120 .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
121 try_join_all(rollout_futures).await?;
122
123 Ok(())
124}
125
126pub async fn reconcile_admins_secret(
127 kanidm: Arc<Kanidm>,
128 ctx: Arc<Context>,
129 status: &KanidmStatus,
130) -> Result<()> {
131 if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
132 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
133 kanidm.patch(&ctx, admins_secret).await?;
134 }
135 Ok(())
136}
137
138pub async fn reconcile_replication_secrets(
139 kanidm: Arc<Kanidm>,
140 ctx: Arc<Context>,
141 status: &KanidmStatus,
142) -> Result<()> {
143 let expected_secret_names = if kanidm.is_replication_enabled() {
144 status
145 .replica_statuses
146 .iter()
147 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
148 .collect::<Vec<_>>()
149 } else {
150 Vec::new()
151 };
152 let deprecated_secrets = ctx
153 .stores
154 .secret_store
155 .state()
156 .into_iter()
157 .filter(|secret| {
158 secret.namespace() == kanidm.namespace()
159 && kanidm.admins_secret_name() != secret.name_any()
160 && !expected_secret_names.contains(&secret.name_any())
161 && secret
162 .metadata
163 .labels
164 .as_ref()
165 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
166 })
167 .collect::<Vec<_>>();
168 let secret_delete_future = deprecated_secrets
169 .iter()
170 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
171 .collect::<TryJoinAll<_>>();
172 try_join!(secret_delete_future)?;
173
174 if kanidm.is_replication_enabled() {
175 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
176 let statefulset_names = status
177 .replica_statuses
178 .iter()
179 .map(|rs| rs.statefulset_name.clone())
180 .collect::<BTreeSet<_>>()
181 .into_iter()
182 .collect::<Vec<_>>();
183
184 let has_pending = status
185 .replica_statuses
186 .iter()
187 .any(|rs| rs.state == KanidmReplicaState::Pending);
188
189 let sts_api =
190 Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
191
192 if has_pending {
193 stream::iter(
194 status
195 .replica_statuses
196 .iter()
197 .filter(|rs| rs.state == KanidmReplicaState::Pending),
198 )
199 .then(|rs| async {
200 let secret = kanidm
201 .generate_replica_secret(ctx.clone(), &rs.pod_name)
202 .await?;
203 kanidm.patch(&ctx, secret).await
204 })
205 .collect::<Vec<_>>()
206 .await
207 .into_iter()
208 .collect::<Result<Vec<_>>>()?;
209
210 let restart_futures = status
211 .replica_statuses
212 .iter()
213 .map(|rs| sts_api.restart(&rs.statefulset_name));
214
215 if has_single_replica {
216 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
217 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
218 return Err(Error::kube_error(
219 "restart",
220 "StatefulSet",
221 kanidm.get_namespace(),
222 "pending replica statefulsets",
223 first_err,
224 ));
225 }
226 } else {
227 let results: Vec<_> = join_all(restart_futures).await;
228 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
229 return Err(Error::kube_error(
230 "restart",
231 "StatefulSet",
232 kanidm.get_namespace(),
233 "pending replica statefulsets",
234 first_err,
235 ));
236 }
237 }
238 }
239
240 let has_certificate_host_invalid = status
241 .replica_statuses
242 .iter()
243 .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
244
245 let has_certificate_expiring = status
246 .replica_statuses
247 .iter()
248 .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
249
250 let has_non_ready_replicas = status
251 .replica_statuses
252 .iter()
253 .any(|rs| rs.state != KanidmReplicaState::Ready);
254
255 let host_invalid_replicas: Vec<_> = status
256 .replica_statuses
257 .iter()
258 .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
259 .collect();
260
261 let cert_expiring_replicas: Vec<_> = status
262 .replica_statuses
263 .iter()
264 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
265 .collect();
266
267 if !host_invalid_replicas.is_empty() {
268 let restart_futures = host_invalid_replicas
269 .iter()
270 .map(|rs| sts_api.restart(&rs.statefulset_name));
271
272 if has_single_replica {
273 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
274 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
275 return Err(Error::kube_error(
276 "restart",
277 "StatefulSet",
278 kanidm.get_namespace(),
279 "certificate host invalid statefulsets",
280 first_err,
281 ));
282 }
283 } else {
284 let results: Vec<_> = join_all(restart_futures).await;
285 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
286 return Err(Error::kube_error(
287 "restart",
288 "StatefulSet",
289 kanidm.get_namespace(),
290 "certificate host invalid statefulsets",
291 first_err,
292 ));
293 }
294 }
295
296 let restarted_sts_names: Vec<_> = host_invalid_replicas
297 .iter()
298 .map(|rs| rs.statefulset_name.clone())
299 .collect::<BTreeSet<_>>()
300 .into_iter()
301 .collect();
302
303 let namespace = kanidm.get_namespace();
304 let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
305 wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
306 });
307 try_join_all(rollout_futures).await?;
308
309 let renew_futures = host_invalid_replicas
310 .iter()
311 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
312 let results: Vec<_> = join_all(renew_futures).await;
313 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
314 return Err(first_err);
315 }
316
317 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
318
319 let cert_futures = host_invalid_replicas.iter().map(|rs| async {
320 let cert = show_replica_cert_with_retries(
321 &kanidm,
322 ctx.clone(),
323 &rs.pod_name,
324 CERT_SHOW_RETRY_ATTEMPTS,
325 )
326 .await?;
327 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
328 kanidm.patch(&ctx, secret.clone()).await
329 });
330 try_join_all(cert_futures).await?;
331
332 restart_statefulsets_after_replica_secret_update(
333 ctx.clone(),
334 &sts_api,
335 &kanidm.get_namespace(),
336 &statefulset_names,
337 has_single_replica,
338 )
339 .await?;
340
341 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
342 }
343
344 if !cert_expiring_replicas.is_empty() {
345 let renew_futures = cert_expiring_replicas
346 .iter()
347 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
348 let results: Vec<_> = join_all(renew_futures).await;
349 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
350 return Err(first_err);
351 }
352
353 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
354
355 let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
356 let cert = show_replica_cert_with_retries(
357 &kanidm,
358 ctx.clone(),
359 &rs.pod_name,
360 CERT_SHOW_RETRY_ATTEMPTS,
361 )
362 .await?;
363 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
364 kanidm.patch(&ctx, secret.clone()).await
365 });
366 try_join_all(cert_futures).await?;
367
368 restart_statefulsets_after_replica_secret_update(
369 ctx.clone(),
370 &sts_api,
371 &kanidm.get_namespace(),
372 &statefulset_names,
373 has_single_replica,
374 )
375 .await?;
376
377 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
378 }
379
380 if has_non_ready_replicas
381 && !has_pending
382 && !has_certificate_host_invalid
383 && !has_certificate_expiring
384 {
385 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
386
387 let restart_futures = status
388 .replica_statuses
389 .iter()
390 .map(|rs| sts_api.restart(&rs.statefulset_name));
391
392 if has_single_replica {
393 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
394 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
395 return Err(Error::kube_error(
396 "restart",
397 "StatefulSet",
398 kanidm.get_namespace(),
399 "replica statefulsets",
400 first_err,
401 ));
402 }
403 } else {
404 let results: Vec<_> = join_all(restart_futures).await;
405 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
406 return Err(Error::kube_error(
407 "restart",
408 "StatefulSet",
409 kanidm.get_namespace(),
410 "replica statefulsets",
411 first_err,
412 ));
413 }
414 }
415 }
416 }
417
418 Ok(())
419}
420
421#[instrument(skip(ctx, kanidm))]
422pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
423 let trace_id = telemetry::get_trace_id();
424 Span::current().record("trace_id", field::display(&trace_id));
425 let _timer = ctx
426 .kaniop_ctx
427 .metrics
428 .reconcile_count_and_measure(&trace_id);
429 info!(msg = "reconciling Kanidm");
430
431 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
432 debug!(msg = "failed to reconcile status", %e);
433 ctx.kaniop_ctx.metrics.status_update_errors_inc();
434 e
435 })?;
436 let kanidm_api: Api<Kanidm> =
437 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
438 finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
439 match event {
440 Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
441 Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
442 }
443 })
444 .await
445 .map_err(|e| {
446 Error::FinalizerError(
447 "failed on kanidm account finalizer".to_string(),
448 Box::new(e),
449 )
450 })
451}
452
453async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
454 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
455 let replication_secret_futures =
456 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
457
458 let sts_to_delete = {
459 let expected_sts_names = kanidm
460 .spec
461 .replica_groups
462 .iter()
463 .map(|rg| kanidm.statefulset_name(&rg.name))
464 .collect::<Vec<_>>();
465 ctx.stores
466 .stateful_set_store
467 .state()
468 .into_iter()
469 .filter(|sts| {
470 sts.namespace() == kanidm.namespace()
471 && sts
472 .metadata
473 .labels
474 .as_ref()
475 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
476 && !expected_sts_names.contains(&sts.name_any())
477 })
478 .collect::<Vec<_>>()
479 };
480 let sts_delete_futures = sts_to_delete
481 .iter()
482 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
483 .collect::<TryJoinAll<_>>();
484
485 let sts_futures = match kanidm.is_updatable(&status) {
486 true => kanidm
487 .spec
488 .replica_groups
489 .iter()
490 .map(|rg| {
491 let sts = kanidm.create_statefulset(rg)?;
492 Ok(kanidm.patch(&ctx, sts))
493 })
494 .collect::<Result<TryJoinAll<_>, _>>()?,
495 false => {
496 let note = match status.version.as_ref() {
497 Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
498 format!(
499 "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
500 v.image_tag,
501 crate::version::KANIDM_CLIENT_VERSION
502 )
503 }
504 _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
505 };
506 let _ignore_error = ctx
507 .kaniop_ctx
508 .recorder
509 .publish(
510 &Event {
511 type_: EventType::Warning,
512 reason: "UpgradeBlocked".to_string(),
513 note: Some(note),
514 action: "ReconcileStatefulSet".to_string(),
515 secondary: None,
516 },
517 &kanidm.object_ref(&()),
518 )
519 .await
520 .map_err(|e| {
521 warn!(msg = "failed to publish KanidmError event", %e);
522 Error::KubeError("failed to publish event".to_string(), Box::new(e))
523 });
524 try_join_all([])
525 }
526 };
527 let service_future = kanidm.patch(&ctx, kanidm.create_service());
528
529 let deprecated_rg_svcs = {
530 let expected_rg_svcs_names = kanidm
531 .spec
532 .replica_groups
533 .iter()
534 .filter(|rg| rg.services.is_some())
535 .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
536 .collect::<Vec<_>>();
537 ctx.stores
538 .service_store
539 .state()
540 .into_iter()
541 .filter(|svc| {
542 let labels = svc.metadata.labels.as_ref();
543 svc.namespace() == kanidm.namespace()
544 && !expected_rg_svcs_names.contains(&svc.name_any())
545 && labels.is_some_and(|l| {
546 l.contains_key(REPLICA_LABEL)
547 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
548 })
549 })
550 .collect::<Vec<_>>()
551 };
552 let rg_svcs_delete_futures = deprecated_rg_svcs
553 .iter()
554 .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
555 .collect::<TryJoinAll<_>>();
556
557 let rg_services_futures = kanidm
558 .spec
559 .replica_groups
560 .iter()
561 .filter(|rg| rg.services.is_some())
562 .flat_map(|rg| {
563 (0..rg.replicas).map(|i| kanidm.patch(&ctx, kanidm.create_replica_group_service(rg, i)))
564 })
565 .collect::<TryJoinAll<_>>();
566
567 let deprecated_ingresses = {
568 let names = [
569 kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
570 kanidm.generate_region_ingress_name(),
571 ]
572 .into_iter()
573 .flatten()
574 .collect::<Vec<_>>();
575 ctx.stores
576 .ingress_store
577 .state()
578 .into_iter()
579 .filter(|ing| {
580 ing.namespace() == kanidm.namespace()
581 && !names.contains(&ing.name_any())
582 && ing.metadata.labels == Some(kanidm.generate_labels())
583 })
584 .collect::<Vec<_>>()
585 };
586 let ingress_delete_futures = deprecated_ingresses
587 .iter()
588 .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
589 .collect::<TryJoinAll<_>>();
590
591 let ingress_futures = kanidm
592 .create_ingress()
593 .into_iter()
594 .chain(kanidm.create_region_ingress())
595 .map(|ingress| kanidm.patch(&ctx, ingress))
596 .collect::<TryJoinAll<_>>();
597
598 let deprecated_http_routes = match &ctx.stores.http_route_store {
599 Some(store) => {
600 let expected_names = kanidm
601 .spec
602 .gateway
603 .as_ref()
604 .map(|_| vec![kanidm.name_any()])
605 .unwrap_or_default();
606 store
607 .state()
608 .into_iter()
609 .filter(|route| {
610 route.namespace() == kanidm.namespace()
611 && !expected_names.contains(&route.name_any())
612 && route.metadata.labels == Some(kanidm.generate_labels())
613 })
614 .collect::<Vec<_>>()
615 }
616 None => Vec::new(),
617 };
618 let http_route_delete_futures = deprecated_http_routes
619 .iter()
620 .map(|route| kanidm.delete(&ctx, route.as_ref()))
621 .collect::<TryJoinAll<_>>();
622
623 let http_route_futures = if ctx.stores.http_route_store.is_some() {
624 kanidm
625 .create_http_route()
626 .into_iter()
627 .map(|route| kanidm.patch(&ctx, route))
628 .collect::<TryJoinAll<_>>()
629 } else {
630 try_join_all(Vec::new())
631 };
632
633 let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
634 Some(store) => {
635 let expected_names = kanidm
636 .spec
637 .gateway
638 .as_ref()
639 .and_then(|g| g.backend_tls_policy.as_ref())
640 .map(|_| vec![kanidm.name_any()])
641 .unwrap_or_default();
642 store
643 .state()
644 .into_iter()
645 .filter(|policy| {
646 policy.namespace() == kanidm.namespace()
647 && !expected_names.contains(&policy.name_any())
648 && policy.metadata.labels == Some(kanidm.generate_labels())
649 })
650 .collect::<Vec<_>>()
651 }
652 None => Vec::new(),
653 };
654 let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
655 .iter()
656 .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
657 .collect::<TryJoinAll<_>>();
658
659 let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
660 kanidm
661 .create_backend_tls_policy()
662 .into_iter()
663 .map(|policy| kanidm.patch(&ctx, policy))
664 .collect::<TryJoinAll<_>>()
665 } else {
666 try_join_all(Vec::new())
667 };
668
669 try_join!(
670 sts_delete_futures,
671 admin_secret_future,
672 replication_secret_futures,
673 sts_futures,
674 service_future,
675 rg_svcs_delete_futures,
676 rg_services_futures,
677 ingress_delete_futures,
678 ingress_futures,
679 http_route_delete_futures,
680 http_route_futures,
681 backend_tls_policy_delete_futures,
682 backend_tls_policy_futures
683 )?;
684
685 if is_kanidm_available(status.clone()) {
686 let namespace = kanidm.namespace().unwrap();
687 let name = kanidm.name_any();
688 let system_client = crate::controller::kanidm::KanidmClients::create_client(
689 &namespace,
690 &name,
691 crate::controller::kanidm::KanidmUser::Admin,
692 ctx.kaniop_ctx.client.clone(),
693 )
694 .await?;
695 reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
696
697 let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
698 &namespace,
699 &name,
700 crate::controller::kanidm::KanidmUser::IdmAdmin,
701 ctx.kaniop_ctx.client.clone(),
702 )
703 .await?;
704 let mail_sender_status =
705 reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
706 let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
707 if mail_sender_status != current_mail_sender_status {
708 let kanidm_api: Api<Kanidm> =
709 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
710 let status_patch = serde_json::json!({
711 "status": {
712 "mailSender": mail_sender_status
713 }
714 });
715 kanidm_api
716 .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
717 .await
718 .map_err(|e| {
719 Error::KubeError(
720 format!("failed to patch Kanidm/status {namespace}/{name}"),
721 Box::new(e),
722 )
723 })?;
724 }
725 }
726
727 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
728}
729
730async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
731 debug!(msg = "cleanup");
732
733 cleanup_mail_sender_resources(&kanidm, &ctx).await?;
734
735 let namespace = kanidm.namespace().unwrap();
736 let name = kanidm.name_any();
737
738 if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
739 &namespace,
740 &name,
741 crate::controller::kanidm::KanidmUser::IdmAdmin,
742 ctx.kaniop_ctx.client.clone(),
743 )
744 .await
745 {
746 cleanup_mail_sender_in_kanidm(&kanidm_client, &name).await?;
747 }
748
749 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
750 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
751}
752
753async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
754 let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
755 let start = std::time::Instant::now();
756 let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
757 let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
758
759 loop {
760 let sts = sts_api.get(sts_name).await.map_err(|e| {
761 Error::KubeError(
762 format!("failed to get StatefulSet {namespace}/{sts_name}"),
763 Box::new(e),
764 )
765 })?;
766
767 if sts.metadata.deletion_timestamp.is_some() {
768 return Err(Error::ReceiveOutput(format!(
769 "StatefulSet {namespace}/{sts_name} is being deleted"
770 )));
771 }
772
773 let spec = sts.spec.as_ref().ok_or_else(|| {
774 Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
775 })?;
776
777 let status = match sts.status {
778 Some(ref s) => s,
779 None => {
780 if start.elapsed() >= timeout {
781 return Err(Error::ReceiveOutput(format!(
782 "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
783 )));
784 }
785 sleep(poll_interval).await;
786 continue;
787 }
788 };
789
790 let replicas = spec.replicas.unwrap_or(0);
791
792 if replicas == 0 {
793 return Ok(());
794 }
795
796 let updated_replicas = status.updated_replicas.unwrap_or(0);
797 let current_revision = status.current_revision.as_deref().unwrap_or("");
798 let update_revision = status.update_revision.as_deref().unwrap_or("");
799 let ready_replicas = status.ready_replicas.unwrap_or(0);
800
801 if updated_replicas == replicas
802 && current_revision == update_revision
803 && ready_replicas == replicas
804 {
805 return Ok(());
806 }
807
808 if start.elapsed() >= timeout {
809 return Err(Error::ReceiveOutput(format!(
810 "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
811 )));
812 }
813
814 sleep(poll_interval).await;
815 }
816}
817
818async fn show_replica_cert_with_retries(
819 kanidm: &Kanidm,
820 ctx: Arc<Context>,
821 pod_name: &str,
822 max_attempts: u32,
823) -> std::result::Result<String, Error> {
824 let mut last_error = None;
825 for attempt in 0..max_attempts {
826 if attempt > 0 {
827 sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
828 }
829 match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
830 Ok(cert) => return Ok(cert),
831 Err(e) => {
832 info!(
833 msg = format!(
834 "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
835 attempt + 1,
836 max_attempts
837 )
838 );
839 last_error = Some(e);
840 }
841 }
842 }
843 Err(last_error.unwrap_or_else(|| {
844 Error::ReceiveOutput(format!(
845 "failed to get certificate for {pod_name} after {max_attempts} attempts"
846 ))
847 }))
848}
849
850impl Kanidm {
851 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
853 where
854 K: Resource<Scope = NamespaceResourceScope>
855 + Serialize
856 + Clone
857 + std::fmt::Debug
858 + for<'de> Deserialize<'de>,
859 <K as kube::Resource>::DynamicType: Default,
860 <K as Resource>::Scope: std::marker::Sized,
861 {
862 self.kube_delete(
863 ctx.kaniop_ctx.client.clone(),
864 &ctx.kaniop_ctx.metrics,
865 resource,
866 )
867 .await
868 }
869
870 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
871 where
872 K: Resource<Scope = NamespaceResourceScope>
873 + Serialize
874 + Clone
875 + std::fmt::Debug
876 + for<'de> Deserialize<'de>,
877 <K as kube::Resource>::DynamicType: Default,
878 <K as Resource>::Scope: std::marker::Sized,
879 {
880 self.kube_patch(
881 ctx.kaniop_ctx.client.clone(),
882 &ctx.kaniop_ctx.metrics,
883 resource,
884 KANIDM_OPERATOR_NAME,
885 )
886 .await
887 }
888
889 #[inline]
890 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
891 LABELS
892 .clone()
893 .into_iter()
894 .chain([
895 (INSTANCE_LABEL.to_string(), self.name_any()),
896 (CLUSTER_LABEL.to_string(), self.name_any()),
897 ])
898 .collect()
899 }
900
901 #[inline]
902 fn generate_labels(&self) -> BTreeMap<String, String> {
903 self.generate_resource_labels()
904 }
905
906 #[inline]
907 fn get_tls_secret_name(&self) -> String {
908 format!("{}-tls", self.name_any())
909 }
910
911 #[inline]
912 fn get_namespace(&self) -> String {
913 self.namespace().unwrap()
915 }
916
917 #[inline]
918 fn is_replication_enabled(&self) -> bool {
919 self.spec.replica_groups.len() > 1
920 || self
921 .spec
922 .replica_groups
923 .first()
924 .is_some_and(|rg| rg.replicas > 1)
925 || !self.spec.external_replication_nodes.is_empty()
926 }
927
928 #[inline]
929 fn is_updatable(&self, status: &KanidmStatus) -> bool {
930 status
931 .version
932 .as_ref()
933 .map(|v| {
934 if v.compatibility_result == VersionCompatibilityResult::Incompatible {
935 return false;
936 }
937 let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
938 let versions = (parse_semver(¤t_tag), parse_semver(&v.image_tag));
939 match v.upgrade_check_result {
940 KanidmUpgradeCheckResult::Passed => {
941 if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
942 minor <= v_minor + 1
943 } else {
944 true
945 }
946 }
947 KanidmUpgradeCheckResult::Failed => {
948 if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
949 versions
950 {
951 major == v_major && minor == v_minor && patch >= v_patch
952 } else {
953 v.image_tag == current_tag
954 }
955 }
956 }
957 })
958 .unwrap_or(true)
959 }
960
961 async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
962 let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
963 let pod = pod_api.get(pod_name).await.map_err(|e| {
964 Error::KubeError(
965 format!("failed to get pod {namespace}/{pod_name}"),
966 Box::new(e),
967 )
968 })?;
969 Ok(pod
970 .status
971 .as_ref()
972 .and_then(|s| s.conditions.as_ref())
973 .is_some_and(|conditions| {
974 conditions
975 .iter()
976 .any(|c| c.type_ == "Ready" && c.status == "True")
977 }))
978 }
979
980 async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
981 let start = std::time::Instant::now();
982 let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
983 let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
984
985 loop {
986 match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
987 Ok(true) => return Ok(()),
988 Ok(false) => {}
989 Err(e) => {
990 if !e.is_retryable() {
991 return Err(e);
992 }
993 trace!(msg = "pod readiness check failed, retrying", %e);
994 }
995 }
996
997 if start.elapsed() >= timeout {
998 return Err(Error::ReceiveOutput(format!(
999 "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1000 )));
1001 }
1002
1003 sleep(poll_interval).await;
1004 }
1005 }
1006
1007 async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1008 where
1009 I: IntoIterator<Item = T> + Debug,
1010 T: Into<String>,
1011 {
1012 let namespace = &self.get_namespace();
1013 trace!(
1014 msg = "pod exec",
1015 resource.name = &pod_name,
1016 resource.namespace = &namespace,
1017 ?command
1018 );
1019 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1020 let attached = pod
1021 .exec(
1022 pod_name,
1023 command,
1024 &AttachParams::default().container("kanidm"),
1025 )
1026 .await
1027 .map_err(|e| {
1028 Error::KubeError(
1029 format!("failed to exec pod {namespace}/{pod_name}"),
1030 Box::new(e),
1031 )
1032 })?;
1033 get_output(attached).await
1034 }
1035
1036 async fn exec_with_wait<I, T>(
1037 &self,
1038 ctx: Arc<Context>,
1039 pod_name: &str,
1040 command: I,
1041 ) -> Result<String>
1042 where
1043 I: IntoIterator<Item = T> + Debug,
1044 T: Into<String>,
1045 {
1046 let namespace = &self.get_namespace();
1047 Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1048 self.exec(ctx, pod_name, command).await
1049 }
1050
1051 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1052 where
1053 I: IntoIterator<Item = T> + Debug,
1054 T: Into<String>,
1055 {
1056 let first_replica_group = self
1058 .spec
1059 .replica_groups
1060 .first()
1061 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1062 let sts_name = self.statefulset_name(&first_replica_group.name);
1063 let pod_name = format!("{sts_name}-0");
1064 self.exec(ctx, &pod_name, command).await
1065 }
1066
1067 async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1068 where
1069 I: IntoIterator<Item = T> + Debug,
1070 T: Into<String>,
1071 {
1072 let first_replica_group = self
1073 .spec
1074 .replica_groups
1075 .first()
1076 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1077 let sts_name = self.statefulset_name(&first_replica_group.name);
1078 let pod_name = format!("{sts_name}-0");
1079 self.exec_with_wait(ctx, &pod_name, command).await
1080 }
1081}
1082
1083#[cfg(test)]
1084mod test {
1085 use super::statefulset::StatefulSetExt;
1086 use super::{Kanidm, reconcile_kanidm};
1087
1088 use crate::controller::State;
1089 use crate::kanidm::controller::context::{Context, Stores};
1090 use crate::kanidm::crd::KanidmStatus;
1091
1092 use kaniop_k8s_util::error::Result;
1093
1094 use std::sync::Arc;
1095
1096 use http::{Request, Response};
1097 use k8s_openapi::api::apps::v1::StatefulSet;
1098 use k8s_openapi::api::core::v1::Service;
1099 use k8s_openapi::api::networking::v1::Ingress;
1100 use kube::runtime::reflector::store::Writer;
1101 use kube::{Client, Resource, ResourceExt, client::Body};
1102 use opentelemetry::metrics::MeterProvider;
1103 use serde_json::json;
1104
1105 impl Kanidm {
1106 pub fn test() -> Self {
1108 let mut e = Kanidm::new(
1109 "test",
1110 serde_json::from_value(json!({
1111 "domain": "idm.example.com",
1112 "replicaGroups": [
1113 {
1114 "name": "default",
1115 "replicas": 1
1116 }
1117 ]
1118 }))
1119 .unwrap(),
1120 );
1121 e.meta_mut().namespace = Some("default".into());
1122 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1125 e
1126 }
1127
1128 pub fn with_ingress(mut self) -> Self {
1129 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1130 self
1131 }
1132
1133 pub fn with_replicas(mut self, replicas: i32) -> Self {
1135 self.spec.replica_groups[0].replicas = replicas;
1136 self
1137 }
1138
1139 pub fn needs_delete(mut self) -> Self {
1141 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1142 use k8s_openapi::jiff::Timestamp;
1143 let now = Timestamp::from_second(1491138632).unwrap(); self.meta_mut().deletion_timestamp = Some(Time(now));
1145 self
1146 }
1147
1148 pub fn with_status(mut self, status: KanidmStatus) -> Self {
1150 self.status = Some(status);
1151 self
1152 }
1153 }
1154
1155 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1157 pub struct ApiServerVerifier(ApiServerHandle);
1158
1159 pub enum Scenario {
1161 Create(Kanidm),
1162 CreateWithTwoReplicas(Kanidm),
1163 CreateWithIngress(Kanidm),
1164 CreateWithIngressWithTwoReplicas(Kanidm),
1165 }
1166
1167 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1168 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1169 .await
1170 .expect("timeout on mock apiserver")
1171 .expect("scenario succeeded")
1172 }
1173
1174 impl ApiServerVerifier {
1175 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1186 tokio::spawn(async move {
1187 match scenario {
1189 Scenario::Create(kanidm) => {
1190 self.handle_kanidm_get(kanidm.clone())
1191 .await
1192 .unwrap()
1193 .handle_kanidm_status_patch(kanidm.clone())
1194 .await
1195 .unwrap()
1196 .handle_statefulset_patch(kanidm.clone())
1197 .await
1198 .unwrap()
1199 .handle_service_patch(kanidm.clone())
1200 .await
1201 }
1202 Scenario::CreateWithTwoReplicas(kanidm) => {
1203 self.handle_kanidm_get(kanidm.clone())
1204 .await
1205 .unwrap()
1206 .handle_kanidm_status_patch(kanidm.clone())
1207 .await
1208 .unwrap()
1209 .handle_statefulset_patch(kanidm.clone())
1210 .await
1211 .unwrap()
1212 .handle_service_patch(kanidm.clone())
1213 .await
1214 }
1215 Scenario::CreateWithIngress(kanidm) => {
1216 self.handle_kanidm_get(kanidm.clone())
1217 .await
1218 .unwrap()
1219 .handle_kanidm_status_patch(kanidm.clone())
1220 .await
1221 .unwrap()
1222 .handle_statefulset_patch(kanidm.clone())
1223 .await
1224 .unwrap()
1225 .handle_service_patch(kanidm.clone())
1226 .await
1227 .unwrap()
1228 .handle_ingress_patch(kanidm.clone())
1229 .await
1230 }
1231 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1232 self.handle_kanidm_get(kanidm.clone())
1233 .await
1234 .unwrap()
1235 .handle_kanidm_status_patch(kanidm.clone())
1236 .await
1237 .unwrap()
1238 .handle_statefulset_patch(kanidm.clone())
1239 .await
1240 .unwrap()
1241 .handle_service_patch(kanidm.clone())
1242 .await
1243 .unwrap()
1244 .handle_ingress_patch(kanidm.clone())
1245 .await
1246 }
1247 }
1248 .expect("scenario completed without errors");
1249 })
1250 }
1251
1252 async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1253 let (request, send) = self.0.next_request().await.expect("service not called");
1254 assert_eq!(request.method(), http::Method::GET);
1255 let expected_uri_prefix = format!(
1256 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1257 kanidm.name_any()
1258 );
1259 let uri = request.uri().to_string();
1260 assert!(
1261 uri.starts_with(&expected_uri_prefix),
1262 "expected uri to start with {}, got {}",
1263 expected_uri_prefix,
1264 uri
1265 );
1266 let response = serde_json::to_vec(&kanidm).unwrap();
1267 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1268 Ok(self)
1269 }
1270
1271 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1272 let (request, send) = self.0.next_request().await.expect("service not called");
1273 assert_eq!(request.method(), http::Method::PATCH);
1274 assert_eq!(
1275 request.uri().to_string(),
1276 format!(
1277 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1278 kanidm.name_any()
1279 )
1280 );
1281
1282 let req_body = request.into_body().collect_bytes().await.unwrap();
1283 let json: serde_json::Value =
1284 serde_json::from_slice(&req_body).expect("patch object is json");
1285 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1286 .expect("valid kanidm status");
1287 let mut kanidm = Kanidm::test();
1288 kanidm.status = Some(status.clone());
1289 let response = serde_json::to_vec(&kanidm).unwrap();
1290 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1292 Ok(self)
1293 }
1294
1295 async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1296 for rg in kanidm.spec.replica_groups.iter() {
1297 let (request, send) = self.0.next_request().await.expect("service not called");
1298 assert_eq!(request.method(), http::Method::PATCH);
1299 assert_eq!(
1300 request.uri().to_string(),
1301 format!(
1302 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1303 kanidm.statefulset_name(&rg.name)
1304 )
1305 );
1306 let req_body = request.into_body().collect_bytes().await.unwrap();
1307 let json: serde_json::Value =
1308 serde_json::from_slice(&req_body).expect("patch object is json");
1309 let statefulset: StatefulSet =
1310 serde_json::from_value(json).expect("valid statefulset");
1311 assert_eq!(
1312 statefulset.clone().spec.unwrap().replicas.unwrap(),
1313 rg.replicas
1314 );
1315 let response = serde_json::to_vec(&statefulset).unwrap();
1316 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1318 }
1319
1320 Ok(self)
1321 }
1322
1323 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1324 let (request, send) = self.0.next_request().await.expect("service not called");
1325 assert_eq!(request.method(), http::Method::PATCH);
1326 assert_eq!(
1327 request.uri().to_string(),
1328 format!(
1329 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1330 kanidm.name_any()
1331 )
1332 );
1333
1334 let req_body = request.into_body().collect_bytes().await.unwrap();
1335 let json: serde_json::Value =
1336 serde_json::from_slice(&req_body).expect("patch object is json");
1337 let service: Service = serde_json::from_value(json).expect("valid service");
1338 let response = serde_json::to_vec(&service).unwrap();
1339 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1341 Ok(self)
1342 }
1343
1344 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1345 let (request, send) = self.0.next_request().await.expect("service not called");
1346 assert_eq!(request.method(), http::Method::PATCH);
1347 assert_eq!(
1348 request.uri().to_string(),
1349 format!(
1350 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1351 kanidm.name_any()
1352 )
1353 );
1354
1355 let req_body = request.into_body().collect_bytes().await.unwrap();
1356 let json: serde_json::Value =
1357 serde_json::from_slice(&req_body).expect("patch object is json");
1358 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1359 let response = serde_json::to_vec(&ingress).unwrap();
1360 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1362 Ok(self)
1363 }
1364 }
1365
1366 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1367 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1368 let mock_client = Client::new(mock_service, "default");
1369 let stores = Stores {
1370 stateful_set_store: Writer::default().as_reader(),
1371 service_store: Writer::default().as_reader(),
1372 ingress_store: Writer::default().as_reader(),
1373 secret_store: Writer::default().as_reader(),
1374 http_route_store: Some(Writer::default().as_reader()),
1375 backend_tls_policy_store: Some(Writer::default().as_reader()),
1376 deployment_store: Writer::default().as_reader(),
1377 config_map_store: Writer::default().as_reader(),
1378 };
1379 let controller_id = "test";
1380
1381 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
1383 let meter = provider.meter("test");
1384 let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
1385
1386 let state = State::new(
1387 metrics,
1388 Writer::default().as_reader(),
1389 Writer::default().as_reader(),
1390 None,
1391 );
1392 let ctx = Arc::new(Context::new(
1393 state.to_context(mock_client, controller_id),
1394 stores,
1395 ));
1396 (ctx, ApiServerVerifier(handle))
1397 }
1398
1399 #[tokio::test]
1400 async fn kanidm_create() {
1401 let (testctx, fakeserver) = get_test_context();
1402 let kanidm = Kanidm::test();
1403 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
1404 reconcile_kanidm(Arc::new(kanidm), testctx)
1405 .await
1406 .expect("reconciler");
1407 timeout_after_1s(mocksrv).await;
1408 }
1409
1410 #[tokio::test]
1411 async fn kanidm_create_with_two_replicas() {
1412 let (testctx, fakeserver) = get_test_context();
1413 let kanidm = Kanidm::test().with_replicas(2);
1414 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
1415 reconcile_kanidm(Arc::new(kanidm), testctx)
1416 .await
1417 .expect("reconciler");
1418 timeout_after_1s(mocksrv).await;
1419 }
1420
1421 #[tokio::test]
1422 async fn kanidm_create_with_ingress() {
1423 let (testctx, fakeserver) = get_test_context();
1424 let kanidm = Kanidm::test().with_ingress();
1425 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
1426 reconcile_kanidm(Arc::new(kanidm), testctx)
1427 .await
1428 .expect("reconciler");
1429 timeout_after_1s(mocksrv).await;
1430 }
1431
1432 #[tokio::test]
1433 async fn kanidm_create_with_ingress_with_two_replicas() {
1434 let (testctx, fakeserver) = get_test_context();
1435 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
1436 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
1437 reconcile_kanidm(Arc::new(kanidm), testctx)
1438 .await
1439 .expect("reconciler");
1440 timeout_after_1s(mocksrv).await;
1441 }
1442}