1mod ingress;
2pub mod secret;
3mod service;
4pub mod statefulset;
5mod status;
6
7use super::controller::{CONTROLLER_ID, context::Context};
8
9use self::ingress::IngressExt;
10use self::secret::SecretExt;
11use self::service::ServiceExt;
12use self::statefulset::{REPLICA_GROUP_LABEL, StatefulSetExt};
13use self::status::StatusExt;
14use self::status::{is_kanidm_available, is_kanidm_initialized};
15
16use crate::controller::context::KubeOperations;
17use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
18use crate::kanidm::crd::{Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult};
19use crate::telemetry;
20
21use kaniop_k8s_util::client::get_output;
22use kaniop_k8s_util::error::{Error, Result};
23use kaniop_k8s_util::resources::get_image_tag;
24
25use std::collections::BTreeMap;
26use std::fmt::Debug;
27use std::sync::{Arc, LazyLock};
28
29use futures::future::{TryJoinAll, join_all, try_join_all};
30use futures::stream::{self, StreamExt};
31use futures::try_join;
32use k8s_openapi::NamespaceResourceScope;
33use k8s_openapi::api::apps::v1::StatefulSet;
34use k8s_openapi::api::core::v1::Pod;
35use kube::Resource;
36use kube::ResourceExt;
37use kube::api::{Api, AttachParams};
38use kube::runtime::controller::Action;
39use kube::runtime::events::{Event, EventType};
40use kube::runtime::finalizer::{Event as Finalizer, finalizer};
41use serde::{Deserialize, Serialize};
42use tokio::time::Duration;
43use tracing::{Span, debug, field, info, instrument, trace, warn};
44
45pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
46const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
47pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
48
49const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
50
51static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
52 BTreeMap::from([
53 (NAME_LABEL.to_string(), "kanidm".to_string()),
54 (
55 MANAGED_BY_LABEL.to_string(),
56 format!("kaniop-{CONTROLLER_ID}"),
57 ),
58 ])
59});
60
61pub async fn reconcile_admins_secret(
62 kanidm: Arc<Kanidm>,
63 ctx: Arc<Context>,
64 status: &KanidmStatus,
65) -> Result<()> {
66 if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
67 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
68 kanidm.patch(&ctx, admins_secret).await?;
69 }
70 Ok(())
71}
72
73pub async fn reconcile_replication_secrets(
74 kanidm: Arc<Kanidm>,
75 ctx: Arc<Context>,
76 status: &KanidmStatus,
77) -> Result<()> {
78 let secret_names = status
79 .replica_statuses
80 .iter()
81 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
82 .collect::<Vec<_>>();
83 let deprecated_secrets = ctx
84 .stores
85 .secret_store
86 .state()
87 .into_iter()
88 .filter(|secret| {
89 secret
90 .metadata
91 .labels
92 .iter()
93 .any(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
94 && kanidm.admins_secret_name() != secret.name_any()
95 && secret_names.iter().all(|sn| sn != &secret.name_any())
96 })
97 .collect::<Vec<_>>();
98 let secret_delete_future = deprecated_secrets
99 .iter()
100 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
101 .collect::<TryJoinAll<_>>();
102 try_join!(secret_delete_future)?;
103
104 if kanidm.is_replication_enabled() {
105 let pending_secrets = status
106 .replica_statuses
107 .iter()
108 .filter(|rs| rs.state == KanidmReplicaState::Pending)
109 .map(|rs| kanidm.generate_replica_secret(ctx.clone(), &rs.pod_name))
110 .collect::<TryJoinAll<_>>();
111
112 let expiring_secrets = status
113 .replica_statuses
114 .iter()
115 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
116 .map(|rs| kanidm.update_replica_secret(ctx.clone(), &rs.pod_name))
117 .collect::<TryJoinAll<_>>();
118
119 let (pending_results, expiring_results) =
120 futures::try_join!(pending_secrets, expiring_secrets)?;
121
122 let secrets = pending_results
123 .into_iter()
124 .chain(expiring_results)
125 .collect::<Vec<_>>();
126 let secret_futures = secrets
127 .into_iter()
128 .map(|secret| kanidm.patch(&ctx, secret))
129 .collect::<Vec<_>>();
130 try_join_all(secret_futures).await?;
131
132 let has_pending_or_expiring = status.replica_statuses.iter().any(|rs| {
135 rs.state == KanidmReplicaState::Pending
136 || rs.state == KanidmReplicaState::CertificateExpiring
137 });
138
139 if has_pending_or_expiring {
140 let sts_api = Api::<StatefulSet>::namespaced(
141 ctx.kaniop_ctx.client.clone(),
142 &kanidm.get_namespace(),
143 );
144
145 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
147
148 let restart_futures = status
149 .replica_statuses
150 .iter()
151 .map(|rs| sts_api.restart(&rs.statefulset_name));
152
153 if has_single_replica {
154 let _ignore_errors = stream::iter(restart_futures)
156 .then(|f| f)
157 .collect::<Vec<_>>()
158 .await;
159 } else {
160 let _ignore_errors = join_all(restart_futures).await;
162 }
163 }
164 }
165
166 Ok(())
167}
168
169#[instrument(skip(ctx, kanidm))]
170pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
171 let trace_id = telemetry::get_trace_id();
172 Span::current().record("trace_id", field::display(&trace_id));
173 let _timer = ctx
174 .kaniop_ctx
175 .metrics
176 .reconcile_count_and_measure(&trace_id);
177 info!(msg = "reconciling Kanidm");
178
179 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
180 debug!(msg = "failed to reconcile status", %e);
181 ctx.kaniop_ctx.metrics.status_update_errors_inc();
182 e
183 })?;
184 let kanidm_api: Api<Kanidm> =
185 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
186 finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
187 match event {
188 Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
189 Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
190 }
191 })
192 .await
193 .map_err(|e| {
194 Error::FinalizerError(
195 "failed on kanidm account finalizer".to_string(),
196 Box::new(e),
197 )
198 })
199}
200
201async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
202 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
203 let replication_secret_future =
204 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
205
206 let sts_to_delete = ctx
207 .stores
208 .stateful_set_store
209 .state()
210 .into_iter()
211 .filter(|sts| {
212 sts.metadata.labels.iter().any(|l| {
213 l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
214 && match l.get(REPLICA_GROUP_LABEL) {
215 Some(sts_rg_name) => kanidm
216 .spec
217 .replica_groups
218 .iter()
219 .all(|rg| &rg.name != sts_rg_name),
220 None => false,
221 }
222 })
223 })
224 .collect::<Vec<_>>();
225 let sts_delete_future = sts_to_delete
226 .iter()
227 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
228 .collect::<TryJoinAll<_>>();
229
230 let sts_futures = match kanidm.is_updatable(&status) {
231 true => kanidm
232 .spec
233 .replica_groups
234 .iter()
235 .map(|rg| kanidm.patch(&ctx, kanidm.create_statefulset(rg)))
236 .collect::<TryJoinAll<_>>(),
237 false => {
238 let _ignore_error = ctx
239 .kaniop_ctx
240 .recorder
241 .publish(
242 &Event {
243 type_: EventType::Warning,
244 reason: "UpgradeBlocked".to_string(),
245 note: Some(
246 "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
247 ),
248 action: "ReconcileStatefulSet".to_string(),
249 secondary: None,
250 },
251 &kanidm.object_ref(&()),
252 )
253 .await
254 .map_err(|e| {
255 warn!(msg = "failed to publish KanidmError event", %e);
256 Error::KubeError("failed to publish event".to_string(), Box::new(e))
257 });
258 try_join_all([].into_iter())
259 }
260 };
261 let service_future = kanidm.patch(&ctx, kanidm.create_service());
262 let ingress_future = kanidm
263 .create_ingress()
264 .into_iter()
265 .map(|ingress| kanidm.patch(&ctx, ingress))
266 .collect::<TryJoinAll<_>>();
267
268 try_join!(
269 sts_delete_future,
270 admin_secret_future,
271 replication_secret_future,
272 sts_futures,
273 service_future,
274 ingress_future
275 )?;
276 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
277}
278
279async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
280 debug!(msg = "cleanup");
281 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
282 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
283}
284
285impl Kanidm {
286 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
288 where
289 K: Resource<Scope = NamespaceResourceScope>
290 + Serialize
291 + Clone
292 + std::fmt::Debug
293 + for<'de> Deserialize<'de>,
294 <K as kube::Resource>::DynamicType: Default,
295 <K as Resource>::Scope: std::marker::Sized,
296 {
297 self.kube_delete(
298 ctx.kaniop_ctx.client.clone(),
299 &ctx.kaniop_ctx.metrics,
300 resource,
301 )
302 .await
303 }
304
305 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
306 where
307 K: Resource<Scope = NamespaceResourceScope>
308 + Serialize
309 + Clone
310 + std::fmt::Debug
311 + for<'de> Deserialize<'de>,
312 <K as kube::Resource>::DynamicType: Default,
313 <K as Resource>::Scope: std::marker::Sized,
314 {
315 self.kube_patch(
316 ctx.kaniop_ctx.client.clone(),
317 &ctx.kaniop_ctx.metrics,
318 resource,
319 KANIDM_OPERATOR_NAME,
320 )
321 .await
322 }
323
324 #[inline]
325 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
326 LABELS
327 .clone()
328 .into_iter()
329 .chain([
330 (INSTANCE_LABEL.to_string(), self.name_any()),
331 (CLUSTER_LABEL.to_string(), self.name_any()),
332 ])
333 .collect()
334 }
335
336 #[inline]
337 fn get_tls_secret_name(&self) -> String {
338 format!("{}-tls", self.name_any())
339 }
340
341 #[inline]
342 fn get_namespace(&self) -> String {
343 self.namespace().unwrap()
345 }
346
347 #[inline]
348 fn is_replication_enabled(&self) -> bool {
349 self.spec.replica_groups.len() > 1
351 || self.spec.replica_groups.first().unwrap().replicas > 1
352 || !self.spec.external_replication_nodes.is_empty()
353 }
354
355 #[inline]
356 fn is_updatable(&self, status: &KanidmStatus) -> bool {
357 status
358 .version
359 .as_ref()
360 .map(|v| match v.upgrade_check_result {
361 KanidmUpgradeCheckResult::Passed => true,
362 KanidmUpgradeCheckResult::Failed => {
363 v.image_tag == get_image_tag(&self.spec.image).unwrap_or_default()
365 }
366 })
367 .unwrap_or(true)
368 }
369
370 async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
371 where
372 I: IntoIterator<Item = T> + Debug,
373 T: Into<String>,
374 {
375 let namespace = &self.get_namespace();
376 trace!(
377 msg = "pod exec",
378 resource.name = &pod_name,
379 resource.namespace = &namespace,
380 ?command
381 );
382 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
383 let attached = pod
384 .exec(pod_name, command, &AttachParams::default())
385 .await
386 .map_err(|e| {
387 Error::KubeError(
388 format!("failed to exec pod {namespace}/{pod_name}"),
389 Box::new(e),
390 )
391 })?;
392 get_output(attached).await
393 }
394
395 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
396 where
397 I: IntoIterator<Item = T> + Debug,
398 T: Into<String>,
399 {
400 let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
403 let pod_name = format!("{sts_name}-0");
404 self.exec(ctx, &pod_name, command).await
405 }
406}
407
408#[cfg(test)]
409mod test {
410 use super::statefulset::StatefulSetExt;
411 use super::{Kanidm, reconcile_kanidm};
412
413 use crate::controller::State;
414 use crate::kanidm::controller::context::{Context, Stores};
415 use crate::kanidm::crd::KanidmStatus;
416
417 use kaniop_k8s_util::error::Result;
418
419 use std::sync::Arc;
420
421 use http::{Request, Response};
422 use k8s_openapi::api::apps::v1::StatefulSet;
423 use k8s_openapi::api::core::v1::Service;
424 use k8s_openapi::api::networking::v1::Ingress;
425 use kube::runtime::reflector::store::Writer;
426 use kube::{Client, Resource, ResourceExt, client::Body};
427 use serde_json::json;
428
429 impl Kanidm {
430 pub fn test() -> Self {
432 let mut e = Kanidm::new(
433 "test",
434 serde_json::from_value(json!({
435 "domain": "idm.example.com",
436 "replicaGroups": [
437 {
438 "name": "default",
439 "replicas": 1
440 }
441 ]
442 }))
443 .unwrap(),
444 );
445 e.meta_mut().namespace = Some("default".into());
446 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
449 e
450 }
451
452 pub fn with_ingress(mut self) -> Self {
453 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
454 self
455 }
456
457 pub fn with_replicas(mut self, replicas: i32) -> Self {
459 self.spec.replica_groups[0].replicas = replicas;
461 self
462 }
463
464 pub fn needs_delete(mut self) -> Self {
466 use chrono::prelude::{DateTime, TimeZone, Utc};
467 let now: DateTime<Utc> = Utc.with_ymd_and_hms(2017, 4, 2, 12, 50, 32).unwrap();
468 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
469 self.meta_mut().deletion_timestamp = Some(Time(now));
470 self
471 }
472
473 pub fn with_status(mut self, status: KanidmStatus) -> Self {
475 self.status = Some(status);
476 self
477 }
478 }
479
480 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
482 pub struct ApiServerVerifier(ApiServerHandle);
483
484 pub enum Scenario {
486 Create(Kanidm),
487 CreateWithTwoReplicas(Kanidm),
488 CreateWithIngress(Kanidm),
489 CreateWithIngressWithTwoReplicas(Kanidm),
490 }
491
492 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
493 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
494 .await
495 .expect("timeout on mock apiserver")
496 .expect("scenario succeeded")
497 }
498
499 impl ApiServerVerifier {
500 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
511 tokio::spawn(async move {
512 match scenario {
514 Scenario::Create(kanidm) => {
515 self.handle_kanidm_status_patch(kanidm.clone())
516 .await
517 .unwrap()
518 .handle_statefulset_patch(kanidm.clone())
519 .await
520 .unwrap()
521 .handle_service_patch(kanidm.clone())
522 .await
523 }
524 Scenario::CreateWithTwoReplicas(kanidm) => {
525 self.handle_kanidm_status_patch(kanidm.clone())
526 .await
527 .unwrap()
528 .handle_statefulset_patch(kanidm.clone())
529 .await
530 .unwrap()
531 .handle_service_patch(kanidm.clone())
532 .await
533 }
534 Scenario::CreateWithIngress(kanidm) => {
535 self.handle_kanidm_status_patch(kanidm.clone())
536 .await
537 .unwrap()
538 .handle_statefulset_patch(kanidm.clone())
539 .await
540 .unwrap()
541 .handle_service_patch(kanidm.clone())
542 .await
543 .unwrap()
544 .handle_ingress_patch(kanidm.clone())
545 .await
546 }
547 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
548 self.handle_kanidm_status_patch(kanidm.clone())
549 .await
550 .unwrap()
551 .handle_statefulset_patch(kanidm.clone())
552 .await
553 .unwrap()
554 .handle_service_patch(kanidm.clone())
555 .await
556 .unwrap()
557 .handle_ingress_patch(kanidm.clone())
558 .await
559 }
560 }
561 .expect("scenario completed without errors");
562 })
563 }
564
565 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
566 let (request, send) = self.0.next_request().await.expect("service not called");
567 assert_eq!(request.method(), http::Method::PATCH);
568 assert_eq!(
569 request.uri().to_string(),
570 format!(
571 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?&force=true&fieldManager=kanidms.kaniop.rs",
572 kanidm.name_any()
573 )
574 );
575
576 let req_body = request.into_body().collect_bytes().await.unwrap();
577 let json: serde_json::Value =
578 serde_json::from_slice(&req_body).expect("patch object is json");
579 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
580 .expect("valid kanidm status");
581 let mut kanidm = Kanidm::test();
582 kanidm.status = Some(status.clone());
583 let response = serde_json::to_vec(&kanidm).unwrap();
584 send.send_response(Response::builder().body(Body::from(response)).unwrap());
586 Ok(self)
587 }
588
589 async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
590 for rg in kanidm.spec.replica_groups.iter() {
591 let (request, send) = self.0.next_request().await.expect("service not called");
592 assert_eq!(request.method(), http::Method::PATCH);
593 assert_eq!(
594 request.uri().to_string(),
595 format!(
596 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
597 kanidm.statefulset_name(&rg.name)
598 )
599 );
600 let req_body = request.into_body().collect_bytes().await.unwrap();
601 let json: serde_json::Value =
602 serde_json::from_slice(&req_body).expect("patch object is json");
603 let statefulset: StatefulSet =
604 serde_json::from_value(json).expect("valid statefulset");
605 assert_eq!(
606 statefulset.clone().spec.unwrap().replicas.unwrap(),
607 rg.replicas
608 );
609 let response = serde_json::to_vec(&statefulset).unwrap();
610 send.send_response(Response::builder().body(Body::from(response)).unwrap());
612 }
613
614 Ok(self)
615 }
616
617 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
618 let (request, send) = self.0.next_request().await.expect("service not called");
619 assert_eq!(request.method(), http::Method::PATCH);
620 assert_eq!(
621 request.uri().to_string(),
622 format!(
623 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
624 kanidm.name_any()
625 )
626 );
627
628 let req_body = request.into_body().collect_bytes().await.unwrap();
629 let json: serde_json::Value =
630 serde_json::from_slice(&req_body).expect("patch object is json");
631 let service: Service = serde_json::from_value(json).expect("valid service");
632 let response = serde_json::to_vec(&service).unwrap();
633 send.send_response(Response::builder().body(Body::from(response)).unwrap());
635 Ok(self)
636 }
637
638 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
639 let (request, send) = self.0.next_request().await.expect("service not called");
640 assert_eq!(request.method(), http::Method::PATCH);
641 assert_eq!(
642 request.uri().to_string(),
643 format!(
644 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
645 kanidm.name_any()
646 )
647 );
648
649 let req_body = request.into_body().collect_bytes().await.unwrap();
650 let json: serde_json::Value =
651 serde_json::from_slice(&req_body).expect("patch object is json");
652 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
653 let response = serde_json::to_vec(&ingress).unwrap();
654 send.send_response(Response::builder().body(Body::from(response)).unwrap());
656 Ok(self)
657 }
658 }
659
660 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
661 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
662 let mock_client = Client::new(mock_service, "default");
663 let stores = Stores {
664 stateful_set_store: Writer::default().as_reader(),
665 service_store: Writer::default().as_reader(),
666 ingress_store: Writer::default().as_reader(),
667 secret_store: Writer::default().as_reader(),
668 };
669 let controller_id = "test";
670 let state = State::new(
671 Default::default(),
672 &[controller_id],
673 Writer::default().as_reader(),
674 Writer::default().as_reader(),
675 );
676 let ctx = Arc::new(Context::new(
677 state.to_context(mock_client, controller_id),
678 stores,
679 ));
680 (ctx, ApiServerVerifier(handle))
681 }
682
683 #[tokio::test]
684 async fn kanidm_create() {
685 let (testctx, fakeserver) = get_test_context();
686 let kanidm = Kanidm::test();
687 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
688 reconcile_kanidm(Arc::new(kanidm), testctx)
689 .await
690 .expect("reconciler");
691 timeout_after_1s(mocksrv).await;
692 }
693
694 #[tokio::test]
695 async fn kanidm_create_with_two_replicas() {
696 let (testctx, fakeserver) = get_test_context();
697 let kanidm = Kanidm::test().with_replicas(2);
698 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
699 reconcile_kanidm(Arc::new(kanidm), testctx)
700 .await
701 .expect("reconciler");
702 timeout_after_1s(mocksrv).await;
703 }
704
705 #[tokio::test]
706 async fn kanidm_create_with_ingress() {
707 let (testctx, fakeserver) = get_test_context();
708 let kanidm = Kanidm::test().with_ingress();
709 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
710 reconcile_kanidm(Arc::new(kanidm), testctx)
711 .await
712 .expect("reconciler");
713 timeout_after_1s(mocksrv).await;
714 }
715
716 #[tokio::test]
717 async fn kanidm_create_with_ingress_with_two_replicas() {
718 let (testctx, fakeserver) = get_test_context();
719 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
720 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
721 reconcile_kanidm(Arc::new(kanidm), testctx)
722 .await
723 .expect("reconciler");
724 timeout_after_1s(mocksrv).await;
725 }
726}