kaniop_operator/kanidm/reconcile/
mod.rs

1mod ingress;
2pub mod secret;
3mod service;
4pub mod statefulset;
5mod status;
6
7use super::controller::{CONTROLLER_ID, context::Context};
8
9use self::ingress::IngressExt;
10use self::secret::SecretExt;
11use self::service::ServiceExt;
12use self::statefulset::{REPLICA_GROUP_LABEL, StatefulSetExt};
13use self::status::StatusExt;
14
15use crate::controller::context::KubeOperations;
16use crate::controller::{DEFAULT_RECONCILE_INTERVAL, INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
17use crate::error::{Error, Result};
18use crate::kanidm::crd::{Kanidm, KanidmReplicaState, KanidmStatus};
19use crate::telemetry;
20
21use kaniop_k8s_util::client::get_output;
22
23use std::collections::BTreeMap;
24use std::fmt::Debug;
25use std::sync::{Arc, LazyLock};
26
27use futures::future::{TryJoinAll, join_all, try_join_all};
28use futures::stream::{self, StreamExt};
29use futures::try_join;
30use k8s_openapi::NamespaceResourceScope;
31use k8s_openapi::api::apps::v1::StatefulSet;
32use k8s_openapi::api::core::v1::Pod;
33use kube::Resource;
34use kube::ResourceExt;
35use kube::api::{Api, AttachParams};
36use kube::runtime::finalizer::{Event as Finalizer, finalizer};
37use serde::{Deserialize, Serialize};
38
39use kube::runtime::controller::Action;
40use status::{is_kanidm_available, is_kanidm_initialized};
41use tracing::{Span, debug, field, info, instrument, trace};
42
43pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
44const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
45pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
46
47static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
48    BTreeMap::from([
49        (NAME_LABEL.to_string(), "kanidm".to_string()),
50        (
51            MANAGED_BY_LABEL.to_string(),
52            format!("kaniop-{CONTROLLER_ID}"),
53        ),
54    ])
55});
56
57pub async fn reconcile_admins_secret(
58    kanidm: Arc<Kanidm>,
59    ctx: Arc<Context>,
60    status: &Result<KanidmStatus>,
61) -> Result<()> {
62    if let Ok(s) = status {
63        if is_kanidm_available(s.clone()) && !is_kanidm_initialized(s.clone()) {
64            let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
65            kanidm.patch(&ctx, admins_secret).await?;
66        }
67    }
68    Ok(())
69}
70
71pub async fn reconcile_replication_secrets(
72    kanidm: Arc<Kanidm>,
73    ctx: Arc<Context>,
74    status: &Result<KanidmStatus>,
75) -> Result<()> {
76    if let Ok(s) = status {
77        let secret_names = s
78            .replica_statuses
79            .iter()
80            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
81            .collect::<Vec<_>>();
82        let deprecated_secrets = ctx
83            .stores
84            .secret_store
85            .state()
86            .into_iter()
87            .filter(|secret| {
88                secret
89                    .metadata
90                    .labels
91                    .iter()
92                    .any(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
93                    && kanidm.admins_secret_name() != secret.name_any()
94                    && secret_names.iter().all(|sn| sn != &secret.name_any())
95            })
96            .collect::<Vec<_>>();
97        let secret_delete_future = deprecated_secrets
98            .iter()
99            .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
100            .collect::<TryJoinAll<_>>();
101        try_join!(secret_delete_future)?;
102
103        if kanidm.is_replication_enabled() {
104            let pending_secrets = s
105                .replica_statuses
106                .iter()
107                .filter(|rs| rs.state == KanidmReplicaState::Pending)
108                .map(|rs| kanidm.generate_replica_secret(ctx.clone(), &rs.pod_name))
109                .collect::<TryJoinAll<_>>();
110
111            let expiring_secrets = s
112                .replica_statuses
113                .iter()
114                .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
115                .map(|rs| kanidm.update_replica_secret(ctx.clone(), &rs.pod_name))
116                .collect::<TryJoinAll<_>>();
117
118            let (pending_results, expiring_results) =
119                futures::try_join!(pending_secrets, expiring_secrets)?;
120
121            let secrets = pending_results
122                .into_iter()
123                .chain(expiring_results)
124                .collect::<Vec<_>>();
125            let secret_futures = secrets
126                .into_iter()
127                .map(|secret| kanidm.patch(&ctx, secret))
128                .collect::<Vec<_>>();
129            try_join_all(secret_futures).await?;
130
131            // If there are any replicas in pending or cert expiring state, trigger a restart
132            // of all statefulsets to pickup new certs
133            let has_pending_or_expiring = s.replica_statuses.iter().any(|rs| {
134                rs.state == KanidmReplicaState::Pending
135                    || rs.state == KanidmReplicaState::CertificateExpiring
136            });
137
138            if has_pending_or_expiring {
139                let sts_api = Api::<StatefulSet>::namespaced(
140                    ctx.kaniop_ctx.client.clone(),
141                    &kanidm.get_namespace(),
142                );
143
144                // Check if any replica group has only 1 replica
145                let has_single_replica =
146                    kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
147
148                let restart_futures = s
149                    .replica_statuses
150                    .iter()
151                    .map(|rs| sts_api.restart(&rs.statefulset_name));
152
153                if has_single_replica {
154                    // Restart sequentially to avoid downtime
155                    let _ignore_errors = stream::iter(restart_futures)
156                        .then(|f| f)
157                        .collect::<Vec<_>>()
158                        .await;
159                } else {
160                    // Restart concurrently for replica groups with multiple replicas
161                    let _ignore_errors = join_all(restart_futures).await;
162                }
163            }
164        }
165    }
166    Ok(())
167}
168
169#[instrument(skip(ctx, kanidm))]
170pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
171    let trace_id = telemetry::get_trace_id();
172    Span::current().record("trace_id", field::display(&trace_id));
173    let _timer = ctx
174        .kaniop_ctx
175        .metrics
176        .reconcile_count_and_measure(&trace_id);
177    info!(msg = "reconciling Kanidm");
178
179    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
180        debug!(msg = "failed to reconcile status", %e);
181        ctx.kaniop_ctx.metrics.status_update_errors_inc();
182        e
183    });
184    let kanidm_api: Api<Kanidm> =
185        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
186    finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
187        match event {
188            Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
189            Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
190        }
191    })
192    .await
193    .map_err(|e| {
194        Error::FinalizerError(
195            "failed on kanidm account finalizer".to_string(),
196            Box::new(e),
197        )
198    })
199}
200
201async fn reconcile(
202    kanidm: Arc<Kanidm>,
203    ctx: Arc<Context>,
204    // TODO: check if status is not OK, probably is better to not reconcile
205    status: Result<KanidmStatus>,
206) -> Result<Action> {
207    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
208    let replication_secret_future =
209        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
210
211    let sts_to_delete = ctx
212        .stores
213        .stateful_set_store
214        .state()
215        .into_iter()
216        .filter(|sts| {
217            sts.metadata.labels.iter().any(|l| {
218                l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
219                    && match l.get(REPLICA_GROUP_LABEL) {
220                        Some(sts_rg_name) => kanidm
221                            .spec
222                            .replica_groups
223                            .iter()
224                            .all(|rg| &rg.name != sts_rg_name),
225                        None => false,
226                    }
227            })
228        })
229        .collect::<Vec<_>>();
230    let sts_delete_future = sts_to_delete
231        .iter()
232        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
233        .collect::<TryJoinAll<_>>();
234
235    let sts_futures = kanidm
236        .spec
237        .replica_groups
238        .iter()
239        .map(|rg| kanidm.patch(&ctx, kanidm.create_statefulset(rg)))
240        .collect::<TryJoinAll<_>>();
241    let service_future = kanidm.patch(&ctx, kanidm.create_service());
242    let ingress_future = kanidm
243        .create_ingress()
244        .into_iter()
245        .map(|ingress| kanidm.patch(&ctx, ingress))
246        .collect::<TryJoinAll<_>>();
247
248    try_join!(
249        sts_delete_future,
250        admin_secret_future,
251        replication_secret_future,
252        sts_futures,
253        service_future,
254        ingress_future
255    )?;
256    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
257}
258
259async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
260    debug!(msg = "cleanup");
261    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
262    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
263}
264
265impl Kanidm {
266    // Convenience methods that handle context and operator name
267    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
268    where
269        K: Resource<Scope = NamespaceResourceScope>
270            + Serialize
271            + Clone
272            + std::fmt::Debug
273            + for<'de> Deserialize<'de>,
274        <K as kube::Resource>::DynamicType: Default,
275        <K as Resource>::Scope: std::marker::Sized,
276    {
277        self.kube_delete(
278            ctx.kaniop_ctx.client.clone(),
279            &ctx.kaniop_ctx.metrics,
280            resource,
281        )
282        .await
283    }
284
285    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
286    where
287        K: Resource<Scope = NamespaceResourceScope>
288            + Serialize
289            + Clone
290            + std::fmt::Debug
291            + for<'de> Deserialize<'de>,
292        <K as kube::Resource>::DynamicType: Default,
293        <K as Resource>::Scope: std::marker::Sized,
294    {
295        self.kube_patch(
296            ctx.kaniop_ctx.client.clone(),
297            &ctx.kaniop_ctx.metrics,
298            resource,
299            KANIDM_OPERATOR_NAME,
300        )
301        .await
302    }
303
304    #[inline]
305    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
306        LABELS
307            .clone()
308            .into_iter()
309            .chain([
310                (INSTANCE_LABEL.to_string(), self.name_any()),
311                (CLUSTER_LABEL.to_string(), self.name_any()),
312            ])
313            .collect()
314    }
315
316    #[inline]
317    fn get_tls_secret_name(&self) -> String {
318        format!("{}-tls", self.name_any())
319    }
320
321    #[inline]
322    fn get_namespace(&self) -> String {
323        // safe unwrap: Kanidm is namespaced scoped
324        self.namespace().unwrap()
325    }
326
327    #[inline]
328    fn is_replication_enabled(&self) -> bool {
329        // safe unwrap: at least one replica group is required
330        self.spec.replica_groups.len() > 1
331            || self.spec.replica_groups.first().unwrap().replicas > 1
332            || !self.spec.external_replication_nodes.is_empty()
333    }
334
335    async fn exec<I, T>(
336        &self,
337        ctx: Arc<Context>,
338        pod_name: &str,
339        command: I,
340    ) -> Result<Option<String>>
341    where
342        I: IntoIterator<Item = T> + Debug,
343        T: Into<String>,
344    {
345        let namespace = &self.get_namespace();
346        trace!(
347            msg = "pod exec",
348            resource.name = &pod_name,
349            resource.namespace = &namespace,
350            ?command
351        );
352        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
353        let attached = pod
354            .exec(pod_name, command, &AttachParams::default().stderr(false))
355            .await
356            .map_err(|e| {
357                Error::KubeError(
358                    format!("failed to exec pod {namespace}/{pod_name}"),
359                    Box::new(e),
360                )
361            })?;
362        Ok(get_output(attached).await)
363    }
364
365    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<Option<String>>
366    where
367        I: IntoIterator<Item = T> + Debug,
368        T: Into<String>,
369    {
370        // TODO: if replicas > 1 and replicas initialized, exec on pod available
371        // safe unwrap: at least one replica group is required
372        let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
373        let pod_name = format!("{sts_name}-0");
374        self.exec(ctx, &pod_name, command).await
375    }
376}
377
378#[cfg(test)]
379mod test {
380    use super::statefulset::StatefulSetExt;
381    use super::{Kanidm, reconcile_kanidm};
382
383    use crate::controller::State;
384    use crate::error::Result;
385    use crate::kanidm::controller::context::{Context, Stores};
386    use crate::kanidm::crd::KanidmStatus;
387    use k8s_openapi::api::core::v1::Service;
388    use k8s_openapi::api::networking::v1::Ingress;
389
390    use std::sync::Arc;
391
392    use http::{Request, Response};
393    use k8s_openapi::api::apps::v1::StatefulSet;
394    use kube::runtime::reflector::store::Writer;
395    use kube::{Client, Resource, ResourceExt, client::Body};
396    use serde_json::json;
397
398    impl Kanidm {
399        /// A normal test kanidm with a given status
400        pub fn test() -> Self {
401            let mut e = Kanidm::new(
402                "test",
403                serde_json::from_value(json!({
404                    "domain": "idm.example.com",
405                    "replicaGroups": [
406                        {
407                            "name": "default",
408                            "replicas": 1
409                        }
410                    ]
411                }))
412                .unwrap(),
413            );
414            e.meta_mut().namespace = Some("default".into());
415            // Pre-populate finalizer so reconcile callback runs immediately
416            // (skips the "add finalizer + wait for next reconcile" step)
417            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
418            e
419        }
420
421        pub fn with_ingress(mut self) -> Self {
422            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
423            self
424        }
425
426        /// Modify kanidm replicas
427        pub fn with_replicas(mut self, replicas: i32) -> Self {
428            // TODO: manage replica_groups
429            self.spec.replica_groups[0].replicas = replicas;
430            self
431        }
432
433        /// Modify kanidm to set a deletion timestamp
434        pub fn needs_delete(mut self) -> Self {
435            use chrono::prelude::{DateTime, TimeZone, Utc};
436            let now: DateTime<Utc> = Utc.with_ymd_and_hms(2017, 4, 2, 12, 50, 32).unwrap();
437            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
438            self.meta_mut().deletion_timestamp = Some(Time(now));
439            self
440        }
441
442        /// Modify a kanidm to have an expected status
443        pub fn with_status(mut self, status: KanidmStatus) -> Self {
444            self.status = Some(status);
445            self
446        }
447    }
448
449    // We wrap tower_test::mock::Handle
450    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
451    pub struct ApiServerVerifier(ApiServerHandle);
452
453    /// Scenarios we test for in ApiServerVerifier
454    pub enum Scenario {
455        Create(Kanidm),
456        CreateWithTwoReplicas(Kanidm),
457        CreateWithIngress(Kanidm),
458        CreateWithIngressWithTwoReplicas(Kanidm),
459    }
460
461    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
462        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
463            .await
464            .expect("timeout on mock apiserver")
465            .expect("scenario succeeded")
466    }
467
468    impl ApiServerVerifier {
469        /// Tests only get to run specific scenarios that has matching handlers
470        ///
471        /// This setup makes it easy to handle multiple requests by chaining handlers together.
472        ///
473        /// NB: If the controller is making more calls than we are handling in the scenario,
474        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
475        ///
476        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
477        /// scenario runs to completion (i.e. all expected calls were responded to),
478        /// using the timeout to catch missing api calls to Kubernetes.
479        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
480            tokio::spawn(async move {
481                // moving self => one scenario per test
482                match scenario {
483                    Scenario::Create(kanidm) => {
484                        self.handle_kanidm_status_patch(kanidm.clone())
485                            .await
486                            .unwrap()
487                            .handle_statefulset_patch(kanidm.clone())
488                            .await
489                            .unwrap()
490                            .handle_service_patch(kanidm.clone())
491                            .await
492                    }
493                    Scenario::CreateWithTwoReplicas(kanidm) => {
494                        self.handle_kanidm_status_patch(kanidm.clone())
495                            .await
496                            .unwrap()
497                            .handle_statefulset_patch(kanidm.clone())
498                            .await
499                            .unwrap()
500                            .handle_service_patch(kanidm.clone())
501                            .await
502                    }
503                    Scenario::CreateWithIngress(kanidm) => {
504                        self.handle_kanidm_status_patch(kanidm.clone())
505                            .await
506                            .unwrap()
507                            .handle_statefulset_patch(kanidm.clone())
508                            .await
509                            .unwrap()
510                            .handle_service_patch(kanidm.clone())
511                            .await
512                            .unwrap()
513                            .handle_ingress_patch(kanidm.clone())
514                            .await
515                    }
516                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
517                        self.handle_kanidm_status_patch(kanidm.clone())
518                            .await
519                            .unwrap()
520                            .handle_statefulset_patch(kanidm.clone())
521                            .await
522                            .unwrap()
523                            .handle_service_patch(kanidm.clone())
524                            .await
525                            .unwrap()
526                            .handle_ingress_patch(kanidm.clone())
527                            .await
528                    }
529                }
530                .expect("scenario completed without errors");
531            })
532        }
533
534        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
535            let (request, send) = self.0.next_request().await.expect("service not called");
536            assert_eq!(request.method(), http::Method::PATCH);
537            assert_eq!(
538                request.uri().to_string(),
539                format!(
540                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?&force=true&fieldManager=kanidms.kaniop.rs",
541                    kanidm.name_any()
542                )
543            );
544
545            let req_body = request.into_body().collect_bytes().await.unwrap();
546            let json: serde_json::Value =
547                serde_json::from_slice(&req_body).expect("patch object is json");
548            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
549                .expect("valid kanidm status");
550            let response = serde_json::to_vec(&status).unwrap();
551            // pass through kanidm "patch accepted"
552            send.send_response(Response::builder().body(Body::from(response)).unwrap());
553            Ok(self)
554        }
555
556        async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
557            for rg in kanidm.spec.replica_groups.iter() {
558                let (request, send) = self.0.next_request().await.expect("service not called");
559                assert_eq!(request.method(), http::Method::PATCH);
560                assert_eq!(
561                    request.uri().to_string(),
562                    format!(
563                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
564                        kanidm.statefulset_name(&rg.name)
565                    )
566                );
567                let req_body = request.into_body().collect_bytes().await.unwrap();
568                let json: serde_json::Value =
569                    serde_json::from_slice(&req_body).expect("patch object is json");
570                let statefulset: StatefulSet =
571                    serde_json::from_value(json).expect("valid statefulset");
572                assert_eq!(
573                    statefulset.clone().spec.unwrap().replicas.unwrap(),
574                    rg.replicas
575                );
576                let response = serde_json::to_vec(&statefulset).unwrap();
577                // pass through kanidm "patch accepted"
578                send.send_response(Response::builder().body(Body::from(response)).unwrap());
579            }
580
581            Ok(self)
582        }
583
584        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
585            let (request, send) = self.0.next_request().await.expect("service not called");
586            assert_eq!(request.method(), http::Method::PATCH);
587            assert_eq!(
588                request.uri().to_string(),
589                format!(
590                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
591                    kanidm.name_any()
592                )
593            );
594
595            let req_body = request.into_body().collect_bytes().await.unwrap();
596            let json: serde_json::Value =
597                serde_json::from_slice(&req_body).expect("patch object is json");
598            let service: Service = serde_json::from_value(json).expect("valid service");
599            let response = serde_json::to_vec(&service).unwrap();
600            // pass through kanidm "patch accepted"
601            send.send_response(Response::builder().body(Body::from(response)).unwrap());
602            Ok(self)
603        }
604
605        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
606            let (request, send) = self.0.next_request().await.expect("service not called");
607            assert_eq!(request.method(), http::Method::PATCH);
608            assert_eq!(
609                request.uri().to_string(),
610                format!(
611                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
612                    kanidm.name_any()
613                )
614            );
615
616            let req_body = request.into_body().collect_bytes().await.unwrap();
617            let json: serde_json::Value =
618                serde_json::from_slice(&req_body).expect("patch object is json");
619            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
620            let response = serde_json::to_vec(&ingress).unwrap();
621            // pass through kanidm "patch accepted"
622            send.send_response(Response::builder().body(Body::from(response)).unwrap());
623            Ok(self)
624        }
625    }
626
627    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
628        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
629        let mock_client = Client::new(mock_service, "default");
630        let stores = Stores {
631            stateful_set_store: Writer::default().as_reader(),
632            service_store: Writer::default().as_reader(),
633            ingress_store: Writer::default().as_reader(),
634            secret_store: Writer::default().as_reader(),
635        };
636        let controller_id = "test";
637        let state = State::new(
638            Default::default(),
639            &[controller_id],
640            Writer::default().as_reader(),
641            Writer::default().as_reader(),
642        );
643        let ctx = Arc::new(Context::new(
644            state.to_context(mock_client, controller_id),
645            stores,
646        ));
647        (ctx, ApiServerVerifier(handle))
648    }
649
650    #[tokio::test]
651    async fn kanidm_create() {
652        let (testctx, fakeserver) = get_test_context();
653        let kanidm = Kanidm::test();
654        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
655        reconcile_kanidm(Arc::new(kanidm), testctx)
656            .await
657            .expect("reconciler");
658        timeout_after_1s(mocksrv).await;
659    }
660
661    #[tokio::test]
662    async fn kanidm_create_with_two_replicas() {
663        let (testctx, fakeserver) = get_test_context();
664        let kanidm = Kanidm::test().with_replicas(2);
665        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
666        reconcile_kanidm(Arc::new(kanidm), testctx)
667            .await
668            .expect("reconciler");
669        timeout_after_1s(mocksrv).await;
670    }
671
672    #[tokio::test]
673    async fn kanidm_create_with_ingress() {
674        let (testctx, fakeserver) = get_test_context();
675        let kanidm = Kanidm::test().with_ingress();
676        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
677        reconcile_kanidm(Arc::new(kanidm), testctx)
678            .await
679            .expect("reconciler");
680        timeout_after_1s(mocksrv).await;
681    }
682
683    #[tokio::test]
684    async fn kanidm_create_with_ingress_with_two_replicas() {
685        let (testctx, fakeserver) = get_test_context();
686        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
687        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
688        reconcile_kanidm(Arc::new(kanidm), testctx)
689            .await
690            .expect("reconciler");
691        timeout_after_1s(mocksrv).await;
692    }
693}