Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::StatefulSetExt;
21use self::status::StatusExt;
22use self::status::{is_kanidm_available, is_kanidm_initialized};
23
24use crate::controller::context::KubeOperations;
25use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
26use crate::kanidm::crd::{
27    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
28};
29use crate::kanidm::reconcile::statefulset::REPLICA_LABEL;
30use crate::telemetry;
31
32use kaniop_k8s_util::client::get_output;
33use kaniop_k8s_util::error::{Error, Result};
34use kaniop_k8s_util::parse::parse_semver;
35use kaniop_k8s_util::resources::get_image_tag;
36
37use std::collections::{BTreeMap, BTreeSet};
38use std::fmt::Debug;
39use std::sync::{Arc, LazyLock};
40
41use futures::future::{TryJoinAll, join_all, try_join_all};
42use futures::stream::{self, StreamExt};
43use futures::try_join;
44use k8s_openapi::NamespaceResourceScope;
45use k8s_openapi::api::apps::v1::StatefulSet;
46use k8s_openapi::api::core::v1::Pod;
47use kube::Resource;
48use kube::ResourceExt;
49use kube::api::{Api, AttachParams, Patch, PatchParams};
50use kube::runtime::controller::Action;
51use kube::runtime::events::{Event, EventType};
52use kube::runtime::finalizer::{Event as Finalizer, finalizer};
53use serde::{Deserialize, Serialize};
54use tokio::time::{Duration, sleep};
55use tracing::{Span, debug, field, info, instrument, trace, warn};
56
57const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
58const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
59const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
60const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
61const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
62const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
63const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
64
65pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
66const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
67pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
68
69const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
70
71static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
72    BTreeMap::from([
73        (NAME_LABEL.to_string(), "kanidm".to_string()),
74        (
75            MANAGED_BY_LABEL.to_string(),
76            format!("kaniop-{CONTROLLER_ID}"),
77        ),
78    ])
79});
80
81async fn restart_statefulsets_after_replica_secret_update(
82    ctx: Arc<Context>,
83    sts_api: &Api<StatefulSet>,
84    namespace: &str,
85    statefulset_names: &[String],
86    sequential: bool,
87) -> Result<()> {
88    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
89    // Pods must restart after cert secret changes or replication keeps stale TLS material.
90    let restart_futures = statefulset_names
91        .iter()
92        .map(|sts_name| sts_api.restart(sts_name));
93
94    if sequential {
95        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
96        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
97            return Err(Error::kube_error(
98                "restart",
99                "StatefulSet",
100                namespace,
101                "replica certificate secret update statefulsets",
102                first_err,
103            ));
104        }
105    } else {
106        let results: Vec<_> = join_all(restart_futures).await;
107        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
108            return Err(Error::kube_error(
109                "restart",
110                "StatefulSet",
111                namespace,
112                "replica certificate secret update statefulsets",
113                first_err,
114            ));
115        }
116    }
117
118    let rollout_futures = statefulset_names
119        .iter()
120        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
121    try_join_all(rollout_futures).await?;
122
123    Ok(())
124}
125
126pub async fn reconcile_admins_secret(
127    kanidm: Arc<Kanidm>,
128    ctx: Arc<Context>,
129    status: &KanidmStatus,
130) -> Result<()> {
131    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
132        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
133        kanidm.patch(&ctx, admins_secret).await?;
134    }
135    Ok(())
136}
137
138pub async fn reconcile_replication_secrets(
139    kanidm: Arc<Kanidm>,
140    ctx: Arc<Context>,
141    status: &KanidmStatus,
142) -> Result<()> {
143    let expected_secret_names = if kanidm.is_replication_enabled() {
144        status
145            .replica_statuses
146            .iter()
147            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
148            .collect::<Vec<_>>()
149    } else {
150        Vec::new()
151    };
152    let deprecated_secrets = ctx
153        .stores
154        .secret_store
155        .state()
156        .into_iter()
157        .filter(|secret| {
158            secret.namespace() == kanidm.namespace()
159                && kanidm.admins_secret_name() != secret.name_any()
160                && !expected_secret_names.contains(&secret.name_any())
161                && secret
162                    .metadata
163                    .labels
164                    .as_ref()
165                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
166        })
167        .collect::<Vec<_>>();
168    let secret_delete_future = deprecated_secrets
169        .iter()
170        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
171        .collect::<TryJoinAll<_>>();
172    try_join!(secret_delete_future)?;
173
174    if kanidm.is_replication_enabled() {
175        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
176        let statefulset_names = status
177            .replica_statuses
178            .iter()
179            .map(|rs| rs.statefulset_name.clone())
180            .collect::<BTreeSet<_>>()
181            .into_iter()
182            .collect::<Vec<_>>();
183
184        let has_pending = status
185            .replica_statuses
186            .iter()
187            .any(|rs| rs.state == KanidmReplicaState::Pending);
188
189        let sts_api =
190            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
191
192        if has_pending {
193            stream::iter(
194                status
195                    .replica_statuses
196                    .iter()
197                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
198            )
199            .then(|rs| async {
200                let secret = kanidm
201                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
202                    .await?;
203                kanidm.patch(&ctx, secret).await
204            })
205            .collect::<Vec<_>>()
206            .await
207            .into_iter()
208            .collect::<Result<Vec<_>>>()?;
209
210            let restart_futures = status
211                .replica_statuses
212                .iter()
213                .map(|rs| sts_api.restart(&rs.statefulset_name));
214
215            if has_single_replica {
216                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
217                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
218                    return Err(Error::kube_error(
219                        "restart",
220                        "StatefulSet",
221                        kanidm.get_namespace(),
222                        "pending replica statefulsets",
223                        first_err,
224                    ));
225                }
226            } else {
227                let results: Vec<_> = join_all(restart_futures).await;
228                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
229                    return Err(Error::kube_error(
230                        "restart",
231                        "StatefulSet",
232                        kanidm.get_namespace(),
233                        "pending replica statefulsets",
234                        first_err,
235                    ));
236                }
237            }
238        }
239
240        let has_certificate_host_invalid = status
241            .replica_statuses
242            .iter()
243            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
244
245        let has_certificate_expiring = status
246            .replica_statuses
247            .iter()
248            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
249
250        let has_non_ready_replicas = status
251            .replica_statuses
252            .iter()
253            .any(|rs| rs.state != KanidmReplicaState::Ready);
254
255        let host_invalid_replicas: Vec<_> = status
256            .replica_statuses
257            .iter()
258            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
259            .collect();
260
261        let cert_expiring_replicas: Vec<_> = status
262            .replica_statuses
263            .iter()
264            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
265            .collect();
266
267        if !host_invalid_replicas.is_empty() {
268            let restart_futures = host_invalid_replicas
269                .iter()
270                .map(|rs| sts_api.restart(&rs.statefulset_name));
271
272            if has_single_replica {
273                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
274                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
275                    return Err(Error::kube_error(
276                        "restart",
277                        "StatefulSet",
278                        kanidm.get_namespace(),
279                        "certificate host invalid statefulsets",
280                        first_err,
281                    ));
282                }
283            } else {
284                let results: Vec<_> = join_all(restart_futures).await;
285                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
286                    return Err(Error::kube_error(
287                        "restart",
288                        "StatefulSet",
289                        kanidm.get_namespace(),
290                        "certificate host invalid statefulsets",
291                        first_err,
292                    ));
293                }
294            }
295
296            let restarted_sts_names: Vec<_> = host_invalid_replicas
297                .iter()
298                .map(|rs| rs.statefulset_name.clone())
299                .collect::<BTreeSet<_>>()
300                .into_iter()
301                .collect();
302
303            let namespace = kanidm.get_namespace();
304            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
305                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
306            });
307            try_join_all(rollout_futures).await?;
308
309            let renew_futures = host_invalid_replicas
310                .iter()
311                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
312            let results: Vec<_> = join_all(renew_futures).await;
313            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
314                return Err(first_err);
315            }
316
317            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
318
319            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
320                let cert = show_replica_cert_with_retries(
321                    &kanidm,
322                    ctx.clone(),
323                    &rs.pod_name,
324                    CERT_SHOW_RETRY_ATTEMPTS,
325                )
326                .await?;
327                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
328                kanidm.patch(&ctx, secret.clone()).await
329            });
330            try_join_all(cert_futures).await?;
331
332            restart_statefulsets_after_replica_secret_update(
333                ctx.clone(),
334                &sts_api,
335                &kanidm.get_namespace(),
336                &statefulset_names,
337                has_single_replica,
338            )
339            .await?;
340
341            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
342        }
343
344        if !cert_expiring_replicas.is_empty() {
345            let renew_futures = cert_expiring_replicas
346                .iter()
347                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
348            let results: Vec<_> = join_all(renew_futures).await;
349            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
350                return Err(first_err);
351            }
352
353            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
354
355            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
356                let cert = show_replica_cert_with_retries(
357                    &kanidm,
358                    ctx.clone(),
359                    &rs.pod_name,
360                    CERT_SHOW_RETRY_ATTEMPTS,
361                )
362                .await?;
363                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
364                kanidm.patch(&ctx, secret.clone()).await
365            });
366            try_join_all(cert_futures).await?;
367
368            restart_statefulsets_after_replica_secret_update(
369                ctx.clone(),
370                &sts_api,
371                &kanidm.get_namespace(),
372                &statefulset_names,
373                has_single_replica,
374            )
375            .await?;
376
377            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
378        }
379
380        if has_non_ready_replicas
381            && !has_pending
382            && !has_certificate_host_invalid
383            && !has_certificate_expiring
384        {
385            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
386
387            let restart_futures = status
388                .replica_statuses
389                .iter()
390                .map(|rs| sts_api.restart(&rs.statefulset_name));
391
392            if has_single_replica {
393                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
394                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
395                    return Err(Error::kube_error(
396                        "restart",
397                        "StatefulSet",
398                        kanidm.get_namespace(),
399                        "replica statefulsets",
400                        first_err,
401                    ));
402                }
403            } else {
404                let results: Vec<_> = join_all(restart_futures).await;
405                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
406                    return Err(Error::kube_error(
407                        "restart",
408                        "StatefulSet",
409                        kanidm.get_namespace(),
410                        "replica statefulsets",
411                        first_err,
412                    ));
413                }
414            }
415        }
416    }
417
418    Ok(())
419}
420
421#[instrument(skip(ctx, kanidm))]
422pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
423    let trace_id = telemetry::get_trace_id();
424    Span::current().record("trace_id", field::display(&trace_id));
425    let _timer = ctx
426        .kaniop_ctx
427        .metrics
428        .reconcile_count_and_measure(&trace_id);
429    info!(msg = "reconciling Kanidm");
430
431    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
432        debug!(msg = "failed to reconcile status", %e);
433        ctx.kaniop_ctx.metrics.status_update_errors_inc();
434        e
435    })?;
436    let kanidm_api: Api<Kanidm> =
437        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
438    finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
439        match event {
440            Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
441            Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
442        }
443    })
444    .await
445    .map_err(|e| {
446        Error::FinalizerError(
447            "failed on kanidm account finalizer".to_string(),
448            Box::new(e),
449        )
450    })
451}
452
453async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
454    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
455    let replication_secret_futures =
456        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
457
458    let sts_to_delete = {
459        let expected_sts_names = kanidm
460            .spec
461            .replica_groups
462            .iter()
463            .map(|rg| kanidm.statefulset_name(&rg.name))
464            .collect::<Vec<_>>();
465        ctx.stores
466            .stateful_set_store
467            .state()
468            .into_iter()
469            .filter(|sts| {
470                sts.namespace() == kanidm.namespace()
471                    && sts
472                        .metadata
473                        .labels
474                        .as_ref()
475                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
476                    && !expected_sts_names.contains(&sts.name_any())
477            })
478            .collect::<Vec<_>>()
479    };
480    let sts_delete_futures = sts_to_delete
481        .iter()
482        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
483        .collect::<TryJoinAll<_>>();
484
485    let sts_futures = match kanidm.is_updatable(&status) {
486        true => kanidm
487            .spec
488            .replica_groups
489            .iter()
490            .map(|rg| {
491                let sts = kanidm.create_statefulset(rg)?;
492                Ok(kanidm.patch(&ctx, sts))
493            })
494            .collect::<Result<TryJoinAll<_>, _>>()?,
495        false => {
496            let note = match status.version.as_ref() {
497                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
498                    format!(
499                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
500                        v.image_tag,
501                        crate::version::KANIDM_CLIENT_VERSION
502                    )
503                }
504                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
505            };
506            let _ignore_error = ctx
507                .kaniop_ctx
508                .recorder
509                .publish(
510                    &Event {
511                        type_: EventType::Warning,
512                        reason: "UpgradeBlocked".to_string(),
513                        note: Some(note),
514                        action: "ReconcileStatefulSet".to_string(),
515                        secondary: None,
516                    },
517                    &kanidm.object_ref(&()),
518                )
519                .await
520                .map_err(|e| {
521                    warn!(msg = "failed to publish KanidmError event", %e);
522                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
523                });
524            try_join_all([])
525        }
526    };
527    let service_future = kanidm.patch(&ctx, kanidm.create_service());
528
529    let deprecated_rg_svcs = {
530        let expected_rg_svcs_names = kanidm
531            .spec
532            .replica_groups
533            .iter()
534            .filter(|rg| rg.services.is_some())
535            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
536            .collect::<Vec<_>>();
537        ctx.stores
538            .service_store
539            .state()
540            .into_iter()
541            .filter(|svc| {
542                let labels = svc.metadata.labels.as_ref();
543                svc.namespace() == kanidm.namespace()
544                    && !expected_rg_svcs_names.contains(&svc.name_any())
545                    && labels.is_some_and(|l| {
546                        l.contains_key(REPLICA_LABEL)
547                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
548                    })
549            })
550            .collect::<Vec<_>>()
551    };
552    let rg_svcs_delete_futures = deprecated_rg_svcs
553        .iter()
554        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
555        .collect::<TryJoinAll<_>>();
556
557    let rg_services_futures = kanidm
558        .spec
559        .replica_groups
560        .iter()
561        .filter(|rg| rg.services.is_some())
562        .flat_map(|rg| {
563            (0..rg.replicas).map(|i| kanidm.patch(&ctx, kanidm.create_replica_group_service(rg, i)))
564        })
565        .collect::<TryJoinAll<_>>();
566
567    let deprecated_ingresses = {
568        let names = [
569            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
570            kanidm.generate_region_ingress_name(),
571        ]
572        .into_iter()
573        .flatten()
574        .collect::<Vec<_>>();
575        ctx.stores
576            .ingress_store
577            .state()
578            .into_iter()
579            .filter(|ing| {
580                ing.namespace() == kanidm.namespace()
581                    && !names.contains(&ing.name_any())
582                    && ing.metadata.labels == Some(kanidm.generate_labels())
583            })
584            .collect::<Vec<_>>()
585    };
586    let ingress_delete_futures = deprecated_ingresses
587        .iter()
588        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
589        .collect::<TryJoinAll<_>>();
590
591    let ingress_futures = kanidm
592        .create_ingress()
593        .into_iter()
594        .chain(kanidm.create_region_ingress())
595        .map(|ingress| kanidm.patch(&ctx, ingress))
596        .collect::<TryJoinAll<_>>();
597
598    let deprecated_http_routes = match &ctx.stores.http_route_store {
599        Some(store) => {
600            let expected_names = kanidm
601                .spec
602                .gateway
603                .as_ref()
604                .map(|_| vec![kanidm.name_any()])
605                .unwrap_or_default();
606            store
607                .state()
608                .into_iter()
609                .filter(|route| {
610                    route.namespace() == kanidm.namespace()
611                        && !expected_names.contains(&route.name_any())
612                        && route.metadata.labels == Some(kanidm.generate_labels())
613                })
614                .collect::<Vec<_>>()
615        }
616        None => Vec::new(),
617    };
618    let http_route_delete_futures = deprecated_http_routes
619        .iter()
620        .map(|route| kanidm.delete(&ctx, route.as_ref()))
621        .collect::<TryJoinAll<_>>();
622
623    let http_route_futures = if ctx.stores.http_route_store.is_some() {
624        kanidm
625            .create_http_route()
626            .into_iter()
627            .map(|route| kanidm.patch(&ctx, route))
628            .collect::<TryJoinAll<_>>()
629    } else {
630        try_join_all(Vec::new())
631    };
632
633    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
634        Some(store) => {
635            let expected_names = kanidm
636                .spec
637                .gateway
638                .as_ref()
639                .and_then(|g| g.backend_tls_policy.as_ref())
640                .map(|_| vec![kanidm.name_any()])
641                .unwrap_or_default();
642            store
643                .state()
644                .into_iter()
645                .filter(|policy| {
646                    policy.namespace() == kanidm.namespace()
647                        && !expected_names.contains(&policy.name_any())
648                        && policy.metadata.labels == Some(kanidm.generate_labels())
649                })
650                .collect::<Vec<_>>()
651        }
652        None => Vec::new(),
653    };
654    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
655        .iter()
656        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
657        .collect::<TryJoinAll<_>>();
658
659    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
660        kanidm
661            .create_backend_tls_policy()
662            .into_iter()
663            .map(|policy| kanidm.patch(&ctx, policy))
664            .collect::<TryJoinAll<_>>()
665    } else {
666        try_join_all(Vec::new())
667    };
668
669    try_join!(
670        sts_delete_futures,
671        admin_secret_future,
672        replication_secret_futures,
673        sts_futures,
674        service_future,
675        rg_svcs_delete_futures,
676        rg_services_futures,
677        ingress_delete_futures,
678        ingress_futures,
679        http_route_delete_futures,
680        http_route_futures,
681        backend_tls_policy_delete_futures,
682        backend_tls_policy_futures
683    )?;
684
685    if is_kanidm_available(status.clone()) {
686        let namespace = kanidm.namespace().unwrap();
687        let name = kanidm.name_any();
688        let system_client = crate::controller::kanidm::KanidmClients::create_client(
689            &namespace,
690            &name,
691            crate::controller::kanidm::KanidmUser::Admin,
692            ctx.kaniop_ctx.client.clone(),
693        )
694        .await?;
695        reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
696
697        let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
698            &namespace,
699            &name,
700            crate::controller::kanidm::KanidmUser::IdmAdmin,
701            ctx.kaniop_ctx.client.clone(),
702        )
703        .await?;
704        let mail_sender_status =
705            reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
706        let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
707        if mail_sender_status != current_mail_sender_status {
708            let kanidm_api: Api<Kanidm> =
709                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
710            let status_patch = serde_json::json!({
711                "status": {
712                    "mailSender": mail_sender_status
713                }
714            });
715            kanidm_api
716                .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
717                .await
718                .map_err(|e| {
719                    Error::KubeError(
720                        format!("failed to patch Kanidm/status {namespace}/{name}"),
721                        Box::new(e),
722                    )
723                })?;
724        }
725    }
726
727    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
728}
729
730async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
731    debug!(msg = "cleanup");
732
733    cleanup_mail_sender_resources(&kanidm, &ctx).await?;
734
735    let namespace = kanidm.namespace().unwrap();
736    let name = kanidm.name_any();
737
738    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
739        &namespace,
740        &name,
741        crate::controller::kanidm::KanidmUser::IdmAdmin,
742        ctx.kaniop_ctx.client.clone(),
743    )
744    .await
745    {
746        cleanup_mail_sender_in_kanidm(&kanidm_client, &name).await?;
747    }
748
749    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
750    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
751}
752
753async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
754    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
755    let start = std::time::Instant::now();
756    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
757    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
758
759    loop {
760        let sts = sts_api.get(sts_name).await.map_err(|e| {
761            Error::KubeError(
762                format!("failed to get StatefulSet {namespace}/{sts_name}"),
763                Box::new(e),
764            )
765        })?;
766
767        if sts.metadata.deletion_timestamp.is_some() {
768            return Err(Error::ReceiveOutput(format!(
769                "StatefulSet {namespace}/{sts_name} is being deleted"
770            )));
771        }
772
773        let spec = sts.spec.as_ref().ok_or_else(|| {
774            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
775        })?;
776
777        let status = match sts.status {
778            Some(ref s) => s,
779            None => {
780                if start.elapsed() >= timeout {
781                    return Err(Error::ReceiveOutput(format!(
782                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
783                    )));
784                }
785                sleep(poll_interval).await;
786                continue;
787            }
788        };
789
790        let replicas = spec.replicas.unwrap_or(0);
791
792        if replicas == 0 {
793            return Ok(());
794        }
795
796        let updated_replicas = status.updated_replicas.unwrap_or(0);
797        let current_revision = status.current_revision.as_deref().unwrap_or("");
798        let update_revision = status.update_revision.as_deref().unwrap_or("");
799        let ready_replicas = status.ready_replicas.unwrap_or(0);
800
801        if updated_replicas == replicas
802            && current_revision == update_revision
803            && ready_replicas == replicas
804        {
805            return Ok(());
806        }
807
808        if start.elapsed() >= timeout {
809            return Err(Error::ReceiveOutput(format!(
810                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
811            )));
812        }
813
814        sleep(poll_interval).await;
815    }
816}
817
818async fn show_replica_cert_with_retries(
819    kanidm: &Kanidm,
820    ctx: Arc<Context>,
821    pod_name: &str,
822    max_attempts: u32,
823) -> std::result::Result<String, Error> {
824    let mut last_error = None;
825    for attempt in 0..max_attempts {
826        if attempt > 0 {
827            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
828        }
829        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
830            Ok(cert) => return Ok(cert),
831            Err(e) => {
832                info!(
833                    msg = format!(
834                        "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
835                        attempt + 1,
836                        max_attempts
837                    )
838                );
839                last_error = Some(e);
840            }
841        }
842    }
843    Err(last_error.unwrap_or_else(|| {
844        Error::ReceiveOutput(format!(
845            "failed to get certificate for {pod_name} after {max_attempts} attempts"
846        ))
847    }))
848}
849
850impl Kanidm {
851    // Convenience methods that handle context and operator name
852    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
853    where
854        K: Resource<Scope = NamespaceResourceScope>
855            + Serialize
856            + Clone
857            + std::fmt::Debug
858            + for<'de> Deserialize<'de>,
859        <K as kube::Resource>::DynamicType: Default,
860        <K as Resource>::Scope: std::marker::Sized,
861    {
862        self.kube_delete(
863            ctx.kaniop_ctx.client.clone(),
864            &ctx.kaniop_ctx.metrics,
865            resource,
866        )
867        .await
868    }
869
870    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
871    where
872        K: Resource<Scope = NamespaceResourceScope>
873            + Serialize
874            + Clone
875            + std::fmt::Debug
876            + for<'de> Deserialize<'de>,
877        <K as kube::Resource>::DynamicType: Default,
878        <K as Resource>::Scope: std::marker::Sized,
879    {
880        self.kube_patch(
881            ctx.kaniop_ctx.client.clone(),
882            &ctx.kaniop_ctx.metrics,
883            resource,
884            KANIDM_OPERATOR_NAME,
885        )
886        .await
887    }
888
889    #[inline]
890    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
891        LABELS
892            .clone()
893            .into_iter()
894            .chain([
895                (INSTANCE_LABEL.to_string(), self.name_any()),
896                (CLUSTER_LABEL.to_string(), self.name_any()),
897            ])
898            .collect()
899    }
900
901    #[inline]
902    fn generate_labels(&self) -> BTreeMap<String, String> {
903        self.generate_resource_labels()
904    }
905
906    #[inline]
907    fn get_tls_secret_name(&self) -> String {
908        format!("{}-tls", self.name_any())
909    }
910
911    #[inline]
912    fn get_namespace(&self) -> String {
913        // safe unwrap: Kanidm is namespaced scoped
914        self.namespace().unwrap()
915    }
916
917    #[inline]
918    fn is_replication_enabled(&self) -> bool {
919        self.spec.replica_groups.len() > 1
920            || self
921                .spec
922                .replica_groups
923                .first()
924                .is_some_and(|rg| rg.replicas > 1)
925            || !self.spec.external_replication_nodes.is_empty()
926    }
927
928    #[inline]
929    fn is_updatable(&self, status: &KanidmStatus) -> bool {
930        status
931            .version
932            .as_ref()
933            .map(|v| {
934                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
935                    return false;
936                }
937                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
938                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
939                match v.upgrade_check_result {
940                    KanidmUpgradeCheckResult::Passed => {
941                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
942                            minor <= v_minor + 1
943                        } else {
944                            true
945                        }
946                    }
947                    KanidmUpgradeCheckResult::Failed => {
948                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
949                            versions
950                        {
951                            major == v_major && minor == v_minor && patch >= v_patch
952                        } else {
953                            v.image_tag == current_tag
954                        }
955                    }
956                }
957            })
958            .unwrap_or(true)
959    }
960
961    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
962        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
963        let pod = pod_api.get(pod_name).await.map_err(|e| {
964            Error::KubeError(
965                format!("failed to get pod {namespace}/{pod_name}"),
966                Box::new(e),
967            )
968        })?;
969        Ok(pod
970            .status
971            .as_ref()
972            .and_then(|s| s.conditions.as_ref())
973            .is_some_and(|conditions| {
974                conditions
975                    .iter()
976                    .any(|c| c.type_ == "Ready" && c.status == "True")
977            }))
978    }
979
980    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
981        let start = std::time::Instant::now();
982        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
983        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
984
985        loop {
986            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
987                Ok(true) => return Ok(()),
988                Ok(false) => {}
989                Err(e) => {
990                    if !e.is_retryable() {
991                        return Err(e);
992                    }
993                    trace!(msg = "pod readiness check failed, retrying", %e);
994                }
995            }
996
997            if start.elapsed() >= timeout {
998                return Err(Error::ReceiveOutput(format!(
999                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1000                )));
1001            }
1002
1003            sleep(poll_interval).await;
1004        }
1005    }
1006
1007    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1008    where
1009        I: IntoIterator<Item = T> + Debug,
1010        T: Into<String>,
1011    {
1012        let namespace = &self.get_namespace();
1013        trace!(
1014            msg = "pod exec",
1015            resource.name = &pod_name,
1016            resource.namespace = &namespace,
1017            ?command
1018        );
1019        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1020        let attached = pod
1021            .exec(
1022                pod_name,
1023                command,
1024                &AttachParams::default().container("kanidm"),
1025            )
1026            .await
1027            .map_err(|e| {
1028                Error::KubeError(
1029                    format!("failed to exec pod {namespace}/{pod_name}"),
1030                    Box::new(e),
1031                )
1032            })?;
1033        get_output(attached).await
1034    }
1035
1036    async fn exec_with_wait<I, T>(
1037        &self,
1038        ctx: Arc<Context>,
1039        pod_name: &str,
1040        command: I,
1041    ) -> Result<String>
1042    where
1043        I: IntoIterator<Item = T> + Debug,
1044        T: Into<String>,
1045    {
1046        let namespace = &self.get_namespace();
1047        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1048        self.exec(ctx, pod_name, command).await
1049    }
1050
1051    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1052    where
1053        I: IntoIterator<Item = T> + Debug,
1054        T: Into<String>,
1055    {
1056        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1057        let first_replica_group = self
1058            .spec
1059            .replica_groups
1060            .first()
1061            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1062        let sts_name = self.statefulset_name(&first_replica_group.name);
1063        let pod_name = format!("{sts_name}-0");
1064        self.exec(ctx, &pod_name, command).await
1065    }
1066
1067    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1068    where
1069        I: IntoIterator<Item = T> + Debug,
1070        T: Into<String>,
1071    {
1072        let first_replica_group = self
1073            .spec
1074            .replica_groups
1075            .first()
1076            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1077        let sts_name = self.statefulset_name(&first_replica_group.name);
1078        let pod_name = format!("{sts_name}-0");
1079        self.exec_with_wait(ctx, &pod_name, command).await
1080    }
1081}
1082
1083#[cfg(test)]
1084mod test {
1085    use super::statefulset::StatefulSetExt;
1086    use super::{Kanidm, reconcile_kanidm};
1087
1088    use crate::controller::State;
1089    use crate::kanidm::controller::context::{Context, Stores};
1090    use crate::kanidm::crd::KanidmStatus;
1091
1092    use kaniop_k8s_util::error::Result;
1093
1094    use std::sync::Arc;
1095
1096    use http::{Request, Response};
1097    use k8s_openapi::api::apps::v1::StatefulSet;
1098    use k8s_openapi::api::core::v1::Service;
1099    use k8s_openapi::api::networking::v1::Ingress;
1100    use kube::runtime::reflector::store::Writer;
1101    use kube::{Client, Resource, ResourceExt, client::Body};
1102    use opentelemetry::metrics::MeterProvider;
1103    use serde_json::json;
1104
1105    impl Kanidm {
1106        /// A normal test kanidm with a given status
1107        pub fn test() -> Self {
1108            let mut e = Kanidm::new(
1109                "test",
1110                serde_json::from_value(json!({
1111                    "domain": "idm.example.com",
1112                    "replicaGroups": [
1113                        {
1114                            "name": "default",
1115                            "replicas": 1
1116                        }
1117                    ]
1118                }))
1119                .unwrap(),
1120            );
1121            e.meta_mut().namespace = Some("default".into());
1122            // Pre-populate finalizer so reconcile callback runs immediately
1123            // (skips the "add finalizer + wait for next reconcile" step)
1124            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1125            e
1126        }
1127
1128        pub fn with_ingress(mut self) -> Self {
1129            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1130            self
1131        }
1132
1133        /// Modify kanidm replicas
1134        pub fn with_replicas(mut self, replicas: i32) -> Self {
1135            self.spec.replica_groups[0].replicas = replicas;
1136            self
1137        }
1138
1139        /// Modify kanidm to set a deletion timestamp
1140        pub fn needs_delete(mut self) -> Self {
1141            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1142            use k8s_openapi::jiff::Timestamp;
1143            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1144            self.meta_mut().deletion_timestamp = Some(Time(now));
1145            self
1146        }
1147
1148        /// Modify a kanidm to have an expected status
1149        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1150            self.status = Some(status);
1151            self
1152        }
1153    }
1154
1155    // We wrap tower_test::mock::Handle
1156    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1157    pub struct ApiServerVerifier(ApiServerHandle);
1158
1159    /// Scenarios we test for in ApiServerVerifier
1160    pub enum Scenario {
1161        Create(Kanidm),
1162        CreateWithTwoReplicas(Kanidm),
1163        CreateWithIngress(Kanidm),
1164        CreateWithIngressWithTwoReplicas(Kanidm),
1165    }
1166
1167    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1168        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1169            .await
1170            .expect("timeout on mock apiserver")
1171            .expect("scenario succeeded")
1172    }
1173
1174    impl ApiServerVerifier {
1175        /// Tests only get to run specific scenarios that has matching handlers
1176        ///
1177        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1178        ///
1179        /// NB: If the controller is making more calls than we are handling in the scenario,
1180        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1181        ///
1182        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1183        /// scenario runs to completion (i.e. all expected calls were responded to),
1184        /// using the timeout to catch missing api calls to Kubernetes.
1185        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1186            tokio::spawn(async move {
1187                // moving self => one scenario per test
1188                match scenario {
1189                    Scenario::Create(kanidm) => {
1190                        self.handle_kanidm_get(kanidm.clone())
1191                            .await
1192                            .unwrap()
1193                            .handle_kanidm_status_patch(kanidm.clone())
1194                            .await
1195                            .unwrap()
1196                            .handle_statefulset_patch(kanidm.clone())
1197                            .await
1198                            .unwrap()
1199                            .handle_service_patch(kanidm.clone())
1200                            .await
1201                    }
1202                    Scenario::CreateWithTwoReplicas(kanidm) => {
1203                        self.handle_kanidm_get(kanidm.clone())
1204                            .await
1205                            .unwrap()
1206                            .handle_kanidm_status_patch(kanidm.clone())
1207                            .await
1208                            .unwrap()
1209                            .handle_statefulset_patch(kanidm.clone())
1210                            .await
1211                            .unwrap()
1212                            .handle_service_patch(kanidm.clone())
1213                            .await
1214                    }
1215                    Scenario::CreateWithIngress(kanidm) => {
1216                        self.handle_kanidm_get(kanidm.clone())
1217                            .await
1218                            .unwrap()
1219                            .handle_kanidm_status_patch(kanidm.clone())
1220                            .await
1221                            .unwrap()
1222                            .handle_statefulset_patch(kanidm.clone())
1223                            .await
1224                            .unwrap()
1225                            .handle_service_patch(kanidm.clone())
1226                            .await
1227                            .unwrap()
1228                            .handle_ingress_patch(kanidm.clone())
1229                            .await
1230                    }
1231                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1232                        self.handle_kanidm_get(kanidm.clone())
1233                            .await
1234                            .unwrap()
1235                            .handle_kanidm_status_patch(kanidm.clone())
1236                            .await
1237                            .unwrap()
1238                            .handle_statefulset_patch(kanidm.clone())
1239                            .await
1240                            .unwrap()
1241                            .handle_service_patch(kanidm.clone())
1242                            .await
1243                            .unwrap()
1244                            .handle_ingress_patch(kanidm.clone())
1245                            .await
1246                    }
1247                }
1248                .expect("scenario completed without errors");
1249            })
1250        }
1251
1252        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1253            let (request, send) = self.0.next_request().await.expect("service not called");
1254            assert_eq!(request.method(), http::Method::GET);
1255            let expected_uri_prefix = format!(
1256                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1257                kanidm.name_any()
1258            );
1259            let uri = request.uri().to_string();
1260            assert!(
1261                uri.starts_with(&expected_uri_prefix),
1262                "expected uri to start with {}, got {}",
1263                expected_uri_prefix,
1264                uri
1265            );
1266            let response = serde_json::to_vec(&kanidm).unwrap();
1267            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1268            Ok(self)
1269        }
1270
1271        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1272            let (request, send) = self.0.next_request().await.expect("service not called");
1273            assert_eq!(request.method(), http::Method::PATCH);
1274            assert_eq!(
1275                request.uri().to_string(),
1276                format!(
1277                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1278                    kanidm.name_any()
1279                )
1280            );
1281
1282            let req_body = request.into_body().collect_bytes().await.unwrap();
1283            let json: serde_json::Value =
1284                serde_json::from_slice(&req_body).expect("patch object is json");
1285            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1286                .expect("valid kanidm status");
1287            let mut kanidm = Kanidm::test();
1288            kanidm.status = Some(status.clone());
1289            let response = serde_json::to_vec(&kanidm).unwrap();
1290            // pass through kanidm "patch accepted"
1291            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1292            Ok(self)
1293        }
1294
1295        async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1296            for rg in kanidm.spec.replica_groups.iter() {
1297                let (request, send) = self.0.next_request().await.expect("service not called");
1298                assert_eq!(request.method(), http::Method::PATCH);
1299                assert_eq!(
1300                    request.uri().to_string(),
1301                    format!(
1302                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1303                        kanidm.statefulset_name(&rg.name)
1304                    )
1305                );
1306                let req_body = request.into_body().collect_bytes().await.unwrap();
1307                let json: serde_json::Value =
1308                    serde_json::from_slice(&req_body).expect("patch object is json");
1309                let statefulset: StatefulSet =
1310                    serde_json::from_value(json).expect("valid statefulset");
1311                assert_eq!(
1312                    statefulset.clone().spec.unwrap().replicas.unwrap(),
1313                    rg.replicas
1314                );
1315                let response = serde_json::to_vec(&statefulset).unwrap();
1316                // pass through kanidm "patch accepted"
1317                send.send_response(Response::builder().body(Body::from(response)).unwrap());
1318            }
1319
1320            Ok(self)
1321        }
1322
1323        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1324            let (request, send) = self.0.next_request().await.expect("service not called");
1325            assert_eq!(request.method(), http::Method::PATCH);
1326            assert_eq!(
1327                request.uri().to_string(),
1328                format!(
1329                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1330                    kanidm.name_any()
1331                )
1332            );
1333
1334            let req_body = request.into_body().collect_bytes().await.unwrap();
1335            let json: serde_json::Value =
1336                serde_json::from_slice(&req_body).expect("patch object is json");
1337            let service: Service = serde_json::from_value(json).expect("valid service");
1338            let response = serde_json::to_vec(&service).unwrap();
1339            // pass through kanidm "patch accepted"
1340            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1341            Ok(self)
1342        }
1343
1344        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1345            let (request, send) = self.0.next_request().await.expect("service not called");
1346            assert_eq!(request.method(), http::Method::PATCH);
1347            assert_eq!(
1348                request.uri().to_string(),
1349                format!(
1350                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1351                    kanidm.name_any()
1352                )
1353            );
1354
1355            let req_body = request.into_body().collect_bytes().await.unwrap();
1356            let json: serde_json::Value =
1357                serde_json::from_slice(&req_body).expect("patch object is json");
1358            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1359            let response = serde_json::to_vec(&ingress).unwrap();
1360            // pass through kanidm "patch accepted"
1361            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1362            Ok(self)
1363        }
1364    }
1365
1366    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1367        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1368        let mock_client = Client::new(mock_service, "default");
1369        let stores = Stores {
1370            stateful_set_store: Writer::default().as_reader(),
1371            service_store: Writer::default().as_reader(),
1372            ingress_store: Writer::default().as_reader(),
1373            secret_store: Writer::default().as_reader(),
1374            http_route_store: Some(Writer::default().as_reader()),
1375            backend_tls_policy_store: Some(Writer::default().as_reader()),
1376            deployment_store: Writer::default().as_reader(),
1377            config_map_store: Writer::default().as_reader(),
1378        };
1379        let controller_id = "test";
1380
1381        // Create a test meter for metrics
1382        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
1383        let meter = provider.meter("test");
1384        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
1385
1386        let state = State::new(
1387            metrics,
1388            Writer::default().as_reader(),
1389            Writer::default().as_reader(),
1390            None,
1391        );
1392        let ctx = Arc::new(Context::new(
1393            state.to_context(mock_client, controller_id),
1394            stores,
1395        ));
1396        (ctx, ApiServerVerifier(handle))
1397    }
1398
1399    #[tokio::test]
1400    async fn kanidm_create() {
1401        let (testctx, fakeserver) = get_test_context();
1402        let kanidm = Kanidm::test();
1403        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
1404        reconcile_kanidm(Arc::new(kanidm), testctx)
1405            .await
1406            .expect("reconciler");
1407        timeout_after_1s(mocksrv).await;
1408    }
1409
1410    #[tokio::test]
1411    async fn kanidm_create_with_two_replicas() {
1412        let (testctx, fakeserver) = get_test_context();
1413        let kanidm = Kanidm::test().with_replicas(2);
1414        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
1415        reconcile_kanidm(Arc::new(kanidm), testctx)
1416            .await
1417            .expect("reconciler");
1418        timeout_after_1s(mocksrv).await;
1419    }
1420
1421    #[tokio::test]
1422    async fn kanidm_create_with_ingress() {
1423        let (testctx, fakeserver) = get_test_context();
1424        let kanidm = Kanidm::test().with_ingress();
1425        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
1426        reconcile_kanidm(Arc::new(kanidm), testctx)
1427            .await
1428            .expect("reconciler");
1429        timeout_after_1s(mocksrv).await;
1430    }
1431
1432    #[tokio::test]
1433    async fn kanidm_create_with_ingress_with_two_replicas() {
1434        let (testctx, fakeserver) = get_test_context();
1435        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
1436        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
1437        reconcile_kanidm(Arc::new(kanidm), testctx)
1438            .await
1439            .expect("reconciler");
1440        timeout_after_1s(mocksrv).await;
1441    }
1442}