Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::StatefulSetExt;
21use self::status::StatusExt;
22use self::status::{is_kanidm_available, is_kanidm_initialized};
23
24use crate::controller::context::KubeOperations;
25use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
26use crate::kanidm::crd::{
27    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
28};
29use crate::kanidm::reconcile::statefulset::REPLICA_LABEL;
30use crate::telemetry;
31
32use kaniop_k8s_util::client::get_output;
33use kaniop_k8s_util::error::{Error, Result};
34use kaniop_k8s_util::parse::parse_semver;
35use kaniop_k8s_util::resources::get_image_tag;
36
37use std::collections::{BTreeMap, BTreeSet};
38use std::fmt::Debug;
39use std::sync::{Arc, LazyLock};
40
41use futures::future::{TryJoinAll, join_all, try_join_all};
42use futures::stream::{self, StreamExt};
43use futures::try_join;
44use k8s_openapi::NamespaceResourceScope;
45use k8s_openapi::api::apps::v1::StatefulSet;
46use k8s_openapi::api::core::v1::Pod;
47use kube::Resource;
48use kube::ResourceExt;
49use kube::api::{Api, AttachParams, Patch, PatchParams};
50use kube::runtime::controller::Action;
51use kube::runtime::events::{Event, EventType};
52use kube::runtime::finalizer::{Event as Finalizer, finalizer};
53use serde::{Deserialize, Serialize};
54use tokio::time::{Duration, sleep};
55use tracing::{Span, debug, field, info, instrument, trace, warn};
56
57const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
58const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
59const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
60const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
61const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
62const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
63const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
64
65pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
66const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
67pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
68
69const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
70
71static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
72    BTreeMap::from([
73        (NAME_LABEL.to_string(), "kanidm".to_string()),
74        (
75            MANAGED_BY_LABEL.to_string(),
76            format!("kaniop-{CONTROLLER_ID}"),
77        ),
78    ])
79});
80
81async fn restart_statefulsets_after_replica_secret_update(
82    ctx: Arc<Context>,
83    sts_api: &Api<StatefulSet>,
84    namespace: &str,
85    statefulset_names: &[String],
86    sequential: bool,
87) -> Result<()> {
88    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
89    // Pods must restart after cert secret changes or replication keeps stale TLS material.
90    let restart_futures = statefulset_names
91        .iter()
92        .map(|sts_name| sts_api.restart(sts_name));
93
94    if sequential {
95        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
96        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
97            return Err(Error::kube_error(
98                "restart",
99                "StatefulSet",
100                namespace,
101                "replica certificate secret update statefulsets",
102                first_err,
103            ));
104        }
105    } else {
106        let results: Vec<_> = join_all(restart_futures).await;
107        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
108            return Err(Error::kube_error(
109                "restart",
110                "StatefulSet",
111                namespace,
112                "replica certificate secret update statefulsets",
113                first_err,
114            ));
115        }
116    }
117
118    let rollout_futures = statefulset_names
119        .iter()
120        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
121    try_join_all(rollout_futures).await?;
122
123    Ok(())
124}
125
126pub async fn reconcile_admins_secret(
127    kanidm: Arc<Kanidm>,
128    ctx: Arc<Context>,
129    status: &KanidmStatus,
130) -> Result<()> {
131    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
132        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
133        kanidm.patch(&ctx, admins_secret).await?;
134    }
135    Ok(())
136}
137
138pub async fn reconcile_replication_secrets(
139    kanidm: Arc<Kanidm>,
140    ctx: Arc<Context>,
141    status: &KanidmStatus,
142) -> Result<()> {
143    let expected_secret_names = if kanidm.is_replication_enabled() {
144        status
145            .replica_statuses
146            .iter()
147            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
148            .collect::<Vec<_>>()
149    } else {
150        Vec::new()
151    };
152    let deprecated_secrets = ctx
153        .stores
154        .secret_store
155        .state()
156        .into_iter()
157        .filter(|secret| {
158            secret.namespace() == kanidm.namespace()
159                && kanidm.admins_secret_name() != secret.name_any()
160                && !expected_secret_names.contains(&secret.name_any())
161                && secret
162                    .metadata
163                    .labels
164                    .as_ref()
165                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
166        })
167        .collect::<Vec<_>>();
168    let secret_delete_future = deprecated_secrets
169        .iter()
170        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
171        .collect::<TryJoinAll<_>>();
172    try_join!(secret_delete_future)?;
173
174    if kanidm.is_replication_enabled() {
175        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
176        let statefulset_names = status
177            .replica_statuses
178            .iter()
179            .map(|rs| rs.statefulset_name.clone())
180            .collect::<BTreeSet<_>>()
181            .into_iter()
182            .collect::<Vec<_>>();
183
184        let has_pending = status
185            .replica_statuses
186            .iter()
187            .any(|rs| rs.state == KanidmReplicaState::Pending);
188
189        let sts_api =
190            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
191
192        if has_pending {
193            stream::iter(
194                status
195                    .replica_statuses
196                    .iter()
197                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
198            )
199            .then(|rs| async {
200                let secret = kanidm
201                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
202                    .await?;
203                kanidm.patch(&ctx, secret).await
204            })
205            .collect::<Vec<_>>()
206            .await
207            .into_iter()
208            .collect::<Result<Vec<_>>>()?;
209
210            let restart_futures = status
211                .replica_statuses
212                .iter()
213                .map(|rs| sts_api.restart(&rs.statefulset_name));
214
215            if has_single_replica {
216                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
217                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
218                    return Err(Error::kube_error(
219                        "restart",
220                        "StatefulSet",
221                        kanidm.get_namespace(),
222                        "pending replica statefulsets",
223                        first_err,
224                    ));
225                }
226            } else {
227                let results: Vec<_> = join_all(restart_futures).await;
228                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
229                    return Err(Error::kube_error(
230                        "restart",
231                        "StatefulSet",
232                        kanidm.get_namespace(),
233                        "pending replica statefulsets",
234                        first_err,
235                    ));
236                }
237            }
238        }
239
240        let has_certificate_host_invalid = status
241            .replica_statuses
242            .iter()
243            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
244
245        let has_certificate_expiring = status
246            .replica_statuses
247            .iter()
248            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
249
250        let has_non_ready_replicas = status
251            .replica_statuses
252            .iter()
253            .any(|rs| rs.state != KanidmReplicaState::Ready);
254
255        let host_invalid_replicas: Vec<_> = status
256            .replica_statuses
257            .iter()
258            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
259            .collect();
260
261        let cert_expiring_replicas: Vec<_> = status
262            .replica_statuses
263            .iter()
264            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
265            .collect();
266
267        if !host_invalid_replicas.is_empty() {
268            let restart_futures = host_invalid_replicas
269                .iter()
270                .map(|rs| sts_api.restart(&rs.statefulset_name));
271
272            if has_single_replica {
273                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
274                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
275                    return Err(Error::kube_error(
276                        "restart",
277                        "StatefulSet",
278                        kanidm.get_namespace(),
279                        "certificate host invalid statefulsets",
280                        first_err,
281                    ));
282                }
283            } else {
284                let results: Vec<_> = join_all(restart_futures).await;
285                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
286                    return Err(Error::kube_error(
287                        "restart",
288                        "StatefulSet",
289                        kanidm.get_namespace(),
290                        "certificate host invalid statefulsets",
291                        first_err,
292                    ));
293                }
294            }
295
296            let restarted_sts_names: Vec<_> = host_invalid_replicas
297                .iter()
298                .map(|rs| rs.statefulset_name.clone())
299                .collect::<BTreeSet<_>>()
300                .into_iter()
301                .collect();
302
303            let namespace = kanidm.get_namespace();
304            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
305                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
306            });
307            try_join_all(rollout_futures).await?;
308
309            let renew_futures = host_invalid_replicas
310                .iter()
311                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
312            let results: Vec<_> = join_all(renew_futures).await;
313            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
314                return Err(first_err);
315            }
316
317            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
318
319            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
320                let cert = show_replica_cert_with_retries(
321                    &kanidm,
322                    ctx.clone(),
323                    &rs.pod_name,
324                    CERT_SHOW_RETRY_ATTEMPTS,
325                )
326                .await?;
327                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
328                kanidm.patch(&ctx, secret.clone()).await
329            });
330            try_join_all(cert_futures).await?;
331
332            restart_statefulsets_after_replica_secret_update(
333                ctx.clone(),
334                &sts_api,
335                &kanidm.get_namespace(),
336                &statefulset_names,
337                has_single_replica,
338            )
339            .await?;
340
341            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
342        }
343
344        if !cert_expiring_replicas.is_empty() {
345            let renew_futures = cert_expiring_replicas
346                .iter()
347                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
348            let results: Vec<_> = join_all(renew_futures).await;
349            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
350                return Err(first_err);
351            }
352
353            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
354
355            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
356                let cert = show_replica_cert_with_retries(
357                    &kanidm,
358                    ctx.clone(),
359                    &rs.pod_name,
360                    CERT_SHOW_RETRY_ATTEMPTS,
361                )
362                .await?;
363                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
364                kanidm.patch(&ctx, secret.clone()).await
365            });
366            try_join_all(cert_futures).await?;
367
368            restart_statefulsets_after_replica_secret_update(
369                ctx.clone(),
370                &sts_api,
371                &kanidm.get_namespace(),
372                &statefulset_names,
373                has_single_replica,
374            )
375            .await?;
376
377            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
378        }
379
380        if has_non_ready_replicas
381            && !has_pending
382            && !has_certificate_host_invalid
383            && !has_certificate_expiring
384        {
385            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
386
387            let restart_futures = status
388                .replica_statuses
389                .iter()
390                .map(|rs| sts_api.restart(&rs.statefulset_name));
391
392            if has_single_replica {
393                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
394                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
395                    return Err(Error::kube_error(
396                        "restart",
397                        "StatefulSet",
398                        kanidm.get_namespace(),
399                        "replica statefulsets",
400                        first_err,
401                    ));
402                }
403            } else {
404                let results: Vec<_> = join_all(restart_futures).await;
405                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
406                    return Err(Error::kube_error(
407                        "restart",
408                        "StatefulSet",
409                        kanidm.get_namespace(),
410                        "replica statefulsets",
411                        first_err,
412                    ));
413                }
414            }
415        }
416    }
417
418    Ok(())
419}
420
421#[instrument(skip(ctx, kanidm))]
422pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
423    let trace_id = telemetry::get_trace_id();
424    Span::current().record("trace_id", field::display(&trace_id));
425    let _timer = ctx
426        .kaniop_ctx
427        .metrics
428        .reconcile_count_and_measure(&trace_id);
429    info!(msg = "reconciling Kanidm");
430
431    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
432        debug!(msg = "failed to reconcile status", %e);
433        ctx.kaniop_ctx.metrics.status_update_errors_inc();
434        e
435    })?;
436    let kanidm_api: Api<Kanidm> =
437        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
438    finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
439        match event {
440            Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
441            Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
442        }
443    })
444    .await
445    .map_err(|e| {
446        Error::FinalizerError(
447            "failed on kanidm account finalizer".to_string(),
448            Box::new(e),
449        )
450    })
451}
452
453async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
454    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
455    let replication_secret_futures =
456        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
457
458    let sts_to_delete = {
459        let expected_sts_names = kanidm
460            .spec
461            .replica_groups
462            .iter()
463            .map(|rg| kanidm.statefulset_name(&rg.name))
464            .collect::<Vec<_>>();
465        ctx.stores
466            .stateful_set_store
467            .state()
468            .into_iter()
469            .filter(|sts| {
470                sts.namespace() == kanidm.namespace()
471                    && sts
472                        .metadata
473                        .labels
474                        .as_ref()
475                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
476                    && !expected_sts_names.contains(&sts.name_any())
477            })
478            .collect::<Vec<_>>()
479    };
480    let sts_delete_futures = sts_to_delete
481        .iter()
482        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
483        .collect::<TryJoinAll<_>>();
484
485    let sts_futures = match kanidm.is_updatable(&status) {
486        true => kanidm
487            .spec
488            .replica_groups
489            .iter()
490            .map(|rg| {
491                let sts = kanidm.create_statefulset(rg)?;
492                Ok(kanidm.patch(&ctx, sts))
493            })
494            .collect::<Result<TryJoinAll<_>, _>>()?,
495        false => {
496            let note = match status.version.as_ref() {
497                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
498                    format!(
499                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
500                        v.image_tag,
501                        crate::version::KANIDM_CLIENT_VERSION
502                    )
503                }
504                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
505            };
506            let _ignore_error = ctx
507                .kaniop_ctx
508                .recorder
509                .publish(
510                    &Event {
511                        type_: EventType::Warning,
512                        reason: "UpgradeBlocked".to_string(),
513                        note: Some(note),
514                        action: "ReconcileStatefulSet".to_string(),
515                        secondary: None,
516                    },
517                    &kanidm.object_ref(&()),
518                )
519                .await
520                .map_err(|e| {
521                    warn!(msg = "failed to publish KanidmError event", %e);
522                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
523                });
524            try_join_all([])
525        }
526    };
527    let service_future = kanidm.patch(&ctx, kanidm.create_service());
528
529    let deprecated_rg_svcs = {
530        let expected_rg_svcs_names = kanidm
531            .spec
532            .replica_groups
533            .iter()
534            .filter(|rg| rg.services.is_some())
535            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
536            .collect::<Vec<_>>();
537        ctx.stores
538            .service_store
539            .state()
540            .into_iter()
541            .filter(|svc| {
542                let labels = svc.metadata.labels.as_ref();
543                svc.namespace() == kanidm.namespace()
544                    && !expected_rg_svcs_names.contains(&svc.name_any())
545                    && labels.is_some_and(|l| {
546                        l.contains_key(REPLICA_LABEL)
547                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
548                    })
549            })
550            .collect::<Vec<_>>()
551    };
552    let rg_svcs_delete_futures = deprecated_rg_svcs
553        .iter()
554        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
555        .collect::<TryJoinAll<_>>();
556
557    let rg_services_futures = kanidm
558        .spec
559        .replica_groups
560        .iter()
561        .filter(|rg| rg.services.is_some())
562        .flat_map(|rg| {
563            (0..rg.replicas).map(|i| kanidm.patch(&ctx, kanidm.create_replica_group_service(rg, i)))
564        })
565        .collect::<TryJoinAll<_>>();
566
567    let deprecated_ingresses = {
568        let names = [
569            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
570            kanidm.generate_region_ingress_name(),
571        ]
572        .into_iter()
573        .flatten()
574        .collect::<Vec<_>>();
575        ctx.stores
576            .ingress_store
577            .state()
578            .into_iter()
579            .filter(|ing| {
580                ing.namespace() == kanidm.namespace()
581                    && !names.contains(&ing.name_any())
582                    && ing.metadata.labels == Some(kanidm.generate_labels())
583            })
584            .collect::<Vec<_>>()
585    };
586    let ingress_delete_futures = deprecated_ingresses
587        .iter()
588        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
589        .collect::<TryJoinAll<_>>();
590
591    let ingress_futures = kanidm
592        .create_ingress()
593        .into_iter()
594        .chain(kanidm.create_region_ingress())
595        .map(|ingress| kanidm.patch(&ctx, ingress))
596        .collect::<TryJoinAll<_>>();
597
598    let deprecated_http_routes = match &ctx.stores.http_route_store {
599        Some(store) => {
600            let expected_names = kanidm
601                .spec
602                .gateway
603                .as_ref()
604                .map(|_| vec![kanidm.name_any()])
605                .unwrap_or_default();
606            store
607                .state()
608                .into_iter()
609                .filter(|route| {
610                    route.namespace() == kanidm.namespace()
611                        && !expected_names.contains(&route.name_any())
612                        && route.metadata.labels == Some(kanidm.generate_labels())
613                })
614                .collect::<Vec<_>>()
615        }
616        None => Vec::new(),
617    };
618    let http_route_delete_futures = deprecated_http_routes
619        .iter()
620        .map(|route| kanidm.delete(&ctx, route.as_ref()))
621        .collect::<TryJoinAll<_>>();
622
623    let http_route_futures = if ctx.stores.http_route_store.is_some() {
624        kanidm
625            .create_http_route()
626            .into_iter()
627            .map(|route| kanidm.patch(&ctx, route))
628            .collect::<TryJoinAll<_>>()
629    } else {
630        try_join_all(Vec::new())
631    };
632
633    try_join!(
634        sts_delete_futures,
635        admin_secret_future,
636        replication_secret_futures,
637        sts_futures,
638        service_future,
639        rg_svcs_delete_futures,
640        rg_services_futures,
641        ingress_delete_futures,
642        ingress_futures,
643        http_route_delete_futures,
644        http_route_futures
645    )?;
646
647    if is_kanidm_available(status.clone()) {
648        let namespace = kanidm.namespace().unwrap();
649        let name = kanidm.name_any();
650        let system_client = crate::controller::kanidm::KanidmClients::create_client(
651            &namespace,
652            &name,
653            crate::controller::kanidm::KanidmUser::Admin,
654            ctx.kaniop_ctx.client.clone(),
655        )
656        .await?;
657        reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
658
659        let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
660            &namespace,
661            &name,
662            crate::controller::kanidm::KanidmUser::IdmAdmin,
663            ctx.kaniop_ctx.client.clone(),
664        )
665        .await?;
666        let mail_sender_status =
667            reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
668        let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
669        if mail_sender_status != current_mail_sender_status {
670            let kanidm_api: Api<Kanidm> =
671                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
672            let status_patch = serde_json::json!({
673                "status": {
674                    "mailSender": mail_sender_status
675                }
676            });
677            kanidm_api
678                .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
679                .await
680                .map_err(|e| {
681                    Error::KubeError(
682                        format!("failed to patch Kanidm/status {namespace}/{name}"),
683                        Box::new(e),
684                    )
685                })?;
686        }
687    }
688
689    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
690}
691
692async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
693    debug!(msg = "cleanup");
694
695    cleanup_mail_sender_resources(&kanidm, &ctx).await?;
696
697    let namespace = kanidm.namespace().unwrap();
698    let name = kanidm.name_any();
699
700    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
701        &namespace,
702        &name,
703        crate::controller::kanidm::KanidmUser::IdmAdmin,
704        ctx.kaniop_ctx.client.clone(),
705    )
706    .await
707    {
708        cleanup_mail_sender_in_kanidm(&kanidm_client, &name).await?;
709    }
710
711    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
712    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
713}
714
715async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
716    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
717    let start = std::time::Instant::now();
718    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
719    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
720
721    loop {
722        let sts = sts_api.get(sts_name).await.map_err(|e| {
723            Error::KubeError(
724                format!("failed to get StatefulSet {namespace}/{sts_name}"),
725                Box::new(e),
726            )
727        })?;
728
729        if sts.metadata.deletion_timestamp.is_some() {
730            return Err(Error::ReceiveOutput(format!(
731                "StatefulSet {namespace}/{sts_name} is being deleted"
732            )));
733        }
734
735        let spec = sts.spec.as_ref().ok_or_else(|| {
736            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
737        })?;
738
739        let status = match sts.status {
740            Some(ref s) => s,
741            None => {
742                if start.elapsed() >= timeout {
743                    return Err(Error::ReceiveOutput(format!(
744                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
745                    )));
746                }
747                sleep(poll_interval).await;
748                continue;
749            }
750        };
751
752        let replicas = spec.replicas.unwrap_or(0);
753
754        if replicas == 0 {
755            return Ok(());
756        }
757
758        let updated_replicas = status.updated_replicas.unwrap_or(0);
759        let current_revision = status.current_revision.as_deref().unwrap_or("");
760        let update_revision = status.update_revision.as_deref().unwrap_or("");
761        let ready_replicas = status.ready_replicas.unwrap_or(0);
762
763        if updated_replicas == replicas
764            && current_revision == update_revision
765            && ready_replicas == replicas
766        {
767            return Ok(());
768        }
769
770        if start.elapsed() >= timeout {
771            return Err(Error::ReceiveOutput(format!(
772                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
773            )));
774        }
775
776        sleep(poll_interval).await;
777    }
778}
779
780async fn show_replica_cert_with_retries(
781    kanidm: &Kanidm,
782    ctx: Arc<Context>,
783    pod_name: &str,
784    max_attempts: u32,
785) -> std::result::Result<String, Error> {
786    let mut last_error = None;
787    for attempt in 0..max_attempts {
788        if attempt > 0 {
789            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
790        }
791        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
792            Ok(cert) => return Ok(cert),
793            Err(e) => {
794                info!(
795                    msg = format!(
796                        "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
797                        attempt + 1,
798                        max_attempts
799                    )
800                );
801                last_error = Some(e);
802            }
803        }
804    }
805    Err(last_error.unwrap_or_else(|| {
806        Error::ReceiveOutput(format!(
807            "failed to get certificate for {pod_name} after {max_attempts} attempts"
808        ))
809    }))
810}
811
812impl Kanidm {
813    // Convenience methods that handle context and operator name
814    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
815    where
816        K: Resource<Scope = NamespaceResourceScope>
817            + Serialize
818            + Clone
819            + std::fmt::Debug
820            + for<'de> Deserialize<'de>,
821        <K as kube::Resource>::DynamicType: Default,
822        <K as Resource>::Scope: std::marker::Sized,
823    {
824        self.kube_delete(
825            ctx.kaniop_ctx.client.clone(),
826            &ctx.kaniop_ctx.metrics,
827            resource,
828        )
829        .await
830    }
831
832    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
833    where
834        K: Resource<Scope = NamespaceResourceScope>
835            + Serialize
836            + Clone
837            + std::fmt::Debug
838            + for<'de> Deserialize<'de>,
839        <K as kube::Resource>::DynamicType: Default,
840        <K as Resource>::Scope: std::marker::Sized,
841    {
842        self.kube_patch(
843            ctx.kaniop_ctx.client.clone(),
844            &ctx.kaniop_ctx.metrics,
845            resource,
846            KANIDM_OPERATOR_NAME,
847        )
848        .await
849    }
850
851    #[inline]
852    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
853        LABELS
854            .clone()
855            .into_iter()
856            .chain([
857                (INSTANCE_LABEL.to_string(), self.name_any()),
858                (CLUSTER_LABEL.to_string(), self.name_any()),
859            ])
860            .collect()
861    }
862
863    #[inline]
864    fn generate_labels(&self) -> BTreeMap<String, String> {
865        self.generate_resource_labels()
866    }
867
868    #[inline]
869    fn get_tls_secret_name(&self) -> String {
870        format!("{}-tls", self.name_any())
871    }
872
873    #[inline]
874    fn get_namespace(&self) -> String {
875        // safe unwrap: Kanidm is namespaced scoped
876        self.namespace().unwrap()
877    }
878
879    #[inline]
880    fn is_replication_enabled(&self) -> bool {
881        self.spec.replica_groups.len() > 1
882            || self
883                .spec
884                .replica_groups
885                .first()
886                .is_some_and(|rg| rg.replicas > 1)
887            || !self.spec.external_replication_nodes.is_empty()
888    }
889
890    #[inline]
891    fn is_updatable(&self, status: &KanidmStatus) -> bool {
892        status
893            .version
894            .as_ref()
895            .map(|v| {
896                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
897                    return false;
898                }
899                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
900                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
901                match v.upgrade_check_result {
902                    KanidmUpgradeCheckResult::Passed => {
903                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
904                            minor <= v_minor + 1
905                        } else {
906                            true
907                        }
908                    }
909                    KanidmUpgradeCheckResult::Failed => {
910                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
911                            versions
912                        {
913                            major == v_major && minor == v_minor && patch >= v_patch
914                        } else {
915                            v.image_tag == current_tag
916                        }
917                    }
918                }
919            })
920            .unwrap_or(true)
921    }
922
923    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
924        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
925        let pod = pod_api.get(pod_name).await.map_err(|e| {
926            Error::KubeError(
927                format!("failed to get pod {namespace}/{pod_name}"),
928                Box::new(e),
929            )
930        })?;
931        Ok(pod
932            .status
933            .as_ref()
934            .and_then(|s| s.conditions.as_ref())
935            .is_some_and(|conditions| {
936                conditions
937                    .iter()
938                    .any(|c| c.type_ == "Ready" && c.status == "True")
939            }))
940    }
941
942    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
943        let start = std::time::Instant::now();
944        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
945        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
946
947        loop {
948            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
949                Ok(true) => return Ok(()),
950                Ok(false) => {}
951                Err(e) => {
952                    if !e.is_retryable() {
953                        return Err(e);
954                    }
955                    trace!(msg = "pod readiness check failed, retrying", %e);
956                }
957            }
958
959            if start.elapsed() >= timeout {
960                return Err(Error::ReceiveOutput(format!(
961                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
962                )));
963            }
964
965            sleep(poll_interval).await;
966        }
967    }
968
969    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
970    where
971        I: IntoIterator<Item = T> + Debug,
972        T: Into<String>,
973    {
974        let namespace = &self.get_namespace();
975        trace!(
976            msg = "pod exec",
977            resource.name = &pod_name,
978            resource.namespace = &namespace,
979            ?command
980        );
981        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
982        let attached = pod
983            .exec(
984                pod_name,
985                command,
986                &AttachParams::default().container("kanidm"),
987            )
988            .await
989            .map_err(|e| {
990                Error::KubeError(
991                    format!("failed to exec pod {namespace}/{pod_name}"),
992                    Box::new(e),
993                )
994            })?;
995        get_output(attached).await
996    }
997
998    async fn exec_with_wait<I, T>(
999        &self,
1000        ctx: Arc<Context>,
1001        pod_name: &str,
1002        command: I,
1003    ) -> Result<String>
1004    where
1005        I: IntoIterator<Item = T> + Debug,
1006        T: Into<String>,
1007    {
1008        let namespace = &self.get_namespace();
1009        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1010        self.exec(ctx, pod_name, command).await
1011    }
1012
1013    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1014    where
1015        I: IntoIterator<Item = T> + Debug,
1016        T: Into<String>,
1017    {
1018        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1019        let first_replica_group = self
1020            .spec
1021            .replica_groups
1022            .first()
1023            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1024        let sts_name = self.statefulset_name(&first_replica_group.name);
1025        let pod_name = format!("{sts_name}-0");
1026        self.exec(ctx, &pod_name, command).await
1027    }
1028
1029    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1030    where
1031        I: IntoIterator<Item = T> + Debug,
1032        T: Into<String>,
1033    {
1034        let first_replica_group = self
1035            .spec
1036            .replica_groups
1037            .first()
1038            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1039        let sts_name = self.statefulset_name(&first_replica_group.name);
1040        let pod_name = format!("{sts_name}-0");
1041        self.exec_with_wait(ctx, &pod_name, command).await
1042    }
1043}
1044
1045#[cfg(test)]
1046mod test {
1047    use super::statefulset::StatefulSetExt;
1048    use super::{Kanidm, reconcile_kanidm};
1049
1050    use crate::controller::State;
1051    use crate::kanidm::controller::context::{Context, Stores};
1052    use crate::kanidm::crd::KanidmStatus;
1053
1054    use kaniop_k8s_util::error::Result;
1055
1056    use std::sync::Arc;
1057
1058    use http::{Request, Response};
1059    use k8s_openapi::api::apps::v1::StatefulSet;
1060    use k8s_openapi::api::core::v1::Service;
1061    use k8s_openapi::api::networking::v1::Ingress;
1062    use kube::runtime::reflector::store::Writer;
1063    use kube::{Client, Resource, ResourceExt, client::Body};
1064    use opentelemetry::metrics::MeterProvider;
1065    use serde_json::json;
1066
1067    impl Kanidm {
1068        /// A normal test kanidm with a given status
1069        pub fn test() -> Self {
1070            let mut e = Kanidm::new(
1071                "test",
1072                serde_json::from_value(json!({
1073                    "domain": "idm.example.com",
1074                    "replicaGroups": [
1075                        {
1076                            "name": "default",
1077                            "replicas": 1
1078                        }
1079                    ]
1080                }))
1081                .unwrap(),
1082            );
1083            e.meta_mut().namespace = Some("default".into());
1084            // Pre-populate finalizer so reconcile callback runs immediately
1085            // (skips the "add finalizer + wait for next reconcile" step)
1086            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1087            e
1088        }
1089
1090        pub fn with_ingress(mut self) -> Self {
1091            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1092            self
1093        }
1094
1095        /// Modify kanidm replicas
1096        pub fn with_replicas(mut self, replicas: i32) -> Self {
1097            self.spec.replica_groups[0].replicas = replicas;
1098            self
1099        }
1100
1101        /// Modify kanidm to set a deletion timestamp
1102        pub fn needs_delete(mut self) -> Self {
1103            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1104            use k8s_openapi::jiff::Timestamp;
1105            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1106            self.meta_mut().deletion_timestamp = Some(Time(now));
1107            self
1108        }
1109
1110        /// Modify a kanidm to have an expected status
1111        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1112            self.status = Some(status);
1113            self
1114        }
1115    }
1116
1117    // We wrap tower_test::mock::Handle
1118    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1119    pub struct ApiServerVerifier(ApiServerHandle);
1120
1121    /// Scenarios we test for in ApiServerVerifier
1122    pub enum Scenario {
1123        Create(Kanidm),
1124        CreateWithTwoReplicas(Kanidm),
1125        CreateWithIngress(Kanidm),
1126        CreateWithIngressWithTwoReplicas(Kanidm),
1127    }
1128
1129    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1130        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1131            .await
1132            .expect("timeout on mock apiserver")
1133            .expect("scenario succeeded")
1134    }
1135
1136    impl ApiServerVerifier {
1137        /// Tests only get to run specific scenarios that has matching handlers
1138        ///
1139        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1140        ///
1141        /// NB: If the controller is making more calls than we are handling in the scenario,
1142        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1143        ///
1144        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1145        /// scenario runs to completion (i.e. all expected calls were responded to),
1146        /// using the timeout to catch missing api calls to Kubernetes.
1147        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1148            tokio::spawn(async move {
1149                // moving self => one scenario per test
1150                match scenario {
1151                    Scenario::Create(kanidm) => {
1152                        self.handle_kanidm_get(kanidm.clone())
1153                            .await
1154                            .unwrap()
1155                            .handle_kanidm_status_patch(kanidm.clone())
1156                            .await
1157                            .unwrap()
1158                            .handle_statefulset_patch(kanidm.clone())
1159                            .await
1160                            .unwrap()
1161                            .handle_service_patch(kanidm.clone())
1162                            .await
1163                    }
1164                    Scenario::CreateWithTwoReplicas(kanidm) => {
1165                        self.handle_kanidm_get(kanidm.clone())
1166                            .await
1167                            .unwrap()
1168                            .handle_kanidm_status_patch(kanidm.clone())
1169                            .await
1170                            .unwrap()
1171                            .handle_statefulset_patch(kanidm.clone())
1172                            .await
1173                            .unwrap()
1174                            .handle_service_patch(kanidm.clone())
1175                            .await
1176                    }
1177                    Scenario::CreateWithIngress(kanidm) => {
1178                        self.handle_kanidm_get(kanidm.clone())
1179                            .await
1180                            .unwrap()
1181                            .handle_kanidm_status_patch(kanidm.clone())
1182                            .await
1183                            .unwrap()
1184                            .handle_statefulset_patch(kanidm.clone())
1185                            .await
1186                            .unwrap()
1187                            .handle_service_patch(kanidm.clone())
1188                            .await
1189                            .unwrap()
1190                            .handle_ingress_patch(kanidm.clone())
1191                            .await
1192                    }
1193                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1194                        self.handle_kanidm_get(kanidm.clone())
1195                            .await
1196                            .unwrap()
1197                            .handle_kanidm_status_patch(kanidm.clone())
1198                            .await
1199                            .unwrap()
1200                            .handle_statefulset_patch(kanidm.clone())
1201                            .await
1202                            .unwrap()
1203                            .handle_service_patch(kanidm.clone())
1204                            .await
1205                            .unwrap()
1206                            .handle_ingress_patch(kanidm.clone())
1207                            .await
1208                    }
1209                }
1210                .expect("scenario completed without errors");
1211            })
1212        }
1213
1214        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1215            let (request, send) = self.0.next_request().await.expect("service not called");
1216            assert_eq!(request.method(), http::Method::GET);
1217            let expected_uri_prefix = format!(
1218                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1219                kanidm.name_any()
1220            );
1221            let uri = request.uri().to_string();
1222            assert!(
1223                uri.starts_with(&expected_uri_prefix),
1224                "expected uri to start with {}, got {}",
1225                expected_uri_prefix,
1226                uri
1227            );
1228            let response = serde_json::to_vec(&kanidm).unwrap();
1229            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1230            Ok(self)
1231        }
1232
1233        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1234            let (request, send) = self.0.next_request().await.expect("service not called");
1235            assert_eq!(request.method(), http::Method::PATCH);
1236            assert_eq!(
1237                request.uri().to_string(),
1238                format!(
1239                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1240                    kanidm.name_any()
1241                )
1242            );
1243
1244            let req_body = request.into_body().collect_bytes().await.unwrap();
1245            let json: serde_json::Value =
1246                serde_json::from_slice(&req_body).expect("patch object is json");
1247            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1248                .expect("valid kanidm status");
1249            let mut kanidm = Kanidm::test();
1250            kanidm.status = Some(status.clone());
1251            let response = serde_json::to_vec(&kanidm).unwrap();
1252            // pass through kanidm "patch accepted"
1253            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1254            Ok(self)
1255        }
1256
1257        async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1258            for rg in kanidm.spec.replica_groups.iter() {
1259                let (request, send) = self.0.next_request().await.expect("service not called");
1260                assert_eq!(request.method(), http::Method::PATCH);
1261                assert_eq!(
1262                    request.uri().to_string(),
1263                    format!(
1264                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1265                        kanidm.statefulset_name(&rg.name)
1266                    )
1267                );
1268                let req_body = request.into_body().collect_bytes().await.unwrap();
1269                let json: serde_json::Value =
1270                    serde_json::from_slice(&req_body).expect("patch object is json");
1271                let statefulset: StatefulSet =
1272                    serde_json::from_value(json).expect("valid statefulset");
1273                assert_eq!(
1274                    statefulset.clone().spec.unwrap().replicas.unwrap(),
1275                    rg.replicas
1276                );
1277                let response = serde_json::to_vec(&statefulset).unwrap();
1278                // pass through kanidm "patch accepted"
1279                send.send_response(Response::builder().body(Body::from(response)).unwrap());
1280            }
1281
1282            Ok(self)
1283        }
1284
1285        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1286            let (request, send) = self.0.next_request().await.expect("service not called");
1287            assert_eq!(request.method(), http::Method::PATCH);
1288            assert_eq!(
1289                request.uri().to_string(),
1290                format!(
1291                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1292                    kanidm.name_any()
1293                )
1294            );
1295
1296            let req_body = request.into_body().collect_bytes().await.unwrap();
1297            let json: serde_json::Value =
1298                serde_json::from_slice(&req_body).expect("patch object is json");
1299            let service: Service = serde_json::from_value(json).expect("valid service");
1300            let response = serde_json::to_vec(&service).unwrap();
1301            // pass through kanidm "patch accepted"
1302            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1303            Ok(self)
1304        }
1305
1306        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1307            let (request, send) = self.0.next_request().await.expect("service not called");
1308            assert_eq!(request.method(), http::Method::PATCH);
1309            assert_eq!(
1310                request.uri().to_string(),
1311                format!(
1312                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1313                    kanidm.name_any()
1314                )
1315            );
1316
1317            let req_body = request.into_body().collect_bytes().await.unwrap();
1318            let json: serde_json::Value =
1319                serde_json::from_slice(&req_body).expect("patch object is json");
1320            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1321            let response = serde_json::to_vec(&ingress).unwrap();
1322            // pass through kanidm "patch accepted"
1323            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1324            Ok(self)
1325        }
1326    }
1327
1328    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1329        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1330        let mock_client = Client::new(mock_service, "default");
1331        let stores = Stores {
1332            stateful_set_store: Writer::default().as_reader(),
1333            service_store: Writer::default().as_reader(),
1334            ingress_store: Writer::default().as_reader(),
1335            secret_store: Writer::default().as_reader(),
1336            http_route_store: Some(Writer::default().as_reader()),
1337            deployment_store: Writer::default().as_reader(),
1338            config_map_store: Writer::default().as_reader(),
1339        };
1340        let controller_id = "test";
1341
1342        // Create a test meter for metrics
1343        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
1344        let meter = provider.meter("test");
1345        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
1346
1347        let state = State::new(
1348            metrics,
1349            Writer::default().as_reader(),
1350            Writer::default().as_reader(),
1351            None,
1352        );
1353        let ctx = Arc::new(Context::new(
1354            state.to_context(mock_client, controller_id),
1355            stores,
1356        ));
1357        (ctx, ApiServerVerifier(handle))
1358    }
1359
1360    #[tokio::test]
1361    async fn kanidm_create() {
1362        let (testctx, fakeserver) = get_test_context();
1363        let kanidm = Kanidm::test();
1364        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
1365        reconcile_kanidm(Arc::new(kanidm), testctx)
1366            .await
1367            .expect("reconciler");
1368        timeout_after_1s(mocksrv).await;
1369    }
1370
1371    #[tokio::test]
1372    async fn kanidm_create_with_two_replicas() {
1373        let (testctx, fakeserver) = get_test_context();
1374        let kanidm = Kanidm::test().with_replicas(2);
1375        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
1376        reconcile_kanidm(Arc::new(kanidm), testctx)
1377            .await
1378            .expect("reconciler");
1379        timeout_after_1s(mocksrv).await;
1380    }
1381
1382    #[tokio::test]
1383    async fn kanidm_create_with_ingress() {
1384        let (testctx, fakeserver) = get_test_context();
1385        let kanidm = Kanidm::test().with_ingress();
1386        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
1387        reconcile_kanidm(Arc::new(kanidm), testctx)
1388            .await
1389            .expect("reconciler");
1390        timeout_after_1s(mocksrv).await;
1391    }
1392
1393    #[tokio::test]
1394    async fn kanidm_create_with_ingress_with_two_replicas() {
1395        let (testctx, fakeserver) = get_test_context();
1396        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
1397        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
1398        reconcile_kanidm(Arc::new(kanidm), testctx)
1399            .await
1400            .expect("reconciler");
1401        timeout_after_1s(mocksrv).await;
1402    }
1403}