Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::StatefulSetExt;
21use self::status::StatusExt;
22use self::status::{is_kanidm_available, is_kanidm_initialized};
23
24use crate::controller::context::KubeOperations;
25use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
26use crate::kanidm::crd::{
27    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
28};
29use crate::kanidm::reconcile::statefulset::REPLICA_LABEL;
30use crate::telemetry;
31
32use kaniop_k8s_util::client::get_output;
33use kaniop_k8s_util::error::{Error, Result};
34use kaniop_k8s_util::parse::parse_semver;
35use kaniop_k8s_util::resources::get_image_tag;
36
37use std::collections::{BTreeMap, BTreeSet};
38use std::fmt::Debug;
39use std::sync::{Arc, LazyLock};
40
41use futures::future::{TryJoinAll, join_all, try_join_all};
42use futures::stream::{self, StreamExt};
43use futures::try_join;
44use k8s_openapi::NamespaceResourceScope;
45use k8s_openapi::api::apps::v1::StatefulSet;
46use k8s_openapi::api::core::v1::Pod;
47use kube::Resource;
48use kube::ResourceExt;
49use kube::api::{Api, AttachParams, Patch, PatchParams};
50use kube::runtime::controller::Action;
51use kube::runtime::events::{Event, EventType};
52use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
53use serde::{Deserialize, Serialize};
54use tokio::time::{Duration, sleep};
55use tracing::{Span, debug, field, info, instrument, trace, warn};
56
57const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
58const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
59const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
60const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
61const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
62const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
63const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
64
65pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
66const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
67pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
68
69const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
70
71static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
72    BTreeMap::from([
73        (NAME_LABEL.to_string(), "kanidm".to_string()),
74        (
75            MANAGED_BY_LABEL.to_string(),
76            format!("kaniop-{CONTROLLER_ID}"),
77        ),
78    ])
79});
80
81async fn restart_statefulsets_after_replica_secret_update(
82    ctx: Arc<Context>,
83    sts_api: &Api<StatefulSet>,
84    namespace: &str,
85    statefulset_names: &[String],
86    sequential: bool,
87) -> Result<()> {
88    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
89    // Pods must restart after cert secret changes or replication keeps stale TLS material.
90    let restart_futures = statefulset_names
91        .iter()
92        .map(|sts_name| sts_api.restart(sts_name));
93
94    if sequential {
95        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
96        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
97            return Err(Error::kube_error(
98                "restart",
99                "StatefulSet",
100                namespace,
101                "replica certificate secret update statefulsets",
102                first_err,
103            ));
104        }
105    } else {
106        let results: Vec<_> = join_all(restart_futures).await;
107        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
108            return Err(Error::kube_error(
109                "restart",
110                "StatefulSet",
111                namespace,
112                "replica certificate secret update statefulsets",
113                first_err,
114            ));
115        }
116    }
117
118    let rollout_futures = statefulset_names
119        .iter()
120        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
121    try_join_all(rollout_futures).await?;
122
123    Ok(())
124}
125
126pub async fn reconcile_admins_secret(
127    kanidm: Arc<Kanidm>,
128    ctx: Arc<Context>,
129    status: &KanidmStatus,
130) -> Result<()> {
131    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
132        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
133        kanidm.patch(&ctx, admins_secret).await?;
134    }
135    Ok(())
136}
137
138pub async fn reconcile_replication_secrets(
139    kanidm: Arc<Kanidm>,
140    ctx: Arc<Context>,
141    status: &KanidmStatus,
142) -> Result<()> {
143    let expected_secret_names = if kanidm.is_replication_enabled() {
144        status
145            .replica_statuses
146            .iter()
147            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
148            .collect::<Vec<_>>()
149    } else {
150        Vec::new()
151    };
152    let deprecated_secrets = ctx
153        .stores
154        .secret_store
155        .state()
156        .into_iter()
157        .filter(|secret| {
158            secret.namespace() == kanidm.namespace()
159                && kanidm.admins_secret_name() != secret.name_any()
160                && !expected_secret_names.contains(&secret.name_any())
161                && secret
162                    .metadata
163                    .labels
164                    .as_ref()
165                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
166        })
167        .collect::<Vec<_>>();
168    let secret_delete_future = deprecated_secrets
169        .iter()
170        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
171        .collect::<TryJoinAll<_>>();
172    try_join!(secret_delete_future)?;
173
174    if kanidm.is_replication_enabled() {
175        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
176        let statefulset_names = status
177            .replica_statuses
178            .iter()
179            .map(|rs| rs.statefulset_name.clone())
180            .collect::<BTreeSet<_>>()
181            .into_iter()
182            .collect::<Vec<_>>();
183
184        let has_pending = status
185            .replica_statuses
186            .iter()
187            .any(|rs| rs.state == KanidmReplicaState::Pending);
188
189        let sts_api =
190            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
191
192        if has_pending {
193            stream::iter(
194                status
195                    .replica_statuses
196                    .iter()
197                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
198            )
199            .then(|rs| async {
200                let secret = kanidm
201                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
202                    .await?;
203                kanidm.patch(&ctx, secret).await
204            })
205            .collect::<Vec<_>>()
206            .await
207            .into_iter()
208            .collect::<Result<Vec<_>>>()?;
209
210            let restart_futures = status
211                .replica_statuses
212                .iter()
213                .map(|rs| sts_api.restart(&rs.statefulset_name));
214
215            if has_single_replica {
216                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
217                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
218                    return Err(Error::kube_error(
219                        "restart",
220                        "StatefulSet",
221                        kanidm.get_namespace(),
222                        "pending replica statefulsets",
223                        first_err,
224                    ));
225                }
226            } else {
227                let results: Vec<_> = join_all(restart_futures).await;
228                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
229                    return Err(Error::kube_error(
230                        "restart",
231                        "StatefulSet",
232                        kanidm.get_namespace(),
233                        "pending replica statefulsets",
234                        first_err,
235                    ));
236                }
237            }
238        }
239
240        let has_certificate_host_invalid = status
241            .replica_statuses
242            .iter()
243            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
244
245        let has_certificate_expiring = status
246            .replica_statuses
247            .iter()
248            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
249
250        let has_non_ready_replicas = status
251            .replica_statuses
252            .iter()
253            .any(|rs| rs.state != KanidmReplicaState::Ready);
254
255        let host_invalid_replicas: Vec<_> = status
256            .replica_statuses
257            .iter()
258            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
259            .collect();
260
261        let cert_expiring_replicas: Vec<_> = status
262            .replica_statuses
263            .iter()
264            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
265            .collect();
266
267        if !host_invalid_replicas.is_empty() {
268            let restart_futures = host_invalid_replicas
269                .iter()
270                .map(|rs| sts_api.restart(&rs.statefulset_name));
271
272            if has_single_replica {
273                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
274                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
275                    return Err(Error::kube_error(
276                        "restart",
277                        "StatefulSet",
278                        kanidm.get_namespace(),
279                        "certificate host invalid statefulsets",
280                        first_err,
281                    ));
282                }
283            } else {
284                let results: Vec<_> = join_all(restart_futures).await;
285                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
286                    return Err(Error::kube_error(
287                        "restart",
288                        "StatefulSet",
289                        kanidm.get_namespace(),
290                        "certificate host invalid statefulsets",
291                        first_err,
292                    ));
293                }
294            }
295
296            let restarted_sts_names: Vec<_> = host_invalid_replicas
297                .iter()
298                .map(|rs| rs.statefulset_name.clone())
299                .collect::<BTreeSet<_>>()
300                .into_iter()
301                .collect();
302
303            let namespace = kanidm.get_namespace();
304            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
305                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
306            });
307            try_join_all(rollout_futures).await?;
308
309            let renew_futures = host_invalid_replicas
310                .iter()
311                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
312            let results: Vec<_> = join_all(renew_futures).await;
313            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
314                return Err(first_err);
315            }
316
317            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
318
319            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
320                let cert = show_replica_cert_with_retries(
321                    &kanidm,
322                    ctx.clone(),
323                    &rs.pod_name,
324                    CERT_SHOW_RETRY_ATTEMPTS,
325                )
326                .await?;
327                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
328                kanidm.patch(&ctx, secret.clone()).await
329            });
330            try_join_all(cert_futures).await?;
331
332            restart_statefulsets_after_replica_secret_update(
333                ctx.clone(),
334                &sts_api,
335                &kanidm.get_namespace(),
336                &statefulset_names,
337                has_single_replica,
338            )
339            .await?;
340
341            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
342        }
343
344        if !cert_expiring_replicas.is_empty() {
345            let renew_futures = cert_expiring_replicas
346                .iter()
347                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
348            let results: Vec<_> = join_all(renew_futures).await;
349            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
350                return Err(first_err);
351            }
352
353            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
354
355            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
356                let cert = show_replica_cert_with_retries(
357                    &kanidm,
358                    ctx.clone(),
359                    &rs.pod_name,
360                    CERT_SHOW_RETRY_ATTEMPTS,
361                )
362                .await?;
363                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
364                kanidm.patch(&ctx, secret.clone()).await
365            });
366            try_join_all(cert_futures).await?;
367
368            restart_statefulsets_after_replica_secret_update(
369                ctx.clone(),
370                &sts_api,
371                &kanidm.get_namespace(),
372                &statefulset_names,
373                has_single_replica,
374            )
375            .await?;
376
377            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
378        }
379
380        if has_non_ready_replicas
381            && !has_pending
382            && !has_certificate_host_invalid
383            && !has_certificate_expiring
384        {
385            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
386
387            let restart_futures = status
388                .replica_statuses
389                .iter()
390                .map(|rs| sts_api.restart(&rs.statefulset_name));
391
392            if has_single_replica {
393                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
394                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
395                    return Err(Error::kube_error(
396                        "restart",
397                        "StatefulSet",
398                        kanidm.get_namespace(),
399                        "replica statefulsets",
400                        first_err,
401                    ));
402                }
403            } else {
404                let results: Vec<_> = join_all(restart_futures).await;
405                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
406                    return Err(Error::kube_error(
407                        "restart",
408                        "StatefulSet",
409                        kanidm.get_namespace(),
410                        "replica statefulsets",
411                        first_err,
412                    ));
413                }
414            }
415        }
416    }
417
418    Ok(())
419}
420
421#[instrument(skip(ctx, kanidm))]
422pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
423    let trace_id = telemetry::get_trace_id();
424    Span::current().record("trace_id", field::display(&trace_id));
425    let _timer = ctx
426        .kaniop_ctx
427        .metrics
428        .reconcile_count_and_measure(&trace_id);
429    info!(msg = "reconciling Kanidm");
430
431    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
432        debug!(msg = "failed to reconcile status", %e);
433        ctx.kaniop_ctx.metrics.status_update_errors_inc();
434        e
435    })?;
436    let kanidm_api: Api<Kanidm> =
437        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
438    finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
439        match event {
440            Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
441            Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
442        }
443    })
444    .await
445    .or_else(|e| match e {
446        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
447            debug!(msg = "resource already removed during finalizer cleanup");
448            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
449        }
450        _ => Err(Error::FinalizerError(
451            "failed on kanidm account finalizer".to_string(),
452            Box::new(e),
453        )),
454    })
455}
456
457async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
458    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
459    let replication_secret_futures =
460        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
461
462    let sts_to_delete = {
463        let expected_sts_names = kanidm
464            .spec
465            .replica_groups
466            .iter()
467            .map(|rg| kanidm.statefulset_name(&rg.name))
468            .collect::<Vec<_>>();
469        ctx.stores
470            .stateful_set_store
471            .state()
472            .into_iter()
473            .filter(|sts| {
474                sts.namespace() == kanidm.namespace()
475                    && sts
476                        .metadata
477                        .labels
478                        .as_ref()
479                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
480                    && !expected_sts_names.contains(&sts.name_any())
481            })
482            .collect::<Vec<_>>()
483    };
484    let sts_delete_futures = sts_to_delete
485        .iter()
486        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
487        .collect::<TryJoinAll<_>>();
488
489    let sts_futures = match kanidm.is_updatable(&status) {
490        true => kanidm
491            .spec
492            .replica_groups
493            .iter()
494            .map(|rg| {
495                let sts = kanidm.create_statefulset(rg)?;
496                Ok(kanidm.patch(&ctx, sts))
497            })
498            .collect::<Result<TryJoinAll<_>, _>>()?,
499        false => {
500            let note = match status.version.as_ref() {
501                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
502                    format!(
503                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
504                        v.image_tag,
505                        crate::version::KANIDM_CLIENT_VERSION
506                    )
507                }
508                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
509            };
510            let _ignore_error = ctx
511                .kaniop_ctx
512                .recorder
513                .publish(
514                    &Event {
515                        type_: EventType::Warning,
516                        reason: "UpgradeBlocked".to_string(),
517                        note: Some(note),
518                        action: "ReconcileStatefulSet".to_string(),
519                        secondary: None,
520                    },
521                    &kanidm.object_ref(&()),
522                )
523                .await
524                .map_err(|e| {
525                    warn!(msg = "failed to publish KanidmError event", %e);
526                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
527                });
528            try_join_all([])
529        }
530    };
531    let service_future = kanidm.patch(&ctx, kanidm.create_service());
532
533    let deprecated_rg_svcs = {
534        let expected_rg_svcs_names = kanidm
535            .spec
536            .replica_groups
537            .iter()
538            .filter(|rg| rg.services.is_some())
539            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
540            .collect::<Vec<_>>();
541        ctx.stores
542            .service_store
543            .state()
544            .into_iter()
545            .filter(|svc| {
546                let labels = svc.metadata.labels.as_ref();
547                svc.namespace() == kanidm.namespace()
548                    && !expected_rg_svcs_names.contains(&svc.name_any())
549                    && labels.is_some_and(|l| {
550                        l.contains_key(REPLICA_LABEL)
551                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
552                    })
553            })
554            .collect::<Vec<_>>()
555    };
556    let rg_svcs_delete_futures = deprecated_rg_svcs
557        .iter()
558        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
559        .collect::<TryJoinAll<_>>();
560
561    let rg_services_futures = kanidm
562        .spec
563        .replica_groups
564        .iter()
565        .filter(|rg| rg.services.is_some())
566        .flat_map(|rg| {
567            (0..rg.replicas).map(|i| kanidm.patch(&ctx, kanidm.create_replica_group_service(rg, i)))
568        })
569        .collect::<TryJoinAll<_>>();
570
571    let deprecated_ingresses = {
572        let names = [
573            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
574            kanidm.generate_region_ingress_name(),
575        ]
576        .into_iter()
577        .flatten()
578        .collect::<Vec<_>>();
579        ctx.stores
580            .ingress_store
581            .state()
582            .into_iter()
583            .filter(|ing| {
584                ing.namespace() == kanidm.namespace()
585                    && !names.contains(&ing.name_any())
586                    && ing.metadata.labels == Some(kanidm.generate_labels())
587            })
588            .collect::<Vec<_>>()
589    };
590    let ingress_delete_futures = deprecated_ingresses
591        .iter()
592        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
593        .collect::<TryJoinAll<_>>();
594
595    let ingress_futures = kanidm
596        .create_ingress()
597        .into_iter()
598        .chain(kanidm.create_region_ingress())
599        .map(|ingress| kanidm.patch(&ctx, ingress))
600        .collect::<TryJoinAll<_>>();
601
602    let deprecated_http_routes = match &ctx.stores.http_route_store {
603        Some(store) => {
604            let expected_names = kanidm
605                .spec
606                .gateway
607                .as_ref()
608                .map(|_| vec![kanidm.name_any()])
609                .unwrap_or_default();
610            store
611                .state()
612                .into_iter()
613                .filter(|route| {
614                    route.namespace() == kanidm.namespace()
615                        && !expected_names.contains(&route.name_any())
616                        && route.metadata.labels == Some(kanidm.generate_labels())
617                })
618                .collect::<Vec<_>>()
619        }
620        None => Vec::new(),
621    };
622    let http_route_delete_futures = deprecated_http_routes
623        .iter()
624        .map(|route| kanidm.delete(&ctx, route.as_ref()))
625        .collect::<TryJoinAll<_>>();
626
627    let http_route_futures = if ctx.stores.http_route_store.is_some() {
628        kanidm
629            .create_http_route()
630            .into_iter()
631            .map(|route| kanidm.patch(&ctx, route))
632            .collect::<TryJoinAll<_>>()
633    } else {
634        try_join_all(Vec::new())
635    };
636
637    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
638        Some(store) => {
639            let expected_names = kanidm
640                .spec
641                .gateway
642                .as_ref()
643                .and_then(|g| g.backend_tls_policy.as_ref())
644                .map(|_| vec![kanidm.name_any()])
645                .unwrap_or_default();
646            store
647                .state()
648                .into_iter()
649                .filter(|policy| {
650                    policy.namespace() == kanidm.namespace()
651                        && !expected_names.contains(&policy.name_any())
652                        && policy.metadata.labels == Some(kanidm.generate_labels())
653                })
654                .collect::<Vec<_>>()
655        }
656        None => Vec::new(),
657    };
658    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
659        .iter()
660        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
661        .collect::<TryJoinAll<_>>();
662
663    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
664        kanidm
665            .create_backend_tls_policy()
666            .into_iter()
667            .map(|policy| kanidm.patch(&ctx, policy))
668            .collect::<TryJoinAll<_>>()
669    } else {
670        try_join_all(Vec::new())
671    };
672
673    try_join!(
674        sts_delete_futures,
675        admin_secret_future,
676        replication_secret_futures,
677        sts_futures,
678        service_future,
679        rg_svcs_delete_futures,
680        rg_services_futures,
681        ingress_delete_futures,
682        ingress_futures,
683        http_route_delete_futures,
684        http_route_futures,
685        backend_tls_policy_delete_futures,
686        backend_tls_policy_futures
687    )?;
688
689    if is_kanidm_available(status.clone()) {
690        let namespace = kanidm.namespace().unwrap();
691        let name = kanidm.name_any();
692        let system_client = crate::controller::kanidm::KanidmClients::create_client(
693            &namespace,
694            &name,
695            crate::controller::kanidm::KanidmUser::Admin,
696            ctx.kaniop_ctx.client.clone(),
697        )
698        .await?;
699        reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
700
701        let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
702            &namespace,
703            &name,
704            crate::controller::kanidm::KanidmUser::IdmAdmin,
705            ctx.kaniop_ctx.client.clone(),
706        )
707        .await?;
708        let mail_sender_status =
709            reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
710        let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
711        if mail_sender_status != current_mail_sender_status {
712            let kanidm_api: Api<Kanidm> =
713                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
714            let status_patch = serde_json::json!({
715                "status": {
716                    "mailSender": mail_sender_status
717                }
718            });
719            kanidm_api
720                .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
721                .await
722                .map_err(|e| {
723                    Error::KubeError(
724                        format!("failed to patch Kanidm/status {namespace}/{name}"),
725                        Box::new(e),
726                    )
727                })?;
728        }
729    }
730
731    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
732}
733
734async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
735    debug!(msg = "cleanup");
736
737    cleanup_mail_sender_resources(&kanidm, &ctx).await?;
738
739    let namespace = kanidm.namespace().unwrap();
740    let name = kanidm.name_any();
741
742    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
743        &namespace,
744        &name,
745        crate::controller::kanidm::KanidmUser::IdmAdmin,
746        ctx.kaniop_ctx.client.clone(),
747    )
748    .await
749    {
750        cleanup_mail_sender_in_kanidm(&kanidm_client, &name).await?;
751    }
752
753    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
754    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
755}
756
757async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
758    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
759    let start = std::time::Instant::now();
760    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
761    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
762
763    loop {
764        let sts = sts_api.get(sts_name).await.map_err(|e| {
765            Error::KubeError(
766                format!("failed to get StatefulSet {namespace}/{sts_name}"),
767                Box::new(e),
768            )
769        })?;
770
771        if sts.metadata.deletion_timestamp.is_some() {
772            return Err(Error::ReceiveOutput(format!(
773                "StatefulSet {namespace}/{sts_name} is being deleted"
774            )));
775        }
776
777        let spec = sts.spec.as_ref().ok_or_else(|| {
778            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
779        })?;
780
781        let status = match sts.status {
782            Some(ref s) => s,
783            None => {
784                if start.elapsed() >= timeout {
785                    return Err(Error::ReceiveOutput(format!(
786                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
787                    )));
788                }
789                sleep(poll_interval).await;
790                continue;
791            }
792        };
793
794        let replicas = spec.replicas.unwrap_or(0);
795
796        if replicas == 0 {
797            return Ok(());
798        }
799
800        let updated_replicas = status.updated_replicas.unwrap_or(0);
801        let current_revision = status.current_revision.as_deref().unwrap_or("");
802        let update_revision = status.update_revision.as_deref().unwrap_or("");
803        let ready_replicas = status.ready_replicas.unwrap_or(0);
804
805        if updated_replicas == replicas
806            && current_revision == update_revision
807            && ready_replicas == replicas
808        {
809            return Ok(());
810        }
811
812        if start.elapsed() >= timeout {
813            return Err(Error::ReceiveOutput(format!(
814                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
815            )));
816        }
817
818        sleep(poll_interval).await;
819    }
820}
821
822async fn show_replica_cert_with_retries(
823    kanidm: &Kanidm,
824    ctx: Arc<Context>,
825    pod_name: &str,
826    max_attempts: u32,
827) -> std::result::Result<String, Error> {
828    let mut last_error = None;
829    for attempt in 0..max_attempts {
830        if attempt > 0 {
831            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
832        }
833        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
834            Ok(cert) => return Ok(cert),
835            Err(e) => {
836                info!(
837                    msg = format!(
838                        "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
839                        attempt + 1,
840                        max_attempts
841                    )
842                );
843                last_error = Some(e);
844            }
845        }
846    }
847    Err(last_error.unwrap_or_else(|| {
848        Error::ReceiveOutput(format!(
849            "failed to get certificate for {pod_name} after {max_attempts} attempts"
850        ))
851    }))
852}
853
854impl Kanidm {
855    // Convenience methods that handle context and operator name
856    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
857    where
858        K: Resource<Scope = NamespaceResourceScope>
859            + Serialize
860            + Clone
861            + std::fmt::Debug
862            + for<'de> Deserialize<'de>,
863        <K as kube::Resource>::DynamicType: Default,
864        <K as Resource>::Scope: std::marker::Sized,
865    {
866        self.kube_delete(
867            ctx.kaniop_ctx.client.clone(),
868            &ctx.kaniop_ctx.metrics,
869            resource,
870        )
871        .await
872    }
873
874    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
875    where
876        K: Resource<Scope = NamespaceResourceScope>
877            + Serialize
878            + Clone
879            + std::fmt::Debug
880            + for<'de> Deserialize<'de>,
881        <K as kube::Resource>::DynamicType: Default,
882        <K as Resource>::Scope: std::marker::Sized,
883    {
884        self.kube_patch(
885            ctx.kaniop_ctx.client.clone(),
886            &ctx.kaniop_ctx.metrics,
887            resource,
888            KANIDM_OPERATOR_NAME,
889        )
890        .await
891    }
892
893    #[inline]
894    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
895        LABELS
896            .clone()
897            .into_iter()
898            .chain([
899                (INSTANCE_LABEL.to_string(), self.name_any()),
900                (CLUSTER_LABEL.to_string(), self.name_any()),
901            ])
902            .collect()
903    }
904
905    #[inline]
906    fn generate_labels(&self) -> BTreeMap<String, String> {
907        self.generate_resource_labels()
908    }
909
910    #[inline]
911    fn get_tls_secret_name(&self) -> String {
912        format!("{}-tls", self.name_any())
913    }
914
915    #[inline]
916    fn get_namespace(&self) -> String {
917        // safe unwrap: Kanidm is namespaced scoped
918        self.namespace().unwrap()
919    }
920
921    #[inline]
922    fn is_replication_enabled(&self) -> bool {
923        self.spec.replica_groups.len() > 1
924            || self
925                .spec
926                .replica_groups
927                .first()
928                .is_some_and(|rg| rg.replicas > 1)
929            || !self.spec.external_replication_nodes.is_empty()
930    }
931
932    #[inline]
933    fn is_updatable(&self, status: &KanidmStatus) -> bool {
934        status
935            .version
936            .as_ref()
937            .map(|v| {
938                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
939                    return false;
940                }
941                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
942                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
943                match v.upgrade_check_result {
944                    KanidmUpgradeCheckResult::Passed => {
945                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
946                            minor <= v_minor + 1
947                        } else {
948                            true
949                        }
950                    }
951                    KanidmUpgradeCheckResult::Failed => {
952                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
953                            versions
954                        {
955                            major == v_major && minor == v_minor && patch >= v_patch
956                        } else {
957                            v.image_tag == current_tag
958                        }
959                    }
960                }
961            })
962            .unwrap_or(true)
963    }
964
965    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
966        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
967        let pod = pod_api.get(pod_name).await.map_err(|e| {
968            Error::KubeError(
969                format!("failed to get pod {namespace}/{pod_name}"),
970                Box::new(e),
971            )
972        })?;
973        Ok(pod
974            .status
975            .as_ref()
976            .and_then(|s| s.conditions.as_ref())
977            .is_some_and(|conditions| {
978                conditions
979                    .iter()
980                    .any(|c| c.type_ == "Ready" && c.status == "True")
981            }))
982    }
983
984    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
985        let start = std::time::Instant::now();
986        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
987        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
988
989        loop {
990            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
991                Ok(true) => return Ok(()),
992                Ok(false) => {}
993                Err(e) => {
994                    if !e.is_retryable() {
995                        return Err(e);
996                    }
997                    trace!(msg = "pod readiness check failed, retrying", %e);
998                }
999            }
1000
1001            if start.elapsed() >= timeout {
1002                return Err(Error::ReceiveOutput(format!(
1003                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1004                )));
1005            }
1006
1007            sleep(poll_interval).await;
1008        }
1009    }
1010
1011    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1012    where
1013        I: IntoIterator<Item = T> + Debug,
1014        T: Into<String>,
1015    {
1016        let namespace = &self.get_namespace();
1017        trace!(
1018            msg = "pod exec",
1019            resource.name = &pod_name,
1020            resource.namespace = &namespace,
1021            ?command
1022        );
1023        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1024        let attached = pod
1025            .exec(
1026                pod_name,
1027                command,
1028                &AttachParams::default().container("kanidm"),
1029            )
1030            .await
1031            .map_err(|e| {
1032                Error::KubeError(
1033                    format!("failed to exec pod {namespace}/{pod_name}"),
1034                    Box::new(e),
1035                )
1036            })?;
1037        get_output(attached).await
1038    }
1039
1040    async fn exec_with_wait<I, T>(
1041        &self,
1042        ctx: Arc<Context>,
1043        pod_name: &str,
1044        command: I,
1045    ) -> Result<String>
1046    where
1047        I: IntoIterator<Item = T> + Debug,
1048        T: Into<String>,
1049    {
1050        let namespace = &self.get_namespace();
1051        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1052        self.exec(ctx, pod_name, command).await
1053    }
1054
1055    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1056    where
1057        I: IntoIterator<Item = T> + Debug,
1058        T: Into<String>,
1059    {
1060        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1061        let first_replica_group = self
1062            .spec
1063            .replica_groups
1064            .first()
1065            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1066        let sts_name = self.statefulset_name(&first_replica_group.name);
1067        let pod_name = format!("{sts_name}-0");
1068        self.exec(ctx, &pod_name, command).await
1069    }
1070
1071    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1072    where
1073        I: IntoIterator<Item = T> + Debug,
1074        T: Into<String>,
1075    {
1076        let first_replica_group = self
1077            .spec
1078            .replica_groups
1079            .first()
1080            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1081        let sts_name = self.statefulset_name(&first_replica_group.name);
1082        let pod_name = format!("{sts_name}-0");
1083        self.exec_with_wait(ctx, &pod_name, command).await
1084    }
1085}
1086
1087#[cfg(test)]
1088mod test {
1089    use super::statefulset::StatefulSetExt;
1090    use super::{Kanidm, reconcile_kanidm};
1091
1092    use crate::controller::State;
1093    use crate::kanidm::controller::context::{Context, Stores};
1094    use crate::kanidm::crd::KanidmStatus;
1095
1096    use kaniop_k8s_util::error::Result;
1097
1098    use std::sync::Arc;
1099
1100    use http::{Request, Response};
1101    use k8s_openapi::api::apps::v1::StatefulSet;
1102    use k8s_openapi::api::core::v1::Service;
1103    use k8s_openapi::api::networking::v1::Ingress;
1104    use kube::runtime::reflector::store::Writer;
1105    use kube::{Client, Resource, ResourceExt, client::Body};
1106    use opentelemetry::metrics::MeterProvider;
1107    use serde_json::json;
1108
1109    impl Kanidm {
1110        /// A normal test kanidm with a given status
1111        pub fn test() -> Self {
1112            let mut e = Kanidm::new(
1113                "test",
1114                serde_json::from_value(json!({
1115                    "domain": "idm.example.com",
1116                    "replicaGroups": [
1117                        {
1118                            "name": "default",
1119                            "replicas": 1
1120                        }
1121                    ]
1122                }))
1123                .unwrap(),
1124            );
1125            e.meta_mut().namespace = Some("default".into());
1126            // Pre-populate finalizer so reconcile callback runs immediately
1127            // (skips the "add finalizer + wait for next reconcile" step)
1128            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1129            e
1130        }
1131
1132        pub fn with_ingress(mut self) -> Self {
1133            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1134            self
1135        }
1136
1137        /// Modify kanidm replicas
1138        pub fn with_replicas(mut self, replicas: i32) -> Self {
1139            self.spec.replica_groups[0].replicas = replicas;
1140            self
1141        }
1142
1143        /// Modify kanidm to set a deletion timestamp
1144        pub fn needs_delete(mut self) -> Self {
1145            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1146            use k8s_openapi::jiff::Timestamp;
1147            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1148            self.meta_mut().deletion_timestamp = Some(Time(now));
1149            self
1150        }
1151
1152        /// Modify a kanidm to have an expected status
1153        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1154            self.status = Some(status);
1155            self
1156        }
1157    }
1158
1159    // We wrap tower_test::mock::Handle
1160    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1161    pub struct ApiServerVerifier(ApiServerHandle);
1162
1163    /// Scenarios we test for in ApiServerVerifier
1164    pub enum Scenario {
1165        Create(Kanidm),
1166        CreateWithTwoReplicas(Kanidm),
1167        CreateWithIngress(Kanidm),
1168        CreateWithIngressWithTwoReplicas(Kanidm),
1169    }
1170
1171    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1172        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1173            .await
1174            .expect("timeout on mock apiserver")
1175            .expect("scenario succeeded")
1176    }
1177
1178    impl ApiServerVerifier {
1179        /// Tests only get to run specific scenarios that has matching handlers
1180        ///
1181        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1182        ///
1183        /// NB: If the controller is making more calls than we are handling in the scenario,
1184        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1185        ///
1186        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1187        /// scenario runs to completion (i.e. all expected calls were responded to),
1188        /// using the timeout to catch missing api calls to Kubernetes.
1189        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1190            tokio::spawn(async move {
1191                // moving self => one scenario per test
1192                match scenario {
1193                    Scenario::Create(kanidm) => {
1194                        self.handle_kanidm_get(kanidm.clone())
1195                            .await
1196                            .unwrap()
1197                            .handle_kanidm_status_patch(kanidm.clone())
1198                            .await
1199                            .unwrap()
1200                            .handle_statefulset_patch(kanidm.clone())
1201                            .await
1202                            .unwrap()
1203                            .handle_service_patch(kanidm.clone())
1204                            .await
1205                    }
1206                    Scenario::CreateWithTwoReplicas(kanidm) => {
1207                        self.handle_kanidm_get(kanidm.clone())
1208                            .await
1209                            .unwrap()
1210                            .handle_kanidm_status_patch(kanidm.clone())
1211                            .await
1212                            .unwrap()
1213                            .handle_statefulset_patch(kanidm.clone())
1214                            .await
1215                            .unwrap()
1216                            .handle_service_patch(kanidm.clone())
1217                            .await
1218                    }
1219                    Scenario::CreateWithIngress(kanidm) => {
1220                        self.handle_kanidm_get(kanidm.clone())
1221                            .await
1222                            .unwrap()
1223                            .handle_kanidm_status_patch(kanidm.clone())
1224                            .await
1225                            .unwrap()
1226                            .handle_statefulset_patch(kanidm.clone())
1227                            .await
1228                            .unwrap()
1229                            .handle_service_patch(kanidm.clone())
1230                            .await
1231                            .unwrap()
1232                            .handle_ingress_patch(kanidm.clone())
1233                            .await
1234                    }
1235                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1236                        self.handle_kanidm_get(kanidm.clone())
1237                            .await
1238                            .unwrap()
1239                            .handle_kanidm_status_patch(kanidm.clone())
1240                            .await
1241                            .unwrap()
1242                            .handle_statefulset_patch(kanidm.clone())
1243                            .await
1244                            .unwrap()
1245                            .handle_service_patch(kanidm.clone())
1246                            .await
1247                            .unwrap()
1248                            .handle_ingress_patch(kanidm.clone())
1249                            .await
1250                    }
1251                }
1252                .expect("scenario completed without errors");
1253            })
1254        }
1255
1256        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1257            let (request, send) = self.0.next_request().await.expect("service not called");
1258            assert_eq!(request.method(), http::Method::GET);
1259            let expected_uri_prefix = format!(
1260                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1261                kanidm.name_any()
1262            );
1263            let uri = request.uri().to_string();
1264            assert!(
1265                uri.starts_with(&expected_uri_prefix),
1266                "expected uri to start with {}, got {}",
1267                expected_uri_prefix,
1268                uri
1269            );
1270            let response = serde_json::to_vec(&kanidm).unwrap();
1271            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1272            Ok(self)
1273        }
1274
1275        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1276            let (request, send) = self.0.next_request().await.expect("service not called");
1277            assert_eq!(request.method(), http::Method::PATCH);
1278            assert_eq!(
1279                request.uri().to_string(),
1280                format!(
1281                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1282                    kanidm.name_any()
1283                )
1284            );
1285
1286            let req_body = request.into_body().collect_bytes().await.unwrap();
1287            let json: serde_json::Value =
1288                serde_json::from_slice(&req_body).expect("patch object is json");
1289            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1290                .expect("valid kanidm status");
1291            let mut kanidm = Kanidm::test();
1292            kanidm.status = Some(status.clone());
1293            let response = serde_json::to_vec(&kanidm).unwrap();
1294            // pass through kanidm "patch accepted"
1295            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1296            Ok(self)
1297        }
1298
1299        async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1300            for rg in kanidm.spec.replica_groups.iter() {
1301                let (request, send) = self.0.next_request().await.expect("service not called");
1302                assert_eq!(request.method(), http::Method::PATCH);
1303                assert_eq!(
1304                    request.uri().to_string(),
1305                    format!(
1306                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1307                        kanidm.statefulset_name(&rg.name)
1308                    )
1309                );
1310                let req_body = request.into_body().collect_bytes().await.unwrap();
1311                let json: serde_json::Value =
1312                    serde_json::from_slice(&req_body).expect("patch object is json");
1313                let statefulset: StatefulSet =
1314                    serde_json::from_value(json).expect("valid statefulset");
1315                assert_eq!(
1316                    statefulset.clone().spec.unwrap().replicas.unwrap(),
1317                    rg.replicas
1318                );
1319                let response = serde_json::to_vec(&statefulset).unwrap();
1320                // pass through kanidm "patch accepted"
1321                send.send_response(Response::builder().body(Body::from(response)).unwrap());
1322            }
1323
1324            Ok(self)
1325        }
1326
1327        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1328            let (request, send) = self.0.next_request().await.expect("service not called");
1329            assert_eq!(request.method(), http::Method::PATCH);
1330            assert_eq!(
1331                request.uri().to_string(),
1332                format!(
1333                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1334                    kanidm.name_any()
1335                )
1336            );
1337
1338            let req_body = request.into_body().collect_bytes().await.unwrap();
1339            let json: serde_json::Value =
1340                serde_json::from_slice(&req_body).expect("patch object is json");
1341            let service: Service = serde_json::from_value(json).expect("valid service");
1342            let response = serde_json::to_vec(&service).unwrap();
1343            // pass through kanidm "patch accepted"
1344            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1345            Ok(self)
1346        }
1347
1348        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1349            let (request, send) = self.0.next_request().await.expect("service not called");
1350            assert_eq!(request.method(), http::Method::PATCH);
1351            assert_eq!(
1352                request.uri().to_string(),
1353                format!(
1354                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1355                    kanidm.name_any()
1356                )
1357            );
1358
1359            let req_body = request.into_body().collect_bytes().await.unwrap();
1360            let json: serde_json::Value =
1361                serde_json::from_slice(&req_body).expect("patch object is json");
1362            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1363            let response = serde_json::to_vec(&ingress).unwrap();
1364            // pass through kanidm "patch accepted"
1365            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1366            Ok(self)
1367        }
1368    }
1369
1370    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1371        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1372        let mock_client = Client::new(mock_service, "default");
1373        let stores = Stores {
1374            stateful_set_store: Writer::default().as_reader(),
1375            service_store: Writer::default().as_reader(),
1376            ingress_store: Writer::default().as_reader(),
1377            secret_store: Writer::default().as_reader(),
1378            http_route_store: Some(Writer::default().as_reader()),
1379            backend_tls_policy_store: Some(Writer::default().as_reader()),
1380            deployment_store: Writer::default().as_reader(),
1381            config_map_store: Writer::default().as_reader(),
1382        };
1383        let controller_id = "test";
1384
1385        // Create a test meter for metrics
1386        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
1387        let meter = provider.meter("test");
1388        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
1389
1390        let state = State::new(
1391            metrics,
1392            Writer::default().as_reader(),
1393            Writer::default().as_reader(),
1394            None,
1395        );
1396        let ctx = Arc::new(Context::new(
1397            state.to_context(mock_client, controller_id),
1398            stores,
1399        ));
1400        (ctx, ApiServerVerifier(handle))
1401    }
1402
1403    #[tokio::test]
1404    async fn kanidm_create() {
1405        let (testctx, fakeserver) = get_test_context();
1406        let kanidm = Kanidm::test();
1407        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
1408        reconcile_kanidm(Arc::new(kanidm), testctx)
1409            .await
1410            .expect("reconciler");
1411        timeout_after_1s(mocksrv).await;
1412    }
1413
1414    #[tokio::test]
1415    async fn kanidm_create_with_two_replicas() {
1416        let (testctx, fakeserver) = get_test_context();
1417        let kanidm = Kanidm::test().with_replicas(2);
1418        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
1419        reconcile_kanidm(Arc::new(kanidm), testctx)
1420            .await
1421            .expect("reconciler");
1422        timeout_after_1s(mocksrv).await;
1423    }
1424
1425    #[tokio::test]
1426    async fn kanidm_create_with_ingress() {
1427        let (testctx, fakeserver) = get_test_context();
1428        let kanidm = Kanidm::test().with_ingress();
1429        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
1430        reconcile_kanidm(Arc::new(kanidm), testctx)
1431            .await
1432            .expect("reconciler");
1433        timeout_after_1s(mocksrv).await;
1434    }
1435
1436    #[tokio::test]
1437    async fn kanidm_create_with_ingress_with_two_replicas() {
1438        let (testctx, fakeserver) = get_test_context();
1439        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
1440        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
1441        reconcile_kanidm(Arc::new(kanidm), testctx)
1442            .await
1443            .expect("reconciler");
1444        timeout_after_1s(mocksrv).await;
1445    }
1446}