Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::{
21    StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
22    preserve_defaulted_statefulset_fields,
23};
24use self::status::StatusExt;
25use self::status::{is_kanidm_available, is_kanidm_initialized};
26
27use crate::controller::context::KubeOperations;
28use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
29use crate::kanidm::crd::{
30    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
31};
32use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
33use crate::telemetry;
34
35use kaniop_k8s_util::client::get_output;
36use kaniop_k8s_util::error::{Error, Result};
37use kaniop_k8s_util::parse::parse_semver;
38use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
39
40use std::collections::{BTreeMap, BTreeSet};
41use std::fmt::Debug;
42use std::sync::{Arc, LazyLock};
43
44use futures::future::{TryJoinAll, join_all, try_join_all};
45use futures::stream::{self, StreamExt};
46use futures::try_join;
47use k8s_openapi::NamespaceResourceScope;
48use k8s_openapi::api::apps::v1::StatefulSet;
49use k8s_openapi::api::core::v1::{Pod, Secret, Service};
50use kube::Resource;
51use kube::ResourceExt;
52use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
53use kube::runtime::controller::Action;
54use kube::runtime::events::{Event, EventType};
55use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
56use serde::{Deserialize, Serialize};
57use tokio::time::{Duration, sleep};
58use tracing::{Span, debug, field, info, instrument, trace, warn};
59
60const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
61const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
62const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
63const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
64const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
65const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
66const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
67const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
68const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
69
70pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
71const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
72pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
73
74const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
75
76static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
77    BTreeMap::from([
78        (NAME_LABEL.to_string(), "kanidm".to_string()),
79        (
80            MANAGED_BY_LABEL.to_string(),
81            format!("kaniop-{CONTROLLER_ID}"),
82        ),
83    ])
84});
85
86async fn restart_statefulsets_after_replica_secret_update(
87    ctx: Arc<Context>,
88    sts_api: &Api<StatefulSet>,
89    namespace: &str,
90    statefulset_names: &[String],
91    sequential: bool,
92) -> Result<()> {
93    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
94    // Pods must restart after cert secret changes or replication keeps stale TLS material.
95    let restart_futures = statefulset_names
96        .iter()
97        .map(|sts_name| sts_api.restart(sts_name));
98
99    if sequential {
100        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
101        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
102            return Err(Error::kube_error(
103                "restart",
104                "StatefulSet",
105                namespace,
106                "replica certificate secret update statefulsets",
107                first_err,
108            ));
109        }
110    } else {
111        let results: Vec<_> = join_all(restart_futures).await;
112        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
113            return Err(Error::kube_error(
114                "restart",
115                "StatefulSet",
116                namespace,
117                "replica certificate secret update statefulsets",
118                first_err,
119            ));
120        }
121    }
122
123    let rollout_futures = statefulset_names
124        .iter()
125        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
126    try_join_all(rollout_futures).await?;
127
128    Ok(())
129}
130
131pub async fn reconcile_admins_secret(
132    kanidm: Arc<Kanidm>,
133    ctx: Arc<Context>,
134    status: &KanidmStatus,
135) -> Result<()> {
136    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
137        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
138        kanidm.patch(&ctx, admins_secret).await?;
139    }
140    Ok(())
141}
142
143pub async fn reconcile_replication_secrets(
144    kanidm: Arc<Kanidm>,
145    ctx: Arc<Context>,
146    status: &KanidmStatus,
147) -> Result<()> {
148    let expected_secret_names = if kanidm.is_replication_enabled() {
149        status
150            .replica_statuses
151            .iter()
152            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
153            .collect::<Vec<_>>()
154    } else {
155        Vec::new()
156    };
157    let deprecated_secrets = ctx
158        .stores
159        .secret_store
160        .state()
161        .into_iter()
162        .filter(|secret| {
163            secret.namespace() == kanidm.namespace()
164                && kanidm.admins_secret_name() != secret.name_any()
165                && !expected_secret_names.contains(&secret.name_any())
166                && secret
167                    .metadata
168                    .labels
169                    .as_ref()
170                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
171        })
172        .collect::<Vec<_>>();
173    let secret_delete_future = deprecated_secrets
174        .iter()
175        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
176        .collect::<TryJoinAll<_>>();
177    try_join!(secret_delete_future)?;
178
179    if kanidm.is_replication_enabled() {
180        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
181        let statefulset_names = status
182            .replica_statuses
183            .iter()
184            .map(|rs| rs.statefulset_name.clone())
185            .collect::<BTreeSet<_>>()
186            .into_iter()
187            .collect::<Vec<_>>();
188
189        let has_pending = status
190            .replica_statuses
191            .iter()
192            .any(|rs| rs.state == KanidmReplicaState::Pending);
193
194        let sts_api =
195            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
196
197        if has_pending {
198            stream::iter(
199                status
200                    .replica_statuses
201                    .iter()
202                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
203            )
204            .then(|rs| async {
205                let secret = kanidm
206                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
207                    .await?;
208                kanidm.patch(&ctx, secret).await
209            })
210            .collect::<Vec<_>>()
211            .await
212            .into_iter()
213            .collect::<Result<Vec<_>>>()?;
214
215            let restart_futures = status
216                .replica_statuses
217                .iter()
218                .map(|rs| sts_api.restart(&rs.statefulset_name));
219
220            if has_single_replica {
221                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
222                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
223                    return Err(Error::kube_error(
224                        "restart",
225                        "StatefulSet",
226                        kanidm.get_namespace(),
227                        "pending replica statefulsets",
228                        first_err,
229                    ));
230                }
231            } else {
232                let results: Vec<_> = join_all(restart_futures).await;
233                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
234                    return Err(Error::kube_error(
235                        "restart",
236                        "StatefulSet",
237                        kanidm.get_namespace(),
238                        "pending replica statefulsets",
239                        first_err,
240                    ));
241                }
242            }
243        }
244
245        let has_certificate_host_invalid = status
246            .replica_statuses
247            .iter()
248            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
249
250        let has_certificate_expiring = status
251            .replica_statuses
252            .iter()
253            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
254
255        let has_non_ready_replicas = status
256            .replica_statuses
257            .iter()
258            .any(|rs| rs.state != KanidmReplicaState::Ready);
259
260        let host_invalid_replicas: Vec<_> = status
261            .replica_statuses
262            .iter()
263            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
264            .collect();
265
266        let cert_expiring_replicas: Vec<_> = status
267            .replica_statuses
268            .iter()
269            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
270            .collect();
271
272        if !host_invalid_replicas.is_empty() {
273            let restart_futures = host_invalid_replicas
274                .iter()
275                .map(|rs| sts_api.restart(&rs.statefulset_name));
276
277            if has_single_replica {
278                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
279                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
280                    return Err(Error::kube_error(
281                        "restart",
282                        "StatefulSet",
283                        kanidm.get_namespace(),
284                        "certificate host invalid statefulsets",
285                        first_err,
286                    ));
287                }
288            } else {
289                let results: Vec<_> = join_all(restart_futures).await;
290                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
291                    return Err(Error::kube_error(
292                        "restart",
293                        "StatefulSet",
294                        kanidm.get_namespace(),
295                        "certificate host invalid statefulsets",
296                        first_err,
297                    ));
298                }
299            }
300
301            let restarted_sts_names: Vec<_> = host_invalid_replicas
302                .iter()
303                .map(|rs| rs.statefulset_name.clone())
304                .collect::<BTreeSet<_>>()
305                .into_iter()
306                .collect();
307
308            let namespace = kanidm.get_namespace();
309            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
310                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
311            });
312            try_join_all(rollout_futures).await?;
313
314            let renew_futures = host_invalid_replicas
315                .iter()
316                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
317            let results: Vec<_> = join_all(renew_futures).await;
318            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
319                return Err(first_err);
320            }
321
322            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
323
324            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
325                let cert = show_replica_cert_with_retries(
326                    &kanidm,
327                    ctx.clone(),
328                    &rs.pod_name,
329                    CERT_SHOW_RETRY_ATTEMPTS,
330                )
331                .await?;
332                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
333                kanidm.patch(&ctx, secret.clone()).await
334            });
335            try_join_all(cert_futures).await?;
336
337            restart_statefulsets_after_replica_secret_update(
338                ctx.clone(),
339                &sts_api,
340                &kanidm.get_namespace(),
341                &statefulset_names,
342                has_single_replica,
343            )
344            .await?;
345
346            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
347        }
348
349        if !cert_expiring_replicas.is_empty() {
350            let renew_futures = cert_expiring_replicas
351                .iter()
352                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
353            let results: Vec<_> = join_all(renew_futures).await;
354            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
355                return Err(first_err);
356            }
357
358            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
359
360            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
361                let cert = show_replica_cert_with_retries(
362                    &kanidm,
363                    ctx.clone(),
364                    &rs.pod_name,
365                    CERT_SHOW_RETRY_ATTEMPTS,
366                )
367                .await?;
368                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
369                kanidm.patch(&ctx, secret.clone()).await
370            });
371            try_join_all(cert_futures).await?;
372
373            restart_statefulsets_after_replica_secret_update(
374                ctx.clone(),
375                &sts_api,
376                &kanidm.get_namespace(),
377                &statefulset_names,
378                has_single_replica,
379            )
380            .await?;
381
382            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
383        }
384
385        if has_non_ready_replicas
386            && !has_pending
387            && !has_certificate_host_invalid
388            && !has_certificate_expiring
389        {
390            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
391
392            let restart_futures = status
393                .replica_statuses
394                .iter()
395                .map(|rs| sts_api.restart(&rs.statefulset_name));
396
397            if has_single_replica {
398                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
399                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
400                    return Err(Error::kube_error(
401                        "restart",
402                        "StatefulSet",
403                        kanidm.get_namespace(),
404                        "replica statefulsets",
405                        first_err,
406                    ));
407                }
408            } else {
409                let results: Vec<_> = join_all(restart_futures).await;
410                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
411                    return Err(Error::kube_error(
412                        "restart",
413                        "StatefulSet",
414                        kanidm.get_namespace(),
415                        "replica statefulsets",
416                        first_err,
417                    ));
418                }
419            }
420        }
421    }
422
423    Ok(())
424}
425
426#[instrument(skip(ctx, kanidm))]
427pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
428    let trace_id = telemetry::get_trace_id();
429    Span::current().record("trace_id", field::display(&trace_id));
430    let _timer = ctx
431        .kaniop_ctx
432        .metrics
433        .reconcile_count_and_measure(&trace_id);
434    info!(msg = "reconciling Kanidm");
435
436    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
437        debug!(msg = "failed to reconcile status", %e);
438        ctx.kaniop_ctx.metrics.status_update_errors_inc();
439        e
440    })?;
441    let kanidm_api: Api<Kanidm> =
442        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
443    finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
444        match event {
445            Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
446            Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
447        }
448    })
449    .await
450    .or_else(|e| match e {
451        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
452            debug!(msg = "resource already removed during finalizer cleanup");
453            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
454        }
455        _ => Err(Error::FinalizerError(
456            "failed on kanidm account finalizer".to_string(),
457            Box::new(e),
458        )),
459    })
460}
461
462/// Hash of the TLS secret content, injected as a pod template annotation so
463/// certificate renewals roll the statefulsets. `None` if the secret does not
464/// exist yet.
465async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
466    let secret_name = kanidm.effective_tls_secret_name();
467    let secret_api =
468        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
469    let secret = secret_api
470        .get_opt(&secret_name)
471        .await
472        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
473    Ok(secret.as_ref().map(hash_secret_data))
474}
475
476/// Previous TLS secret hash annotation from the existing statefulsets.
477///
478/// Used when the TLS secret is missing: dropping the annotation would roll
479/// the pods onto a missing volume and leave them stuck in ContainerCreating.
480fn previous_tls_secret_hash(
481    kanidm: &Kanidm,
482    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
483) -> Option<String> {
484    stateful_sets
485        .into_iter()
486        .filter(|sts| {
487            sts.namespace() == kanidm.namespace()
488                && sts
489                    .metadata
490                    .labels
491                    .as_ref()
492                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
493        })
494        .find_map(|sts| {
495            sts.spec
496                .as_ref()?
497                .template
498                .metadata
499                .as_ref()?
500                .annotations
501                .as_ref()?
502                .get(TLS_SECRET_HASH_ANNOTATION)
503                .cloned()
504        })
505}
506
507fn is_headless_service(service: &Service) -> bool {
508    service
509        .spec
510        .as_ref()
511        .and_then(|spec| spec.cluster_ip.as_deref())
512        == Some("None")
513}
514
515fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
516    if is_headless_service(desired) != is_headless_service(current) {
517        return;
518    }
519    let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
520    else {
521        return;
522    };
523    if desired_spec.cluster_ip.is_none() {
524        desired_spec.cluster_ip = current_spec.cluster_ip.clone();
525    }
526    if desired_spec.cluster_ips.is_none() {
527        desired_spec.cluster_ips = current_spec.cluster_ips.clone();
528    }
529}
530
531async fn wait_for_statefulset_deletion(
532    client: kube::Client,
533    namespace: &str,
534    name: &str,
535) -> Result<()> {
536    let api = Api::<StatefulSet>::namespaced(client, namespace);
537    let start = std::time::Instant::now();
538    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
539    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
540
541    loop {
542        match api.get_opt(name).await {
543            Ok(None) => return Ok(()),
544            Ok(Some(_)) => {}
545            Err(e) => {
546                return Err(Error::kube_error(
547                    "wait for deletion of",
548                    "StatefulSet",
549                    namespace,
550                    name,
551                    e,
552                ));
553            }
554        }
555
556        if start.elapsed() >= timeout {
557            return Err(Error::ReceiveOutput(format!(
558                "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
559            )));
560        }
561        sleep(poll_interval).await;
562    }
563}
564
565async fn reconcile_service(
566    kanidm: &Kanidm,
567    ctx: &Context,
568    mut desired: Service,
569) -> Result<Service> {
570    let namespace = kanidm.get_namespace();
571    let name = desired.name_any();
572    let desired_headless = is_headless_service(&desired);
573
574    if let Some(cached) = ctx
575        .stores
576        .service_store
577        .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
578    {
579        if is_headless_service(cached.as_ref()) == desired_headless {
580            preserve_defaulted_service_fields(&mut desired, cached.as_ref());
581        } else {
582            let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
583            let current = service_api
584                .get_opt(&name)
585                .await
586                .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
587
588            if let Some(current) = current {
589                let current_headless = is_headless_service(&current);
590                if current_headless != desired_headless {
591                    info!(
592                        msg = "recreating Service to change headless mode",
593                        resource.name = &name,
594                        resource.namespace = &namespace,
595                        current_headless,
596                        desired_headless,
597                    );
598                    kanidm.delete(ctx, &current).await?;
599                    ctx.kaniop_ctx
600                        .metrics
601                        .reconcile_deploy_delete_create_inc("Service", "headless_transition");
602                    wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
603                        .await?;
604                } else {
605                    preserve_defaulted_service_fields(&mut desired, &current);
606                }
607            }
608        }
609    }
610
611    kanidm.patch(ctx, desired).await
612}
613
614async fn wait_for_service_deletion(
615    client: kube::Client,
616    namespace: &str,
617    name: &str,
618) -> Result<()> {
619    let api = Api::<Service>::namespaced(client, namespace);
620    let start = std::time::Instant::now();
621    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
622    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
623
624    loop {
625        match api.get_opt(name).await {
626            Ok(None) => return Ok(()),
627            Ok(Some(_)) => {}
628            Err(e) => {
629                return Err(Error::kube_error(
630                    "wait for deletion of",
631                    "Service",
632                    namespace,
633                    name,
634                    e,
635                ));
636            }
637        }
638
639        if start.elapsed() >= timeout {
640            return Err(Error::ReceiveOutput(format!(
641                "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
642            )));
643        }
644        sleep(poll_interval).await;
645    }
646}
647
648async fn validate_statefulset_replacement(
649    api: &Api<StatefulSet>,
650    desired: &StatefulSet,
651    namespace: &str,
652    name: &str,
653) -> Result<()> {
654    let mut candidate = desired.clone();
655    candidate.metadata.name = None;
656    candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
657    candidate.metadata.namespace = None;
658    candidate.metadata.resource_version = None;
659    candidate.metadata.uid = None;
660    candidate.metadata.creation_timestamp = None;
661    candidate.metadata.generation = None;
662    candidate.metadata.managed_fields = None;
663    candidate.metadata.deletion_timestamp = None;
664    candidate.metadata.deletion_grace_period_seconds = None;
665    candidate.status = None;
666
667    api.create(
668        &PostParams {
669            dry_run: true,
670            ..Default::default()
671        },
672        &candidate,
673    )
674    .await
675    .map(|_| ())
676    .map_err(|e| {
677        Error::kube_error(
678            "validate replacement for",
679            "StatefulSet",
680            namespace,
681            name,
682            e,
683        )
684    })
685}
686
687async fn reconcile_statefulset(
688    kanidm: Arc<Kanidm>,
689    ctx: Arc<Context>,
690    desired: StatefulSet,
691) -> Result<StatefulSet> {
692    let namespace = kanidm.namespace().unwrap();
693    let name = desired.name_any();
694    let cached_current = ctx.stores.stateful_set_store.find(|current| {
695        current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
696    });
697
698    // On a cold reflector cache, keep the normal create/update path GET-free. If the optimistic
699    // apply fails with HTTP 422, the object may already exist with an immutable conflict that the
700    // cache has not observed yet. Only 422 errors enter the live classification/recreate recovery
701    // path; 429, 5xx, 403, and transport errors are returned immediately and non-destructively.
702    let cache_miss_422_error = if let Some(cached_current) = cached_current {
703        let mut cached_desired = desired.clone();
704        preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
705        if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
706            == StatefulSetApplyStrategy::Apply
707        {
708            return kanidm.apply(&ctx, cached_desired).await;
709        }
710        None
711    } else {
712        match kanidm.apply(&ctx, desired.clone()).await {
713            Ok(applied) => return Ok(applied),
714            Err(error) => {
715                // Only HTTP 422 can indicate an immutable-field conflict that is safe to
716                // recover from via delete/recreate. Arbitrary failures must not trigger
717                // destruction.
718                if is_kube_422(&error) {
719                    Some(error)
720                } else {
721                    return Err(error);
722                }
723            }
724        }
725    };
726
727    // Never make a destructive decision from reflector state or an SSA error alone. Re-read the
728    // live object and classify it immediately before deletion; another actor or an earlier
729    // reconcile may already have converged the immutable fields while the cache was catching up.
730    let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
731    let Some(live_current) = api
732        .get_opt(&name)
733        .await
734        .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
735    else {
736        return match cache_miss_422_error {
737            Some(error) => Err(error),
738            None => kanidm.apply(&ctx, desired).await,
739        };
740    };
741
742    // Start from the original desired object again. Defaults copied from a stale reflector entry
743    // must not survive into the live re-check and become the reason for a recreation.
744    let mut live_desired = desired.clone();
745    preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
746    match classify_statefulset_change(&live_current, &live_desired) {
747        StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
748            Some(error) => Err(error),
749            None => kanidm.apply(&ctx, live_desired).await,
750        },
751        StatefulSetApplyStrategy::Recreate { .. } => {
752            // An update 422 can combine an immutable conflict with independently-invalid desired
753            // state. Validate the replacement as a dry-run CREATE before deleting the live object,
754            // so create-time validation runs without the existing object's immutable-update check.
755            validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
756
757            // Re-read and reclassify immediately before destruction. Another actor or an earlier
758            // reconcile may have changed the live immutable fields or PVC-retention semantics.
759            let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
760                Error::kube_error(
761                    "re-read before deletion",
762                    "StatefulSet",
763                    &namespace,
764                    &name,
765                    e,
766                )
767            })?
768            else {
769                return kanidm.apply(&ctx, live_desired).await;
770            };
771            let mut latest_desired = live_desired.clone();
772            preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
773
774            match classify_statefulset_change(&latest_current, &latest_desired) {
775                StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
776                StatefulSetApplyStrategy::Recreate { immutable_fields } => {
777                    // Validate the replacement StatefulSet via a dry-run CREATE before deleting
778                    // the live object. This prevents deleting on a mixed 422 (immutable conflict
779                    // + independently-invalid desired state) where the replacement would still
780                    // be rejected by the API.
781                    validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
782                        .await?;
783
784                    info!(
785                        msg = "recreating StatefulSet because immutable fields changed",
786                        resource.name = %name,
787                        resource.namespace = %namespace,
788                        ?immutable_fields,
789                    );
790                    kanidm.delete(&ctx, &latest_current).await?;
791                    ctx.kaniop_ctx
792                        .metrics
793                        .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
794                    wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
795                        .await?;
796                    kanidm.apply(&ctx, latest_desired).await
797                }
798            }
799        }
800    }
801}
802
803/// Return true only when the error structurally wraps a Kubernetes HTTP 422 response.
804fn is_kube_422(error: &Error) -> bool {
805    matches!(
806        error,
807        Error::KubeError(_, cause)
808            if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
809    )
810}
811
812#[cfg(test)]
813mod statefulset_recreate_recovery_tests {
814    use super::{Error, is_kube_422};
815
816    fn kube_api_error(code: u16) -> Error {
817        Error::KubeError(
818            "test error".to_string(),
819            Box::new(kube::Error::Api(Box::new(kube::error::Status {
820                code,
821                message: "test".to_string(),
822                reason: "test".to_string(),
823                ..Default::default()
824            }))),
825        )
826    }
827
828    #[test]
829    fn recreate_recovery_accepts_only_422() {
830        assert!(is_kube_422(&kube_api_error(422)));
831        assert!(!is_kube_422(&kube_api_error(403)));
832        assert!(!is_kube_422(&kube_api_error(429)));
833        assert!(!is_kube_422(&kube_api_error(500)));
834    }
835
836    #[test]
837    fn recreate_recovery_rejects_429_too_many_requests() {
838        assert!(!is_kube_422(&kube_api_error(429)));
839    }
840
841    #[test]
842    fn recreate_recovery_rejects_5xx_server_errors() {
843        assert!(!is_kube_422(&kube_api_error(500)));
844        assert!(!is_kube_422(&kube_api_error(502)));
845        assert!(!is_kube_422(&kube_api_error(503)));
846        assert!(!is_kube_422(&kube_api_error(504)));
847    }
848
849    #[test]
850    fn recreate_recovery_rejects_400_and_403() {
851        assert!(!is_kube_422(&kube_api_error(400)));
852        assert!(!is_kube_422(&kube_api_error(403)));
853    }
854}
855
856async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
857    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
858    let replication_secret_futures =
859        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
860
861    let sts_to_delete = {
862        let expected_sts_names = kanidm
863            .spec
864            .replica_groups
865            .iter()
866            .map(|rg| kanidm.statefulset_name(&rg.name))
867            .collect::<Vec<_>>();
868        ctx.stores
869            .stateful_set_store
870            .state()
871            .into_iter()
872            .filter(|sts| {
873                sts.namespace() == kanidm.namespace()
874                    && sts
875                        .metadata
876                        .labels
877                        .as_ref()
878                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
879                    && !expected_sts_names.contains(&sts.name_any())
880            })
881            .collect::<Vec<_>>()
882    };
883    let sts_delete_futures = sts_to_delete
884        .iter()
885        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
886        .collect::<TryJoinAll<_>>();
887
888    let sts_futures = match kanidm.is_updatable(&status) {
889        true => {
890            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
891                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
892            });
893            kanidm
894                .spec
895                .replica_groups
896                .iter()
897                .map(|rg| {
898                    let sts = kanidm.create_statefulset(rg, tls_secret_hash.as_deref())?;
899                    Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
900                })
901                .collect::<Result<TryJoinAll<_>, _>>()?
902        }
903        false => {
904            let note = match status.version.as_ref() {
905                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
906                    format!(
907                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
908                        v.image_tag,
909                        crate::version::KANIDM_CLIENT_VERSION
910                    )
911                }
912                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
913            };
914            let _ignore_error = ctx
915                .kaniop_ctx
916                .recorder
917                .publish(
918                    &Event {
919                        type_: EventType::Warning,
920                        reason: "UpgradeBlocked".to_string(),
921                        note: Some(note),
922                        action: "ReconcileStatefulSet".to_string(),
923                        secondary: None,
924                    },
925                    &kanidm.object_ref(&()),
926                )
927                .await
928                .map_err(|e| {
929                    warn!(msg = "failed to publish KanidmError event", %e);
930                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
931                });
932            try_join_all([])
933        }
934    };
935    let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
936
937    let deprecated_rg_svcs = {
938        let expected_rg_svcs_names = kanidm
939            .spec
940            .replica_groups
941            .iter()
942            .filter(|rg| rg.services.is_some())
943            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
944            .collect::<Vec<_>>();
945        ctx.stores
946            .service_store
947            .state()
948            .into_iter()
949            .filter(|svc| {
950                let labels = svc.metadata.labels.as_ref();
951                svc.namespace() == kanidm.namespace()
952                    && !expected_rg_svcs_names.contains(&svc.name_any())
953                    && labels.is_some_and(|l| {
954                        l.contains_key(REPLICA_LABEL)
955                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
956                    })
957            })
958            .collect::<Vec<_>>()
959    };
960    let rg_svcs_delete_futures = deprecated_rg_svcs
961        .iter()
962        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
963        .collect::<TryJoinAll<_>>();
964
965    let rg_services_futures = kanidm
966        .spec
967        .replica_groups
968        .iter()
969        .filter(|rg| rg.services.is_some())
970        .flat_map(|rg| {
971            (0..rg.replicas).map(|i| {
972                let mut svc = kanidm.create_replica_group_service(rg, i);
973                if let Some(current) = ctx.stores.service_store.find(|current| {
974                    current.namespace() == kanidm.namespace()
975                        && current.name_any() == svc.name_any()
976                }) {
977                    preserve_defaulted_service_fields(&mut svc, current.as_ref());
978                }
979                kanidm.patch(&ctx, svc)
980            })
981        })
982        .collect::<TryJoinAll<_>>();
983
984    let deprecated_ingresses = {
985        let names = [
986            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
987            kanidm.generate_region_ingress_name(),
988        ]
989        .into_iter()
990        .flatten()
991        .collect::<Vec<_>>();
992        ctx.stores
993            .ingress_store
994            .state()
995            .into_iter()
996            .filter(|ing| {
997                ing.namespace() == kanidm.namespace()
998                    && !names.contains(&ing.name_any())
999                    && ing.metadata.labels == Some(kanidm.generate_labels())
1000            })
1001            .collect::<Vec<_>>()
1002    };
1003    let ingress_delete_futures = deprecated_ingresses
1004        .iter()
1005        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1006        .collect::<TryJoinAll<_>>();
1007
1008    let ingress_futures = kanidm
1009        .create_ingress()
1010        .into_iter()
1011        .chain(kanidm.create_region_ingress())
1012        .map(|ingress| kanidm.patch(&ctx, ingress))
1013        .collect::<TryJoinAll<_>>();
1014
1015    let deprecated_http_routes = match &ctx.stores.http_route_store {
1016        Some(store) => {
1017            let expected_names = kanidm
1018                .spec
1019                .gateway
1020                .as_ref()
1021                .map(|_| vec![kanidm.name_any()])
1022                .unwrap_or_default();
1023            store
1024                .state()
1025                .into_iter()
1026                .filter(|route| {
1027                    route.namespace() == kanidm.namespace()
1028                        && !expected_names.contains(&route.name_any())
1029                        && route.metadata.labels == Some(kanidm.generate_labels())
1030                })
1031                .collect::<Vec<_>>()
1032        }
1033        None => Vec::new(),
1034    };
1035    let http_route_delete_futures = deprecated_http_routes
1036        .iter()
1037        .map(|route| kanidm.delete(&ctx, route.as_ref()))
1038        .collect::<TryJoinAll<_>>();
1039
1040    let http_route_futures = if ctx.stores.http_route_store.is_some() {
1041        kanidm
1042            .create_http_route()
1043            .into_iter()
1044            .map(|route| kanidm.patch(&ctx, route))
1045            .collect::<TryJoinAll<_>>()
1046    } else {
1047        try_join_all(Vec::new())
1048    };
1049
1050    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1051        Some(store) => {
1052            let expected_names = kanidm
1053                .spec
1054                .gateway
1055                .as_ref()
1056                .and_then(|g| g.backend_tls_policy.as_ref())
1057                .map(|_| vec![kanidm.name_any()])
1058                .unwrap_or_default();
1059            store
1060                .state()
1061                .into_iter()
1062                .filter(|policy| {
1063                    policy.namespace() == kanidm.namespace()
1064                        && !expected_names.contains(&policy.name_any())
1065                        && policy.metadata.labels == Some(kanidm.generate_labels())
1066                })
1067                .collect::<Vec<_>>()
1068        }
1069        None => Vec::new(),
1070    };
1071    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1072        .iter()
1073        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1074        .collect::<TryJoinAll<_>>();
1075
1076    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1077        kanidm
1078            .create_backend_tls_policy()
1079            .into_iter()
1080            .map(|policy| kanidm.patch(&ctx, policy))
1081            .collect::<TryJoinAll<_>>()
1082    } else {
1083        try_join_all(Vec::new())
1084    };
1085
1086    try_join!(
1087        sts_delete_futures,
1088        admin_secret_future,
1089        replication_secret_futures,
1090        sts_futures,
1091        service_future,
1092        rg_svcs_delete_futures,
1093        rg_services_futures,
1094        ingress_delete_futures,
1095        ingress_futures,
1096        http_route_delete_futures,
1097        http_route_futures,
1098        backend_tls_policy_delete_futures,
1099        backend_tls_policy_futures
1100    )?;
1101
1102    if is_kanidm_available(status.clone()) {
1103        let namespace = kanidm.namespace().unwrap();
1104        let name = kanidm.name_any();
1105        let system_client = crate::controller::kanidm::KanidmClients::create_client(
1106            &namespace,
1107            &name,
1108            crate::controller::kanidm::KanidmUser::Admin,
1109            ctx.kaniop_ctx.client.clone(),
1110        )
1111        .await?;
1112        reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
1113
1114        let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
1115            &namespace,
1116            &name,
1117            crate::controller::kanidm::KanidmUser::IdmAdmin,
1118            ctx.kaniop_ctx.client.clone(),
1119        )
1120        .await?;
1121        let mail_sender_status =
1122            reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
1123        let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1124        if mail_sender_status != current_mail_sender_status {
1125            let kanidm_api: Api<Kanidm> =
1126                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1127            let status_patch = serde_json::json!({
1128                "status": {
1129                    "mailSender": mail_sender_status
1130                }
1131            });
1132            kanidm_api
1133                .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1134                .await
1135                .map_err(|e| {
1136                    Error::KubeError(
1137                        format!("failed to patch Kanidm/status {namespace}/{name}"),
1138                        Box::new(e),
1139                    )
1140                })?;
1141        }
1142    }
1143
1144    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
1145}
1146
1147async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
1148    debug!(msg = "cleanup");
1149
1150    cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1151
1152    let namespace = kanidm.namespace().unwrap();
1153    let name = kanidm.name_any();
1154
1155    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1156        &namespace,
1157        &name,
1158        crate::controller::kanidm::KanidmUser::IdmAdmin,
1159        ctx.kaniop_ctx.client.clone(),
1160    )
1161    .await
1162    {
1163        cleanup_mail_sender_in_kanidm(&kanidm_client, &name).await?;
1164    }
1165
1166    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1167    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
1168}
1169
1170async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1171    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1172    let start = std::time::Instant::now();
1173    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1174    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1175
1176    loop {
1177        let sts = sts_api.get(sts_name).await.map_err(|e| {
1178            Error::KubeError(
1179                format!("failed to get StatefulSet {namespace}/{sts_name}"),
1180                Box::new(e),
1181            )
1182        })?;
1183
1184        if sts.metadata.deletion_timestamp.is_some() {
1185            return Err(Error::ReceiveOutput(format!(
1186                "StatefulSet {namespace}/{sts_name} is being deleted"
1187            )));
1188        }
1189
1190        let spec = sts.spec.as_ref().ok_or_else(|| {
1191            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1192        })?;
1193
1194        let status = match sts.status {
1195            Some(ref s) => s,
1196            None => {
1197                if start.elapsed() >= timeout {
1198                    return Err(Error::ReceiveOutput(format!(
1199                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1200                    )));
1201                }
1202                sleep(poll_interval).await;
1203                continue;
1204            }
1205        };
1206
1207        let replicas = spec.replicas.unwrap_or(0);
1208
1209        if replicas == 0 {
1210            return Ok(());
1211        }
1212
1213        let updated_replicas = status.updated_replicas.unwrap_or(0);
1214        let current_revision = status.current_revision.as_deref().unwrap_or("");
1215        let update_revision = status.update_revision.as_deref().unwrap_or("");
1216        let ready_replicas = status.ready_replicas.unwrap_or(0);
1217
1218        if updated_replicas == replicas
1219            && current_revision == update_revision
1220            && ready_replicas == replicas
1221        {
1222            return Ok(());
1223        }
1224
1225        if start.elapsed() >= timeout {
1226            return Err(Error::ReceiveOutput(format!(
1227                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1228            )));
1229        }
1230
1231        sleep(poll_interval).await;
1232    }
1233}
1234
1235async fn show_replica_cert_with_retries(
1236    kanidm: &Kanidm,
1237    ctx: Arc<Context>,
1238    pod_name: &str,
1239    max_attempts: u32,
1240) -> std::result::Result<String, Error> {
1241    let mut last_error = None;
1242    for attempt in 0..max_attempts {
1243        if attempt > 0 {
1244            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1245        }
1246        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1247            Ok(cert) => return Ok(cert),
1248            Err(e) => {
1249                info!(
1250                    msg = format!(
1251                        "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1252                        attempt + 1,
1253                        max_attempts
1254                    )
1255                );
1256                last_error = Some(e);
1257            }
1258        }
1259    }
1260    Err(last_error.unwrap_or_else(|| {
1261        Error::ReceiveOutput(format!(
1262            "failed to get certificate for {pod_name} after {max_attempts} attempts"
1263        ))
1264    }))
1265}
1266
1267impl Kanidm {
1268    // Convenience methods that handle context and operator name
1269    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1270    where
1271        K: Resource<Scope = NamespaceResourceScope>
1272            + Serialize
1273            + Clone
1274            + std::fmt::Debug
1275            + for<'de> Deserialize<'de>,
1276        <K as kube::Resource>::DynamicType: Default,
1277        <K as Resource>::Scope: std::marker::Sized,
1278    {
1279        self.kube_delete(
1280            ctx.kaniop_ctx.client.clone(),
1281            &ctx.kaniop_ctx.metrics,
1282            resource,
1283        )
1284        .await
1285    }
1286
1287    /// Server-side apply without any generic delete/recreate fallback.
1288    /// Resource-specific reconcilers use this when they own lifecycle decisions.
1289    pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1290    where
1291        K: Resource<Scope = NamespaceResourceScope>
1292            + Serialize
1293            + Clone
1294            + std::fmt::Debug
1295            + for<'de> Deserialize<'de>,
1296        <K as kube::Resource>::DynamicType: Default,
1297        <K as Resource>::Scope: std::marker::Sized,
1298    {
1299        self.kube_apply(
1300            ctx.kaniop_ctx.client.clone(),
1301            resource,
1302            KANIDM_OPERATOR_NAME,
1303        )
1304        .await
1305    }
1306
1307    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1308    where
1309        K: Resource<Scope = NamespaceResourceScope>
1310            + Serialize
1311            + Clone
1312            + std::fmt::Debug
1313            + for<'de> Deserialize<'de>,
1314        <K as kube::Resource>::DynamicType: Default,
1315        <K as Resource>::Scope: std::marker::Sized,
1316    {
1317        self.kube_patch(
1318            ctx.kaniop_ctx.client.clone(),
1319            &ctx.kaniop_ctx.metrics,
1320            resource,
1321            KANIDM_OPERATOR_NAME,
1322        )
1323        .await
1324    }
1325
1326    #[inline]
1327    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1328        LABELS
1329            .clone()
1330            .into_iter()
1331            .chain([
1332                (INSTANCE_LABEL.to_string(), self.name_any()),
1333                (CLUSTER_LABEL.to_string(), self.name_any()),
1334            ])
1335            .collect()
1336    }
1337
1338    #[inline]
1339    fn generate_labels(&self) -> BTreeMap<String, String> {
1340        self.generate_resource_labels()
1341    }
1342
1343    #[inline]
1344    fn get_tls_secret_name(&self) -> String {
1345        format!("{}-tls", self.name_any())
1346    }
1347
1348    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
1349    /// falling back to the ingress TLS secret and then to the default name.
1350    pub(crate) fn effective_tls_secret_name(&self) -> String {
1351        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1352            self.spec
1353                .ingress
1354                .as_ref()
1355                .and_then(|i| i.tls_secret_name.clone())
1356                .unwrap_or_else(|| self.get_tls_secret_name())
1357        })
1358    }
1359
1360    #[inline]
1361    fn get_namespace(&self) -> String {
1362        // safe unwrap: Kanidm is namespaced scoped
1363        self.namespace().unwrap()
1364    }
1365
1366    #[inline]
1367    fn is_replication_enabled(&self) -> bool {
1368        self.spec.replica_groups.len() > 1
1369            || self
1370                .spec
1371                .replica_groups
1372                .first()
1373                .is_some_and(|rg| rg.replicas > 1)
1374            || !self.spec.external_replication_nodes.is_empty()
1375    }
1376
1377    #[inline]
1378    fn is_updatable(&self, status: &KanidmStatus) -> bool {
1379        status
1380            .version
1381            .as_ref()
1382            .map(|v| {
1383                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1384                    return false;
1385                }
1386                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1387                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1388                match v.upgrade_check_result {
1389                    KanidmUpgradeCheckResult::Passed => {
1390                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1391                            minor <= v_minor + 1
1392                        } else {
1393                            true
1394                        }
1395                    }
1396                    KanidmUpgradeCheckResult::Failed => {
1397                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1398                            versions
1399                        {
1400                            major == v_major && minor == v_minor && patch >= v_patch
1401                        } else {
1402                            v.image_tag == current_tag
1403                        }
1404                    }
1405                }
1406            })
1407            .unwrap_or(true)
1408    }
1409
1410    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1411        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1412        let pod = pod_api.get(pod_name).await.map_err(|e| {
1413            Error::KubeError(
1414                format!("failed to get pod {namespace}/{pod_name}"),
1415                Box::new(e),
1416            )
1417        })?;
1418        Ok(pod
1419            .status
1420            .as_ref()
1421            .and_then(|s| s.conditions.as_ref())
1422            .is_some_and(|conditions| {
1423                conditions
1424                    .iter()
1425                    .any(|c| c.type_ == "Ready" && c.status == "True")
1426            }))
1427    }
1428
1429    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1430        let start = std::time::Instant::now();
1431        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1432        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1433
1434        loop {
1435            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1436                Ok(true) => return Ok(()),
1437                Ok(false) => {}
1438                Err(e) => {
1439                    if !e.is_retryable() {
1440                        return Err(e);
1441                    }
1442                    trace!(msg = "pod readiness check failed, retrying", %e);
1443                }
1444            }
1445
1446            if start.elapsed() >= timeout {
1447                return Err(Error::ReceiveOutput(format!(
1448                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1449                )));
1450            }
1451
1452            sleep(poll_interval).await;
1453        }
1454    }
1455
1456    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1457    where
1458        I: IntoIterator<Item = T> + Debug,
1459        T: Into<String>,
1460    {
1461        let namespace = &self.get_namespace();
1462        trace!(
1463            msg = "pod exec",
1464            resource.name = &pod_name,
1465            resource.namespace = &namespace,
1466            ?command
1467        );
1468        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1469        let attached = pod
1470            .exec(
1471                pod_name,
1472                command,
1473                &AttachParams::default().container("kanidm"),
1474            )
1475            .await
1476            .map_err(|e| {
1477                Error::KubeError(
1478                    format!("failed to exec pod {namespace}/{pod_name}"),
1479                    Box::new(e),
1480                )
1481            })?;
1482        get_output(attached).await
1483    }
1484
1485    async fn exec_with_wait<I, T>(
1486        &self,
1487        ctx: Arc<Context>,
1488        pod_name: &str,
1489        command: I,
1490    ) -> Result<String>
1491    where
1492        I: IntoIterator<Item = T> + Debug,
1493        T: Into<String>,
1494    {
1495        let namespace = &self.get_namespace();
1496        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1497        self.exec(ctx, pod_name, command).await
1498    }
1499
1500    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1501    where
1502        I: IntoIterator<Item = T> + Debug,
1503        T: Into<String>,
1504    {
1505        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1506        let first_replica_group = self
1507            .spec
1508            .replica_groups
1509            .first()
1510            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1511        let sts_name = self.statefulset_name(&first_replica_group.name);
1512        let pod_name = format!("{sts_name}-0");
1513        self.exec(ctx, &pod_name, command).await
1514    }
1515
1516    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1517    where
1518        I: IntoIterator<Item = T> + Debug,
1519        T: Into<String>,
1520    {
1521        let first_replica_group = self
1522            .spec
1523            .replica_groups
1524            .first()
1525            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1526        let sts_name = self.statefulset_name(&first_replica_group.name);
1527        let pod_name = format!("{sts_name}-0");
1528        self.exec_with_wait(ctx, &pod_name, command).await
1529    }
1530}
1531
1532#[cfg(test)]
1533mod test {
1534    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1535    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1536
1537    use crate::controller::State;
1538    use crate::kanidm::controller::context::{Context, Stores};
1539    use crate::kanidm::crd::KanidmStatus;
1540
1541    use kaniop_k8s_util::error::Result;
1542    use kaniop_k8s_util::resources::hash_secret_data;
1543
1544    use std::collections::BTreeMap;
1545    use std::sync::Arc;
1546
1547    use http::{Request, Response};
1548    use k8s_openapi::ByteString;
1549    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1550    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1551    use k8s_openapi::api::networking::v1::Ingress;
1552    use kube::api::ObjectMeta;
1553    use kube::runtime::reflector::store::Writer;
1554    use kube::{Client, Resource, ResourceExt, client::Body};
1555    use opentelemetry::metrics::MeterProvider;
1556    use serde_json::json;
1557
1558    impl Kanidm {
1559        /// A normal test kanidm with a given status
1560        pub fn test() -> Self {
1561            let mut e = Kanidm::new(
1562                "test",
1563                serde_json::from_value(json!({
1564                    "domain": "idm.example.com",
1565                    "replicaGroups": [
1566                        {
1567                            "name": "default",
1568                            "replicas": 1
1569                        }
1570                    ]
1571                }))
1572                .unwrap(),
1573            );
1574            e.meta_mut().namespace = Some("default".into());
1575            // Pre-populate finalizer so reconcile callback runs immediately
1576            // (skips the "add finalizer + wait for next reconcile" step)
1577            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1578            e
1579        }
1580
1581        pub fn with_ingress(mut self) -> Self {
1582            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1583            self
1584        }
1585
1586        /// Modify kanidm replicas
1587        pub fn with_replicas(mut self, replicas: i32) -> Self {
1588            self.spec.replica_groups[0].replicas = replicas;
1589            self
1590        }
1591
1592        /// Modify kanidm to set a deletion timestamp
1593        pub fn needs_delete(mut self) -> Self {
1594            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1595            use k8s_openapi::jiff::Timestamp;
1596            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1597            self.meta_mut().deletion_timestamp = Some(Time(now));
1598            self
1599        }
1600
1601        /// Modify a kanidm to have an expected status
1602        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1603            self.status = Some(status);
1604            self
1605        }
1606    }
1607
1608    // We wrap tower_test::mock::Handle
1609    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1610    pub struct ApiServerVerifier(ApiServerHandle);
1611
1612    /// Scenarios we test for in ApiServerVerifier
1613    pub enum Scenario {
1614        Create(Kanidm),
1615        CreateWithTwoReplicas(Kanidm),
1616        CreateWithIngress(Kanidm),
1617        CreateWithIngressWithTwoReplicas(Kanidm),
1618        CreateWithTlsSecret(Kanidm, Box<Secret>),
1619    }
1620
1621    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1622        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1623            .await
1624            .expect("timeout on mock apiserver")
1625            .expect("scenario succeeded")
1626    }
1627
1628    impl ApiServerVerifier {
1629        /// Tests only get to run specific scenarios that has matching handlers
1630        ///
1631        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1632        ///
1633        /// NB: If the controller is making more calls than we are handling in the scenario,
1634        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1635        ///
1636        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1637        /// scenario runs to completion (i.e. all expected calls were responded to),
1638        /// using the timeout to catch missing api calls to Kubernetes.
1639        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1640            tokio::spawn(async move {
1641                // moving self => one scenario per test
1642                match scenario {
1643                    Scenario::Create(kanidm) => {
1644                        self.handle_kanidm_get(kanidm.clone())
1645                            .await
1646                            .unwrap()
1647                            .handle_kanidm_status_patch(kanidm.clone())
1648                            .await
1649                            .unwrap()
1650                            .handle_tls_secret_get_not_found(kanidm.clone())
1651                            .await
1652                            .unwrap()
1653                            .handle_statefulset_patch(kanidm.clone(), None)
1654                            .await
1655                            .unwrap()
1656                            .handle_service_patch(kanidm.clone())
1657                            .await
1658                    }
1659                    Scenario::CreateWithTwoReplicas(kanidm) => {
1660                        self.handle_kanidm_get(kanidm.clone())
1661                            .await
1662                            .unwrap()
1663                            .handle_kanidm_status_patch(kanidm.clone())
1664                            .await
1665                            .unwrap()
1666                            .handle_tls_secret_get_not_found(kanidm.clone())
1667                            .await
1668                            .unwrap()
1669                            .handle_statefulset_patch(kanidm.clone(), None)
1670                            .await
1671                            .unwrap()
1672                            .handle_service_patch(kanidm.clone())
1673                            .await
1674                    }
1675                    Scenario::CreateWithIngress(kanidm) => {
1676                        self.handle_kanidm_get(kanidm.clone())
1677                            .await
1678                            .unwrap()
1679                            .handle_kanidm_status_patch(kanidm.clone())
1680                            .await
1681                            .unwrap()
1682                            .handle_tls_secret_get_not_found(kanidm.clone())
1683                            .await
1684                            .unwrap()
1685                            .handle_statefulset_patch(kanidm.clone(), None)
1686                            .await
1687                            .unwrap()
1688                            .handle_service_patch(kanidm.clone())
1689                            .await
1690                            .unwrap()
1691                            .handle_ingress_patch(kanidm.clone())
1692                            .await
1693                    }
1694                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1695                        self.handle_kanidm_get(kanidm.clone())
1696                            .await
1697                            .unwrap()
1698                            .handle_kanidm_status_patch(kanidm.clone())
1699                            .await
1700                            .unwrap()
1701                            .handle_tls_secret_get_not_found(kanidm.clone())
1702                            .await
1703                            .unwrap()
1704                            .handle_statefulset_patch(kanidm.clone(), None)
1705                            .await
1706                            .unwrap()
1707                            .handle_service_patch(kanidm.clone())
1708                            .await
1709                            .unwrap()
1710                            .handle_ingress_patch(kanidm.clone())
1711                            .await
1712                    }
1713                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1714                        let expected_hash = hash_secret_data(&secret);
1715                        self.handle_kanidm_get(kanidm.clone())
1716                            .await
1717                            .unwrap()
1718                            .handle_kanidm_status_patch(kanidm.clone())
1719                            .await
1720                            .unwrap()
1721                            .handle_tls_secret_get(kanidm.clone(), *secret)
1722                            .await
1723                            .unwrap()
1724                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1725                            .await
1726                            .unwrap()
1727                            .handle_service_patch(kanidm.clone())
1728                            .await
1729                    }
1730                }
1731                .expect("scenario completed without errors");
1732            })
1733        }
1734
1735        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1736            let (request, send) = self.0.next_request().await.expect("service not called");
1737            assert_eq!(request.method(), http::Method::GET);
1738            let expected_uri_prefix = format!(
1739                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1740                kanidm.name_any()
1741            );
1742            let uri = request.uri().to_string();
1743            assert!(
1744                uri.starts_with(&expected_uri_prefix),
1745                "expected uri to start with {}, got {}",
1746                expected_uri_prefix,
1747                uri
1748            );
1749            let response = serde_json::to_vec(&kanidm).unwrap();
1750            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1751            Ok(self)
1752        }
1753
1754        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1755            let (request, send) = self.0.next_request().await.expect("service not called");
1756            assert_eq!(request.method(), http::Method::PATCH);
1757            assert_eq!(
1758                request.uri().to_string(),
1759                format!(
1760                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1761                    kanidm.name_any()
1762                )
1763            );
1764
1765            let req_body = request.into_body().collect_bytes().await.unwrap();
1766            let json: serde_json::Value =
1767                serde_json::from_slice(&req_body).expect("patch object is json");
1768            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1769                .expect("valid kanidm status");
1770            let mut kanidm = Kanidm::test();
1771            kanidm.status = Some(status.clone());
1772            let response = serde_json::to_vec(&kanidm).unwrap();
1773            // pass through kanidm "patch accepted"
1774            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1775            Ok(self)
1776        }
1777
1778        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1779            let (request, send) = self.0.next_request().await.expect("service not called");
1780            assert_eq!(request.method(), http::Method::GET);
1781            let expected_uri_prefix = format!(
1782                "/api/v1/namespaces/default/secrets/{}",
1783                kanidm.effective_tls_secret_name()
1784            );
1785            let uri = request.uri().to_string();
1786            assert!(
1787                uri.starts_with(&expected_uri_prefix),
1788                "expected uri to start with {expected_uri_prefix}, got {uri}"
1789            );
1790            let status = serde_json::json!({
1791                "kind": "Status",
1792                "apiVersion": "v1",
1793                "metadata": {},
1794                "status": "Failure",
1795                "message": "secret not found",
1796                "reason": "NotFound",
1797                "code": 404
1798            });
1799            send.send_response(
1800                Response::builder()
1801                    .status(404)
1802                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1803                    .unwrap(),
1804            );
1805            Ok(self)
1806        }
1807
1808        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1809            let (request, send) = self.0.next_request().await.expect("service not called");
1810            assert_eq!(request.method(), http::Method::GET);
1811            let expected_uri_prefix = format!(
1812                "/api/v1/namespaces/default/secrets/{}",
1813                kanidm.effective_tls_secret_name()
1814            );
1815            let uri = request.uri().to_string();
1816            assert!(
1817                uri.starts_with(&expected_uri_prefix),
1818                "expected uri to start with {expected_uri_prefix}, got {uri}"
1819            );
1820            let response = serde_json::to_vec(&secret).unwrap();
1821            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1822            Ok(self)
1823        }
1824
1825        async fn handle_statefulset_patch(
1826            mut self,
1827            kanidm: Kanidm,
1828            expected_tls_hash: Option<String>,
1829        ) -> Result<Self> {
1830            for rg in kanidm.spec.replica_groups.iter() {
1831                let (request, send) = self.0.next_request().await.expect("service not called");
1832                assert_eq!(request.method(), http::Method::PATCH);
1833                assert_eq!(
1834                    request.uri().to_string(),
1835                    format!(
1836                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1837                        kanidm.statefulset_name(&rg.name)
1838                    )
1839                );
1840                let req_body = request.into_body().collect_bytes().await.unwrap();
1841                let json: serde_json::Value =
1842                    serde_json::from_slice(&req_body).expect("patch object is json");
1843                let statefulset: StatefulSet =
1844                    serde_json::from_value(json).expect("valid statefulset");
1845                assert_eq!(
1846                    statefulset.clone().spec.unwrap().replicas.unwrap(),
1847                    rg.replicas
1848                );
1849                let tls_hash_annotation = statefulset
1850                    .spec
1851                    .as_ref()
1852                    .unwrap()
1853                    .template
1854                    .metadata
1855                    .as_ref()
1856                    .unwrap()
1857                    .annotations
1858                    .as_ref()
1859                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
1860                    .cloned();
1861                assert_eq!(tls_hash_annotation, expected_tls_hash);
1862                let response = serde_json::to_vec(&statefulset).unwrap();
1863                // pass through kanidm "patch accepted"
1864                send.send_response(Response::builder().body(Body::from(response)).unwrap());
1865            }
1866
1867            Ok(self)
1868        }
1869
1870        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1871            let (request, send) = self.0.next_request().await.expect("service not called");
1872            assert_eq!(request.method(), http::Method::PATCH);
1873            assert_eq!(
1874                request.uri().to_string(),
1875                format!(
1876                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1877                    kanidm.name_any()
1878                )
1879            );
1880
1881            let req_body = request.into_body().collect_bytes().await.unwrap();
1882            let json: serde_json::Value =
1883                serde_json::from_slice(&req_body).expect("patch object is json");
1884            let service: Service = serde_json::from_value(json).expect("valid service");
1885            let response = serde_json::to_vec(&service).unwrap();
1886            // pass through kanidm "patch accepted"
1887            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1888            Ok(self)
1889        }
1890
1891        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1892            let (request, send) = self.0.next_request().await.expect("service not called");
1893            assert_eq!(request.method(), http::Method::PATCH);
1894            assert_eq!(
1895                request.uri().to_string(),
1896                format!(
1897                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1898                    kanidm.name_any()
1899                )
1900            );
1901
1902            let req_body = request.into_body().collect_bytes().await.unwrap();
1903            let json: serde_json::Value =
1904                serde_json::from_slice(&req_body).expect("patch object is json");
1905            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1906            let response = serde_json::to_vec(&ingress).unwrap();
1907            // pass through kanidm "patch accepted"
1908            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1909            Ok(self)
1910        }
1911    }
1912
1913    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1914        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1915        let mock_client = Client::new(mock_service, "default");
1916        let stores = Stores {
1917            stateful_set_store: Writer::default().as_reader(),
1918            service_store: Writer::default().as_reader(),
1919            ingress_store: Writer::default().as_reader(),
1920            secret_store: Writer::default().as_reader(),
1921            http_route_store: Some(Writer::default().as_reader()),
1922            backend_tls_policy_store: Some(Writer::default().as_reader()),
1923            deployment_store: Writer::default().as_reader(),
1924            config_map_store: Writer::default().as_reader(),
1925        };
1926        let controller_id = "test";
1927
1928        // Create a test meter for metrics
1929        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
1930        let meter = provider.meter("test");
1931        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
1932
1933        let state = State::new(
1934            metrics,
1935            Writer::default().as_reader(),
1936            Writer::default().as_reader(),
1937            None,
1938        );
1939        let ctx = Arc::new(Context::new(
1940            state.to_context(mock_client, controller_id),
1941            stores,
1942        ));
1943        (ctx, ApiServerVerifier(handle))
1944    }
1945
1946    #[tokio::test]
1947    async fn kanidm_create() {
1948        let (testctx, fakeserver) = get_test_context();
1949        let kanidm = Kanidm::test();
1950        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
1951        reconcile_kanidm(Arc::new(kanidm), testctx)
1952            .await
1953            .expect("reconciler");
1954        timeout_after_1s(mocksrv).await;
1955    }
1956
1957    #[tokio::test]
1958    async fn kanidm_create_with_two_replicas() {
1959        let (testctx, fakeserver) = get_test_context();
1960        let kanidm = Kanidm::test().with_replicas(2);
1961        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
1962        reconcile_kanidm(Arc::new(kanidm), testctx)
1963            .await
1964            .expect("reconciler");
1965        timeout_after_1s(mocksrv).await;
1966    }
1967
1968    #[tokio::test]
1969    async fn kanidm_create_with_ingress() {
1970        let (testctx, fakeserver) = get_test_context();
1971        let kanidm = Kanidm::test().with_ingress();
1972        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
1973        reconcile_kanidm(Arc::new(kanidm), testctx)
1974            .await
1975            .expect("reconciler");
1976        timeout_after_1s(mocksrv).await;
1977    }
1978
1979    #[tokio::test]
1980    async fn kanidm_create_with_ingress_with_two_replicas() {
1981        let (testctx, fakeserver) = get_test_context();
1982        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
1983        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
1984        reconcile_kanidm(Arc::new(kanidm), testctx)
1985            .await
1986            .expect("reconciler");
1987        timeout_after_1s(mocksrv).await;
1988    }
1989
1990    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
1991        Arc::new(StatefulSet {
1992            metadata: ObjectMeta {
1993                name: Some(format!("{cluster}-default")),
1994                namespace: Some(namespace.to_string()),
1995                labels: Some(BTreeMap::from([(
1996                    CLUSTER_LABEL.to_string(),
1997                    cluster.to_string(),
1998                )])),
1999                ..Default::default()
2000            },
2001            spec: Some(StatefulSetSpec {
2002                template: PodTemplateSpec {
2003                    metadata: Some(ObjectMeta {
2004                        annotations: tls_hash.map(|h| {
2005                            BTreeMap::from([(
2006                                TLS_SECRET_HASH_ANNOTATION.to_string(),
2007                                h.to_string(),
2008                            )])
2009                        }),
2010                        ..Default::default()
2011                    }),
2012                    ..Default::default()
2013                },
2014                ..Default::default()
2015            }),
2016            ..Default::default()
2017        })
2018    }
2019
2020    #[test]
2021    fn test_previous_tls_secret_hash() {
2022        let kanidm = Kanidm::test();
2023
2024        // annotation carried over from an existing statefulset
2025        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2026        assert_eq!(
2027            previous_tls_secret_hash(&kanidm, stss),
2028            Some("abc123".to_string())
2029        );
2030
2031        // no annotation on the existing statefulset
2032        let stss = vec![statefulset_of("test", "default", None)];
2033        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2034
2035        // statefulsets from another cluster or namespace are ignored
2036        let stss = vec![
2037            statefulset_of("other", "default", Some("abc123")),
2038            statefulset_of("test", "other-ns", Some("abc123")),
2039        ];
2040        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2041    }
2042
2043    #[tokio::test]
2044    async fn kanidm_create_with_tls_secret() {
2045        let (testctx, fakeserver) = get_test_context();
2046        let kanidm = Kanidm::test();
2047        let secret = Secret {
2048            metadata: ObjectMeta {
2049                name: Some(kanidm.effective_tls_secret_name()),
2050                namespace: Some("default".to_string()),
2051                ..Default::default()
2052            },
2053            data: Some(BTreeMap::from([
2054                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2055                ("tls.key".to_string(), ByteString(b"key".to_vec())),
2056            ])),
2057            ..Default::default()
2058        };
2059        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2060            kanidm.clone(),
2061            Box::new(secret),
2062        ));
2063        reconcile_kanidm(Arc::new(kanidm), testctx)
2064            .await
2065            .expect("reconciler");
2066        timeout_after_1s(mocksrv).await;
2067    }
2068}