Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::{
21    StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
22    preserve_defaulted_statefulset_fields,
23};
24use self::status::StatusExt;
25use self::status::{is_kanidm_available, is_kanidm_initialized};
26
27use crate::controller::context::KubeOperations;
28use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
29use crate::kanidm::crd::{
30    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
31};
32use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
33use crate::telemetry;
34
35use kaniop_k8s_util::client::get_output;
36use kaniop_k8s_util::error::{Error, Result};
37use kaniop_k8s_util::parse::parse_semver;
38use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
39
40use std::collections::{BTreeMap, BTreeSet};
41use std::fmt::Debug;
42use std::sync::{Arc, LazyLock};
43
44use futures::future::{TryJoinAll, join_all, try_join_all};
45use futures::stream::{self, StreamExt};
46use futures::try_join;
47use k8s_openapi::NamespaceResourceScope;
48use k8s_openapi::api::apps::v1::StatefulSet;
49use k8s_openapi::api::core::v1::{Pod, Secret, Service};
50use kube::Resource;
51use kube::ResourceExt;
52use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
53use kube::runtime::controller::Action;
54use kube::runtime::events::{Event, EventType};
55use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
56use serde::{Deserialize, Serialize};
57use tokio::time::{Duration, sleep};
58use tracing::{Span, debug, field, info, instrument, trace, warn};
59
60const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
61const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
62const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
63const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
64const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
65const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
66const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
67const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
68const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
69
70pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
71const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
72pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
73
74const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
75
76static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
77    BTreeMap::from([
78        (NAME_LABEL.to_string(), "kanidm".to_string()),
79        (
80            MANAGED_BY_LABEL.to_string(),
81            format!("kaniop-{CONTROLLER_ID}"),
82        ),
83    ])
84});
85
86async fn restart_statefulsets_after_replica_secret_update(
87    ctx: Arc<Context>,
88    sts_api: &Api<StatefulSet>,
89    namespace: &str,
90    statefulset_names: &[String],
91    sequential: bool,
92) -> Result<()> {
93    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
94    // Pods must restart after cert secret changes or replication keeps stale TLS material.
95    let restart_futures = statefulset_names
96        .iter()
97        .map(|sts_name| sts_api.restart(sts_name));
98
99    if sequential {
100        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
101        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
102            return Err(Error::kube_error(
103                "restart",
104                "StatefulSet",
105                namespace,
106                "replica certificate secret update statefulsets",
107                first_err,
108            ));
109        }
110    } else {
111        let results: Vec<_> = join_all(restart_futures).await;
112        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
113            return Err(Error::kube_error(
114                "restart",
115                "StatefulSet",
116                namespace,
117                "replica certificate secret update statefulsets",
118                first_err,
119            ));
120        }
121    }
122
123    let rollout_futures = statefulset_names
124        .iter()
125        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
126    try_join_all(rollout_futures).await?;
127
128    Ok(())
129}
130
131pub async fn reconcile_admins_secret(
132    kanidm: Arc<Kanidm>,
133    ctx: Arc<Context>,
134    status: &KanidmStatus,
135) -> Result<bool> {
136    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
137        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
138        kanidm.patch(&ctx, admins_secret).await?;
139        return Ok(true);
140    }
141    Ok(false)
142}
143
144pub async fn reconcile_replication_secrets(
145    kanidm: Arc<Kanidm>,
146    ctx: Arc<Context>,
147    status: &KanidmStatus,
148) -> Result<bool> {
149    let mut changed = false;
150    let expected_secret_names = if kanidm.is_replication_enabled() {
151        status
152            .replica_statuses
153            .iter()
154            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
155            .collect::<Vec<_>>()
156    } else {
157        Vec::new()
158    };
159    let deprecated_secrets = ctx
160        .stores
161        .secret_store
162        .state()
163        .into_iter()
164        .filter(|secret| {
165            secret.namespace() == kanidm.namespace()
166                && kanidm.admins_secret_name() != secret.name_any()
167                && !expected_secret_names.contains(&secret.name_any())
168                && secret
169                    .metadata
170                    .labels
171                    .as_ref()
172                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
173        })
174        .collect::<Vec<_>>();
175    if !deprecated_secrets.is_empty() {
176        changed = true;
177    }
178    let secret_delete_future = deprecated_secrets
179        .iter()
180        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
181        .collect::<TryJoinAll<_>>();
182    try_join!(secret_delete_future)?;
183
184    if kanidm.is_replication_enabled() {
185        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
186        let statefulset_names = status
187            .replica_statuses
188            .iter()
189            .map(|rs| rs.statefulset_name.clone())
190            .collect::<BTreeSet<_>>()
191            .into_iter()
192            .collect::<Vec<_>>();
193
194        let has_pending = status
195            .replica_statuses
196            .iter()
197            .any(|rs| rs.state == KanidmReplicaState::Pending);
198
199        let sts_api =
200            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
201
202        if has_pending {
203            changed = true;
204            stream::iter(
205                status
206                    .replica_statuses
207                    .iter()
208                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
209            )
210            .then(|rs| async {
211                let secret = kanidm
212                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
213                    .await?;
214                kanidm.patch(&ctx, secret).await
215            })
216            .collect::<Vec<_>>()
217            .await
218            .into_iter()
219            .collect::<Result<Vec<_>>>()?;
220
221            let restart_futures = status
222                .replica_statuses
223                .iter()
224                .map(|rs| sts_api.restart(&rs.statefulset_name));
225
226            if has_single_replica {
227                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
228                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
229                    return Err(Error::kube_error(
230                        "restart",
231                        "StatefulSet",
232                        kanidm.get_namespace(),
233                        "pending replica statefulsets",
234                        first_err,
235                    ));
236                }
237            } else {
238                let results: Vec<_> = join_all(restart_futures).await;
239                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
240                    return Err(Error::kube_error(
241                        "restart",
242                        "StatefulSet",
243                        kanidm.get_namespace(),
244                        "pending replica statefulsets",
245                        first_err,
246                    ));
247                }
248            }
249        }
250
251        let has_certificate_host_invalid = status
252            .replica_statuses
253            .iter()
254            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
255
256        let has_certificate_expiring = status
257            .replica_statuses
258            .iter()
259            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
260
261        let has_non_ready_replicas = status
262            .replica_statuses
263            .iter()
264            .any(|rs| rs.state != KanidmReplicaState::Ready);
265
266        let host_invalid_replicas: Vec<_> = status
267            .replica_statuses
268            .iter()
269            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
270            .collect();
271
272        let cert_expiring_replicas: Vec<_> = status
273            .replica_statuses
274            .iter()
275            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
276            .collect();
277
278        if !host_invalid_replicas.is_empty() {
279            changed = true;
280            let restart_futures = host_invalid_replicas
281                .iter()
282                .map(|rs| sts_api.restart(&rs.statefulset_name));
283
284            if has_single_replica {
285                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
286                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
287                    return Err(Error::kube_error(
288                        "restart",
289                        "StatefulSet",
290                        kanidm.get_namespace(),
291                        "certificate host invalid statefulsets",
292                        first_err,
293                    ));
294                }
295            } else {
296                let results: Vec<_> = join_all(restart_futures).await;
297                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
298                    return Err(Error::kube_error(
299                        "restart",
300                        "StatefulSet",
301                        kanidm.get_namespace(),
302                        "certificate host invalid statefulsets",
303                        first_err,
304                    ));
305                }
306            }
307
308            let restarted_sts_names: Vec<_> = host_invalid_replicas
309                .iter()
310                .map(|rs| rs.statefulset_name.clone())
311                .collect::<BTreeSet<_>>()
312                .into_iter()
313                .collect();
314
315            let namespace = kanidm.get_namespace();
316            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
317                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
318            });
319            try_join_all(rollout_futures).await?;
320
321            let renew_futures = host_invalid_replicas
322                .iter()
323                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
324            let results: Vec<_> = join_all(renew_futures).await;
325            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
326                return Err(first_err);
327            }
328
329            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
330
331            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
332                let cert = show_replica_cert_with_retries(
333                    &kanidm,
334                    ctx.clone(),
335                    &rs.pod_name,
336                    CERT_SHOW_RETRY_ATTEMPTS,
337                )
338                .await?;
339                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
340                kanidm.patch(&ctx, secret.clone()).await
341            });
342            try_join_all(cert_futures).await?;
343
344            restart_statefulsets_after_replica_secret_update(
345                ctx.clone(),
346                &sts_api,
347                &kanidm.get_namespace(),
348                &statefulset_names,
349                has_single_replica,
350            )
351            .await?;
352
353            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
354        }
355
356        if !cert_expiring_replicas.is_empty() {
357            changed = true;
358            let renew_futures = cert_expiring_replicas
359                .iter()
360                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
361            let results: Vec<_> = join_all(renew_futures).await;
362            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
363                return Err(first_err);
364            }
365
366            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
367
368            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
369                let cert = show_replica_cert_with_retries(
370                    &kanidm,
371                    ctx.clone(),
372                    &rs.pod_name,
373                    CERT_SHOW_RETRY_ATTEMPTS,
374                )
375                .await?;
376                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
377                kanidm.patch(&ctx, secret.clone()).await
378            });
379            try_join_all(cert_futures).await?;
380
381            restart_statefulsets_after_replica_secret_update(
382                ctx.clone(),
383                &sts_api,
384                &kanidm.get_namespace(),
385                &statefulset_names,
386                has_single_replica,
387            )
388            .await?;
389
390            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
391        }
392
393        if has_non_ready_replicas
394            && !has_pending
395            && !has_certificate_host_invalid
396            && !has_certificate_expiring
397        {
398            changed = true;
399            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
400
401            let restart_futures = status
402                .replica_statuses
403                .iter()
404                .map(|rs| sts_api.restart(&rs.statefulset_name));
405
406            if has_single_replica {
407                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
408                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
409                    return Err(Error::kube_error(
410                        "restart",
411                        "StatefulSet",
412                        kanidm.get_namespace(),
413                        "replica statefulsets",
414                        first_err,
415                    ));
416                }
417            } else {
418                let results: Vec<_> = join_all(restart_futures).await;
419                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
420                    return Err(Error::kube_error(
421                        "restart",
422                        "StatefulSet",
423                        kanidm.get_namespace(),
424                        "replica statefulsets",
425                        first_err,
426                    ));
427                }
428            }
429        }
430    }
431
432    Ok(changed)
433}
434
435#[instrument(skip(ctx, kanidm))]
436pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
437    let trace_id = telemetry::get_trace_id();
438    Span::current().record("trace_id", field::display(&trace_id));
439    let _timer = ctx
440        .kaniop_ctx
441        .metrics
442        .reconcile_count_and_measure(&trace_id);
443    info!("reconciling Kanidm");
444
445    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
446        debug!(%e, "failed to reconcile status");
447        ctx.kaniop_ctx.metrics.status_update_errors_inc();
448        e
449    })?;
450    let kanidm_api: Api<Kanidm> =
451        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
452    let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
453    let outcome_clone = outcome.clone();
454    let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
455        let outcome = outcome_clone.clone();
456        let ctx = ctx.clone();
457        let status = status.clone();
458        async move {
459            match event {
460                Finalizer::Apply(kanidm) => {
461                    let (action, changed) = reconcile(kanidm, ctx, status).await?;
462                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
463                    Ok(action)
464                }
465                Finalizer::Cleanup(kanidm) => {
466                    let (action, changed) = cleanup(kanidm, ctx).await?;
467                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
468                    Ok(action)
469                }
470            }
471        }
472    })
473    .await
474    .or_else(|e| match e {
475        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
476            debug!("resource already removed during finalizer cleanup");
477            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
478        }
479        _ => Err(Error::FinalizerError(
480            "failed on kanidm account finalizer".to_string(),
481            Box::new(e),
482        )),
483    })?;
484    let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
485    Ok((action, changed))
486}
487
488/// Hash of the TLS secret content, injected as a pod template annotation so
489/// certificate renewals roll the statefulsets. `None` if the secret does not
490/// exist yet.
491async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
492    let secret_name = kanidm.effective_tls_secret_name();
493    let secret_api =
494        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
495    let secret = secret_api
496        .get_opt(&secret_name)
497        .await
498        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
499    Ok(secret.as_ref().map(hash_secret_data))
500}
501
502/// Previous TLS secret hash annotation from the existing statefulsets.
503///
504/// Used when the TLS secret is missing: dropping the annotation would roll
505/// the pods onto a missing volume and leave them stuck in ContainerCreating.
506fn previous_tls_secret_hash(
507    kanidm: &Kanidm,
508    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
509) -> Option<String> {
510    stateful_sets
511        .into_iter()
512        .filter(|sts| {
513            sts.namespace() == kanidm.namespace()
514                && sts
515                    .metadata
516                    .labels
517                    .as_ref()
518                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
519        })
520        .find_map(|sts| {
521            sts.spec
522                .as_ref()?
523                .template
524                .metadata
525                .as_ref()?
526                .annotations
527                .as_ref()?
528                .get(TLS_SECRET_HASH_ANNOTATION)
529                .cloned()
530        })
531}
532
533fn is_headless_service(service: &Service) -> bool {
534    service
535        .spec
536        .as_ref()
537        .and_then(|spec| spec.cluster_ip.as_deref())
538        == Some("None")
539}
540
541fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
542    if is_headless_service(desired) != is_headless_service(current) {
543        return;
544    }
545    let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
546    else {
547        return;
548    };
549    if desired_spec.cluster_ip.is_none() {
550        desired_spec.cluster_ip = current_spec.cluster_ip.clone();
551    }
552    if desired_spec.cluster_ips.is_none() {
553        desired_spec.cluster_ips = current_spec.cluster_ips.clone();
554    }
555}
556
557async fn wait_for_statefulset_deletion(
558    client: kube::Client,
559    namespace: &str,
560    name: &str,
561) -> Result<()> {
562    let api = Api::<StatefulSet>::namespaced(client, namespace);
563    let start = std::time::Instant::now();
564    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
565    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
566
567    loop {
568        match api.get_opt(name).await {
569            Ok(None) => return Ok(()),
570            Ok(Some(_)) => {}
571            Err(e) => {
572                return Err(Error::kube_error(
573                    "wait for deletion of",
574                    "StatefulSet",
575                    namespace,
576                    name,
577                    e,
578                ));
579            }
580        }
581
582        if start.elapsed() >= timeout {
583            return Err(Error::ReceiveOutput(format!(
584                "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
585            )));
586        }
587        sleep(poll_interval).await;
588    }
589}
590
591async fn reconcile_service(
592    kanidm: &Kanidm,
593    ctx: &Context,
594    mut desired: Service,
595) -> Result<Service> {
596    let namespace = kanidm.get_namespace();
597    let name = desired.name_any();
598    let desired_headless = is_headless_service(&desired);
599
600    if let Some(cached) = ctx
601        .stores
602        .service_store
603        .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
604    {
605        if is_headless_service(cached.as_ref()) == desired_headless {
606            preserve_defaulted_service_fields(&mut desired, cached.as_ref());
607        } else {
608            let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
609            let current = service_api
610                .get_opt(&name)
611                .await
612                .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
613
614            if let Some(current) = current {
615                let current_headless = is_headless_service(&current);
616                if current_headless != desired_headless {
617                    info!(
618                        resource.name = &name,
619                        resource.namespace = &namespace,
620                        current_headless,
621                        desired_headless,
622                        "recreating Service to change headless mode"
623                    );
624                    kanidm.delete(ctx, &current).await?;
625                    ctx.kaniop_ctx
626                        .metrics
627                        .reconcile_deploy_delete_create_inc("Service", "headless_transition");
628                    wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
629                        .await?;
630                } else {
631                    preserve_defaulted_service_fields(&mut desired, &current);
632                }
633            }
634        }
635    }
636
637    kanidm.patch(ctx, desired).await
638}
639
640async fn wait_for_service_deletion(
641    client: kube::Client,
642    namespace: &str,
643    name: &str,
644) -> Result<()> {
645    let api = Api::<Service>::namespaced(client, namespace);
646    let start = std::time::Instant::now();
647    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
648    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
649
650    loop {
651        match api.get_opt(name).await {
652            Ok(None) => return Ok(()),
653            Ok(Some(_)) => {}
654            Err(e) => {
655                return Err(Error::kube_error(
656                    "wait for deletion of",
657                    "Service",
658                    namespace,
659                    name,
660                    e,
661                ));
662            }
663        }
664
665        if start.elapsed() >= timeout {
666            return Err(Error::ReceiveOutput(format!(
667                "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
668            )));
669        }
670        sleep(poll_interval).await;
671    }
672}
673
674async fn validate_statefulset_replacement(
675    api: &Api<StatefulSet>,
676    desired: &StatefulSet,
677    namespace: &str,
678    name: &str,
679) -> Result<()> {
680    let mut candidate = desired.clone();
681    candidate.metadata.name = None;
682    candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
683    candidate.metadata.namespace = None;
684    candidate.metadata.resource_version = None;
685    candidate.metadata.uid = None;
686    candidate.metadata.creation_timestamp = None;
687    candidate.metadata.generation = None;
688    candidate.metadata.managed_fields = None;
689    candidate.metadata.deletion_timestamp = None;
690    candidate.metadata.deletion_grace_period_seconds = None;
691    candidate.status = None;
692
693    api.create(
694        &PostParams {
695            dry_run: true,
696            ..Default::default()
697        },
698        &candidate,
699    )
700    .await
701    .map(|_| ())
702    .map_err(|e| {
703        Error::kube_error(
704            "validate replacement for",
705            "StatefulSet",
706            namespace,
707            name,
708            e,
709        )
710    })
711}
712
713async fn reconcile_statefulset(
714    kanidm: Arc<Kanidm>,
715    ctx: Arc<Context>,
716    desired: StatefulSet,
717) -> Result<StatefulSet> {
718    let namespace = kanidm.namespace().unwrap();
719    let name = desired.name_any();
720    let cached_current = ctx.stores.stateful_set_store.find(|current| {
721        current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
722    });
723
724    // On a cold reflector cache, keep the normal create/update path GET-free. If the optimistic
725    // apply fails with HTTP 422, the object may already exist with an immutable conflict that the
726    // cache has not observed yet. Only 422 errors enter the live classification/recreate recovery
727    // path; 429, 5xx, 403, and transport errors are returned immediately and non-destructively.
728    let cache_miss_422_error = if let Some(cached_current) = cached_current {
729        let mut cached_desired = desired.clone();
730        preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
731        if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
732            == StatefulSetApplyStrategy::Apply
733        {
734            return kanidm.apply(&ctx, cached_desired).await;
735        }
736        None
737    } else {
738        match kanidm.apply(&ctx, desired.clone()).await {
739            Ok(applied) => return Ok(applied),
740            Err(error) => {
741                // Only HTTP 422 can indicate an immutable-field conflict that is safe to
742                // recover from via delete/recreate. Arbitrary failures must not trigger
743                // destruction.
744                if is_kube_422(&error) {
745                    Some(error)
746                } else {
747                    return Err(error);
748                }
749            }
750        }
751    };
752
753    // Never make a destructive decision from reflector state or an SSA error alone. Re-read the
754    // live object and classify it immediately before deletion; another actor or an earlier
755    // reconcile may already have converged the immutable fields while the cache was catching up.
756    let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
757    let Some(live_current) = api
758        .get_opt(&name)
759        .await
760        .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
761    else {
762        return match cache_miss_422_error {
763            Some(error) => Err(error),
764            None => kanidm.apply(&ctx, desired).await,
765        };
766    };
767
768    // Start from the original desired object again. Defaults copied from a stale reflector entry
769    // must not survive into the live re-check and become the reason for a recreation.
770    let mut live_desired = desired.clone();
771    preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
772    match classify_statefulset_change(&live_current, &live_desired) {
773        StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
774            Some(error) => Err(error),
775            None => kanidm.apply(&ctx, live_desired).await,
776        },
777        StatefulSetApplyStrategy::Recreate { .. } => {
778            // An update 422 can combine an immutable conflict with independently-invalid desired
779            // state. Validate the replacement as a dry-run CREATE before deleting the live object,
780            // so create-time validation runs without the existing object's immutable-update check.
781            validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
782
783            // Re-read and reclassify immediately before destruction. Another actor or an earlier
784            // reconcile may have changed the live immutable fields or PVC-retention semantics.
785            let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
786                Error::kube_error(
787                    "re-read before deletion",
788                    "StatefulSet",
789                    &namespace,
790                    &name,
791                    e,
792                )
793            })?
794            else {
795                return kanidm.apply(&ctx, live_desired).await;
796            };
797            let mut latest_desired = live_desired.clone();
798            preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
799
800            match classify_statefulset_change(&latest_current, &latest_desired) {
801                StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
802                StatefulSetApplyStrategy::Recreate { immutable_fields } => {
803                    // Validate the replacement StatefulSet via a dry-run CREATE before deleting
804                    // the live object. This prevents deleting on a mixed 422 (immutable conflict
805                    // + independently-invalid desired state) where the replacement would still
806                    // be rejected by the API.
807                    validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
808                        .await?;
809
810                    info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
811                    kanidm.delete(&ctx, &latest_current).await?;
812                    ctx.kaniop_ctx
813                        .metrics
814                        .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
815                    wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
816                        .await?;
817                    kanidm.apply(&ctx, latest_desired).await
818                }
819            }
820        }
821    }
822}
823
824/// Return true only when the error structurally wraps a Kubernetes HTTP 422 response.
825fn is_kube_422(error: &Error) -> bool {
826    matches!(
827        error,
828        Error::KubeError(_, cause)
829            if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
830    )
831}
832
833#[cfg(test)]
834mod statefulset_recreate_recovery_tests {
835    use super::{Error, is_kube_422};
836
837    fn kube_api_error(code: u16) -> Error {
838        Error::KubeError(
839            "test error".to_string(),
840            Box::new(kube::Error::Api(Box::new(kube::error::Status {
841                code,
842                message: "test".to_string(),
843                reason: "test".to_string(),
844                ..Default::default()
845            }))),
846        )
847    }
848
849    #[test]
850    fn recreate_recovery_accepts_only_422() {
851        assert!(is_kube_422(&kube_api_error(422)));
852        assert!(!is_kube_422(&kube_api_error(403)));
853        assert!(!is_kube_422(&kube_api_error(429)));
854        assert!(!is_kube_422(&kube_api_error(500)));
855    }
856
857    #[test]
858    fn recreate_recovery_rejects_429_too_many_requests() {
859        assert!(!is_kube_422(&kube_api_error(429)));
860    }
861
862    #[test]
863    fn recreate_recovery_rejects_5xx_server_errors() {
864        assert!(!is_kube_422(&kube_api_error(500)));
865        assert!(!is_kube_422(&kube_api_error(502)));
866        assert!(!is_kube_422(&kube_api_error(503)));
867        assert!(!is_kube_422(&kube_api_error(504)));
868    }
869
870    #[test]
871    fn recreate_recovery_rejects_400_and_403() {
872        assert!(!is_kube_422(&kube_api_error(400)));
873        assert!(!is_kube_422(&kube_api_error(403)));
874    }
875}
876
877async fn reconcile(
878    kanidm: Arc<Kanidm>,
879    ctx: Arc<Context>,
880    status: KanidmStatus,
881) -> Result<(Action, bool)> {
882    if kanidm
883        .annotations()
884        .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
885    {
886        ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
887        return Ok((Action::requeue(Duration::from_secs(5)), false));
888    }
889
890    let mut changed = false;
891    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
892    let replication_secret_futures =
893        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
894
895    let sts_to_delete = {
896        let expected_sts_names = kanidm
897            .spec
898            .replica_groups
899            .iter()
900            .map(|rg| kanidm.statefulset_name(&rg.name))
901            .collect::<Vec<_>>();
902        ctx.stores
903            .stateful_set_store
904            .state()
905            .into_iter()
906            .filter(|sts| {
907                sts.namespace() == kanidm.namespace()
908                    && sts
909                        .metadata
910                        .labels
911                        .as_ref()
912                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
913                    && !expected_sts_names.contains(&sts.name_any())
914            })
915            .collect::<Vec<_>>()
916    };
917    if !sts_to_delete.is_empty() {
918        changed = true;
919    }
920    let sts_delete_futures = sts_to_delete
921        .iter()
922        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
923        .collect::<TryJoinAll<_>>();
924
925    let sts_futures = match kanidm.is_updatable(&status) {
926        true => {
927            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
928                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
929            });
930            kanidm
931                .spec
932                .replica_groups
933                .iter()
934                .map(|rg| {
935                    let sts = kanidm.create_statefulset(rg, tls_secret_hash.as_deref())?;
936                    Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
937                })
938                .collect::<Result<TryJoinAll<_>, _>>()?
939        }
940        false => {
941            let note = match status.version.as_ref() {
942                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
943                    format!(
944                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
945                        v.image_tag,
946                        crate::version::KANIDM_CLIENT_VERSION
947                    )
948                }
949                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
950            };
951            let _ignore_error = ctx
952                .kaniop_ctx
953                .recorder
954                .publish(
955                    &Event {
956                        type_: EventType::Warning,
957                        reason: "UpgradeBlocked".to_string(),
958                        note: Some(note),
959                        action: "ReconcileStatefulSet".to_string(),
960                        secondary: None,
961                    },
962                    &kanidm.object_ref(&()),
963                )
964                .await
965                .map_err(|e| {
966                    warn!(%e, "failed to publish KanidmError event");
967                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
968                });
969            try_join_all([])
970        }
971    };
972    let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
973
974    let deprecated_rg_svcs = {
975        let expected_rg_svcs_names = kanidm
976            .spec
977            .replica_groups
978            .iter()
979            .filter(|rg| rg.services.is_some())
980            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
981            .collect::<Vec<_>>();
982        ctx.stores
983            .service_store
984            .state()
985            .into_iter()
986            .filter(|svc| {
987                let labels = svc.metadata.labels.as_ref();
988                svc.namespace() == kanidm.namespace()
989                    && !expected_rg_svcs_names.contains(&svc.name_any())
990                    && labels.is_some_and(|l| {
991                        l.contains_key(REPLICA_LABEL)
992                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
993                    })
994            })
995            .collect::<Vec<_>>()
996    };
997    if !deprecated_rg_svcs.is_empty() {
998        changed = true;
999    }
1000    let rg_svcs_delete_futures = deprecated_rg_svcs
1001        .iter()
1002        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1003        .collect::<TryJoinAll<_>>();
1004
1005    let rg_services_futures = kanidm
1006        .spec
1007        .replica_groups
1008        .iter()
1009        .filter(|rg| rg.services.is_some())
1010        .flat_map(|rg| {
1011            (0..rg.replicas).map(|i| {
1012                let mut svc = kanidm.create_replica_group_service(rg, i);
1013                if let Some(current) = ctx.stores.service_store.find(|current| {
1014                    current.namespace() == kanidm.namespace()
1015                        && current.name_any() == svc.name_any()
1016                }) {
1017                    preserve_defaulted_service_fields(&mut svc, current.as_ref());
1018                }
1019                kanidm.patch(&ctx, svc)
1020            })
1021        })
1022        .collect::<TryJoinAll<_>>();
1023
1024    let deprecated_ingresses = {
1025        let names = [
1026            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1027            kanidm.generate_region_ingress_name(),
1028        ]
1029        .into_iter()
1030        .flatten()
1031        .collect::<Vec<_>>();
1032        ctx.stores
1033            .ingress_store
1034            .state()
1035            .into_iter()
1036            .filter(|ing| {
1037                ing.namespace() == kanidm.namespace()
1038                    && !names.contains(&ing.name_any())
1039                    && ing.metadata.labels == Some(kanidm.generate_labels())
1040            })
1041            .collect::<Vec<_>>()
1042    };
1043    if !deprecated_ingresses.is_empty() {
1044        changed = true;
1045    }
1046    let ingress_delete_futures = deprecated_ingresses
1047        .iter()
1048        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1049        .collect::<TryJoinAll<_>>();
1050
1051    let ingress_futures = kanidm
1052        .create_ingress()
1053        .into_iter()
1054        .chain(kanidm.create_region_ingress())
1055        .map(|ingress| kanidm.patch(&ctx, ingress))
1056        .collect::<TryJoinAll<_>>();
1057
1058    let deprecated_http_routes = match &ctx.stores.http_route_store {
1059        Some(store) => {
1060            let expected_names = kanidm
1061                .spec
1062                .gateway
1063                .as_ref()
1064                .map(|_| vec![kanidm.name_any()])
1065                .unwrap_or_default();
1066            let routes: Vec<_> = store
1067                .state()
1068                .into_iter()
1069                .filter(|route| {
1070                    route.namespace() == kanidm.namespace()
1071                        && !expected_names.contains(&route.name_any())
1072                        && route.metadata.labels == Some(kanidm.generate_labels())
1073                })
1074                .collect();
1075            if !routes.is_empty() {
1076                changed = true;
1077            }
1078            routes
1079        }
1080        None => Vec::new(),
1081    };
1082    let http_route_delete_futures = deprecated_http_routes
1083        .iter()
1084        .map(|route| kanidm.delete(&ctx, route.as_ref()))
1085        .collect::<TryJoinAll<_>>();
1086
1087    let http_route_futures = if ctx.stores.http_route_store.is_some() {
1088        kanidm
1089            .create_http_route()
1090            .into_iter()
1091            .map(|route| kanidm.patch(&ctx, route))
1092            .collect::<TryJoinAll<_>>()
1093    } else {
1094        try_join_all(Vec::new())
1095    };
1096
1097    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1098        Some(store) => {
1099            let expected_names = kanidm
1100                .spec
1101                .gateway
1102                .as_ref()
1103                .and_then(|g| g.backend_tls_policy.as_ref())
1104                .map(|_| vec![kanidm.name_any()])
1105                .unwrap_or_default();
1106            let policies: Vec<_> = store
1107                .state()
1108                .into_iter()
1109                .filter(|policy| {
1110                    policy.namespace() == kanidm.namespace()
1111                        && !expected_names.contains(&policy.name_any())
1112                        && policy.metadata.labels == Some(kanidm.generate_labels())
1113                })
1114                .collect();
1115            if !policies.is_empty() {
1116                changed = true;
1117            }
1118            policies
1119        }
1120        None => Vec::new(),
1121    };
1122    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1123        .iter()
1124        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1125        .collect::<TryJoinAll<_>>();
1126
1127    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1128        kanidm
1129            .create_backend_tls_policy()
1130            .into_iter()
1131            .map(|policy| kanidm.patch(&ctx, policy))
1132            .collect::<TryJoinAll<_>>()
1133    } else {
1134        try_join_all(Vec::new())
1135    };
1136
1137    let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1138        sts_delete_futures,
1139        admin_secret_future,
1140        replication_secret_futures,
1141        sts_futures,
1142        service_future,
1143        rg_svcs_delete_futures,
1144        rg_services_futures,
1145        ingress_delete_futures,
1146        ingress_futures,
1147        http_route_delete_futures,
1148        http_route_futures,
1149        backend_tls_policy_delete_futures,
1150        backend_tls_policy_futures
1151    )?;
1152
1153    if admin_secret_changed || replication_secrets_changed {
1154        changed = true;
1155    }
1156
1157    if is_kanidm_available(status.clone()) {
1158        let namespace = kanidm.namespace().unwrap();
1159        let name = kanidm.name_any();
1160        match crate::controller::kanidm::KanidmClients::create_client(
1161            &namespace,
1162            &name,
1163            crate::controller::kanidm::KanidmUser::Admin,
1164            ctx.kaniop_ctx.client.clone(),
1165        )
1166        .await
1167        {
1168            Ok(system_client) => {
1169                if let Err(e) =
1170                    reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1171                {
1172                    warn!(%e, "failed to reconcile domain appearance");
1173                }
1174            }
1175            Err(e) => {
1176                warn!(%e, "failed to create admin client for domain appearance reconciliation");
1177            }
1178        }
1179
1180        match crate::controller::kanidm::KanidmClients::create_client(
1181            &namespace,
1182            &name,
1183            crate::controller::kanidm::KanidmUser::IdmAdmin,
1184            ctx.kaniop_ctx.client.clone(),
1185        )
1186        .await
1187        {
1188            Ok(kanidm_client) => {
1189                let (mail_sender_status, mail_sender_mutated) =
1190                    reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1191                        .await
1192                        .unwrap_or_else(|e| {
1193                            warn!(%e, "failed to reconcile mail sender");
1194                            (None, false)
1195                        });
1196                if mail_sender_mutated {
1197                    changed = true;
1198                }
1199                let current_mail_sender_status =
1200                    kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1201                if mail_sender_status != current_mail_sender_status {
1202                    let kanidm_api: Api<Kanidm> =
1203                        Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1204                    let status_patch = serde_json::json!({
1205                        "status": {
1206                            "mailSender": mail_sender_status
1207                        }
1208                    });
1209                    if let Err(e) = kanidm_api
1210                        .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1211                        .await
1212                    {
1213                        warn!(%e, "failed to patch Kanidm/status for mail sender");
1214                    }
1215                }
1216            }
1217            Err(e) => {
1218                warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1219            }
1220        }
1221    }
1222
1223    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1224}
1225
1226async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1227    debug!("cleanup");
1228
1229    let mut changed = false;
1230    let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1231    if mail_resources_cleaned {
1232        changed = true;
1233    }
1234
1235    let namespace = kanidm.namespace().unwrap();
1236    let name = kanidm.name_any();
1237
1238    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1239        &namespace,
1240        &name,
1241        crate::controller::kanidm::KanidmUser::IdmAdmin,
1242        ctx.kaniop_ctx.client.clone(),
1243    )
1244    .await
1245    {
1246        let kanidm_cleanup_done =
1247            cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1248        if kanidm_cleanup_done {
1249            changed = true;
1250        }
1251    }
1252
1253    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1254    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1255}
1256
1257async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1258    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1259    let start = std::time::Instant::now();
1260    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1261    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1262
1263    loop {
1264        let sts = sts_api.get(sts_name).await.map_err(|e| {
1265            Error::KubeError(
1266                format!("failed to get StatefulSet {namespace}/{sts_name}"),
1267                Box::new(e),
1268            )
1269        })?;
1270
1271        if sts.metadata.deletion_timestamp.is_some() {
1272            return Err(Error::ReceiveOutput(format!(
1273                "StatefulSet {namespace}/{sts_name} is being deleted"
1274            )));
1275        }
1276
1277        let spec = sts.spec.as_ref().ok_or_else(|| {
1278            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1279        })?;
1280
1281        let status = match sts.status {
1282            Some(ref s) => s,
1283            None => {
1284                if start.elapsed() >= timeout {
1285                    return Err(Error::ReceiveOutput(format!(
1286                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1287                    )));
1288                }
1289                sleep(poll_interval).await;
1290                continue;
1291            }
1292        };
1293
1294        let replicas = spec.replicas.unwrap_or(0);
1295
1296        if replicas == 0 {
1297            return Ok(());
1298        }
1299
1300        let updated_replicas = status.updated_replicas.unwrap_or(0);
1301        let current_revision = status.current_revision.as_deref().unwrap_or("");
1302        let update_revision = status.update_revision.as_deref().unwrap_or("");
1303        let ready_replicas = status.ready_replicas.unwrap_or(0);
1304
1305        if updated_replicas == replicas
1306            && current_revision == update_revision
1307            && ready_replicas == replicas
1308        {
1309            return Ok(());
1310        }
1311
1312        if start.elapsed() >= timeout {
1313            return Err(Error::ReceiveOutput(format!(
1314                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1315            )));
1316        }
1317
1318        sleep(poll_interval).await;
1319    }
1320}
1321
1322async fn show_replica_cert_with_retries(
1323    kanidm: &Kanidm,
1324    ctx: Arc<Context>,
1325    pod_name: &str,
1326    max_attempts: u32,
1327) -> std::result::Result<String, Error> {
1328    let mut last_error = None;
1329    for attempt in 0..max_attempts {
1330        if attempt > 0 {
1331            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1332        }
1333        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1334            Ok(cert) => return Ok(cert),
1335            Err(e) => {
1336                info!(
1337                    "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1338                    attempt + 1,
1339                    max_attempts
1340                );
1341                last_error = Some(e);
1342            }
1343        }
1344    }
1345    Err(last_error.unwrap_or_else(|| {
1346        Error::ReceiveOutput(format!(
1347            "failed to get certificate for {pod_name} after {max_attempts} attempts"
1348        ))
1349    }))
1350}
1351
1352impl Kanidm {
1353    // Convenience methods that handle context and operator name
1354    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1355    where
1356        K: Resource<Scope = NamespaceResourceScope>
1357            + Serialize
1358            + Clone
1359            + std::fmt::Debug
1360            + for<'de> Deserialize<'de>,
1361        <K as kube::Resource>::DynamicType: Default,
1362        <K as Resource>::Scope: std::marker::Sized,
1363    {
1364        self.kube_delete(
1365            ctx.kaniop_ctx.client.clone(),
1366            &ctx.kaniop_ctx.metrics,
1367            resource,
1368        )
1369        .await
1370    }
1371
1372    /// Server-side apply without any generic delete/recreate fallback.
1373    /// Resource-specific reconcilers use this when they own lifecycle decisions.
1374    pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1375    where
1376        K: Resource<Scope = NamespaceResourceScope>
1377            + Serialize
1378            + Clone
1379            + std::fmt::Debug
1380            + for<'de> Deserialize<'de>,
1381        <K as kube::Resource>::DynamicType: Default,
1382        <K as Resource>::Scope: std::marker::Sized,
1383    {
1384        self.kube_apply(
1385            ctx.kaniop_ctx.client.clone(),
1386            resource,
1387            KANIDM_OPERATOR_NAME,
1388        )
1389        .await
1390    }
1391
1392    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1393    where
1394        K: Resource<Scope = NamespaceResourceScope>
1395            + Serialize
1396            + Clone
1397            + std::fmt::Debug
1398            + for<'de> Deserialize<'de>,
1399        <K as kube::Resource>::DynamicType: Default,
1400        <K as Resource>::Scope: std::marker::Sized,
1401    {
1402        self.kube_patch(
1403            ctx.kaniop_ctx.client.clone(),
1404            &ctx.kaniop_ctx.metrics,
1405            resource,
1406            KANIDM_OPERATOR_NAME,
1407        )
1408        .await
1409    }
1410
1411    #[inline]
1412    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1413        LABELS
1414            .clone()
1415            .into_iter()
1416            .chain([
1417                (INSTANCE_LABEL.to_string(), self.name_any()),
1418                (CLUSTER_LABEL.to_string(), self.name_any()),
1419            ])
1420            .collect()
1421    }
1422
1423    #[inline]
1424    fn generate_labels(&self) -> BTreeMap<String, String> {
1425        self.generate_resource_labels()
1426    }
1427
1428    #[inline]
1429    fn get_tls_secret_name(&self) -> String {
1430        format!("{}-tls", self.name_any())
1431    }
1432
1433    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
1434    /// falling back to the ingress TLS secret and then to the default name.
1435    pub(crate) fn effective_tls_secret_name(&self) -> String {
1436        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1437            self.spec
1438                .ingress
1439                .as_ref()
1440                .and_then(|i| i.tls_secret_name.clone())
1441                .unwrap_or_else(|| self.get_tls_secret_name())
1442        })
1443    }
1444
1445    #[inline]
1446    fn get_namespace(&self) -> String {
1447        // safe unwrap: Kanidm is namespaced scoped
1448        self.namespace().unwrap()
1449    }
1450
1451    #[inline]
1452    fn is_replication_enabled(&self) -> bool {
1453        self.spec.replica_groups.len() > 1
1454            || self
1455                .spec
1456                .replica_groups
1457                .first()
1458                .is_some_and(|rg| rg.replicas > 1)
1459            || !self.spec.external_replication_nodes.is_empty()
1460    }
1461
1462    #[inline]
1463    fn is_updatable(&self, status: &KanidmStatus) -> bool {
1464        status
1465            .version
1466            .as_ref()
1467            .map(|v| {
1468                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1469                    return false;
1470                }
1471                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1472                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1473                match v.upgrade_check_result {
1474                    KanidmUpgradeCheckResult::Passed => {
1475                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1476                            minor <= v_minor + 1
1477                        } else {
1478                            true
1479                        }
1480                    }
1481                    KanidmUpgradeCheckResult::Failed => {
1482                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1483                            versions
1484                        {
1485                            major == v_major && minor == v_minor && patch >= v_patch
1486                        } else {
1487                            v.image_tag == current_tag
1488                        }
1489                    }
1490                }
1491            })
1492            .unwrap_or(true)
1493    }
1494
1495    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1496        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1497        let pod = pod_api.get(pod_name).await.map_err(|e| {
1498            Error::KubeError(
1499                format!("failed to get pod {namespace}/{pod_name}"),
1500                Box::new(e),
1501            )
1502        })?;
1503        Ok(pod
1504            .status
1505            .as_ref()
1506            .and_then(|s| s.conditions.as_ref())
1507            .is_some_and(|conditions| {
1508                conditions
1509                    .iter()
1510                    .any(|c| c.type_ == "Ready" && c.status == "True")
1511            }))
1512    }
1513
1514    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1515        let start = std::time::Instant::now();
1516        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1517        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1518
1519        loop {
1520            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1521                Ok(true) => return Ok(()),
1522                Ok(false) => {}
1523                Err(e) => {
1524                    if !e.is_retryable() {
1525                        return Err(e);
1526                    }
1527                    trace!(%e, "pod readiness check failed, retrying");
1528                }
1529            }
1530
1531            if start.elapsed() >= timeout {
1532                return Err(Error::ReceiveOutput(format!(
1533                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1534                )));
1535            }
1536
1537            sleep(poll_interval).await;
1538        }
1539    }
1540
1541    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1542    where
1543        I: IntoIterator<Item = T> + Debug,
1544        T: Into<String>,
1545    {
1546        let namespace = &self.get_namespace();
1547        trace!(
1548            resource.name = &pod_name,
1549            resource.namespace = &namespace,
1550            ?command,
1551            "pod exec"
1552        );
1553        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1554        let attached = pod
1555            .exec(
1556                pod_name,
1557                command,
1558                &AttachParams::default().container("kanidm"),
1559            )
1560            .await
1561            .map_err(|e| {
1562                Error::KubeError(
1563                    format!("failed to exec pod {namespace}/{pod_name}"),
1564                    Box::new(e),
1565                )
1566            })?;
1567        get_output(attached).await
1568    }
1569
1570    async fn exec_with_wait<I, T>(
1571        &self,
1572        ctx: Arc<Context>,
1573        pod_name: &str,
1574        command: I,
1575    ) -> Result<String>
1576    where
1577        I: IntoIterator<Item = T> + Debug,
1578        T: Into<String>,
1579    {
1580        let namespace = &self.get_namespace();
1581        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1582        self.exec(ctx, pod_name, command).await
1583    }
1584
1585    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1586    where
1587        I: IntoIterator<Item = T> + Debug,
1588        T: Into<String>,
1589    {
1590        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1591        let first_replica_group = self
1592            .spec
1593            .replica_groups
1594            .first()
1595            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1596        let sts_name = self.statefulset_name(&first_replica_group.name);
1597        let pod_name = format!("{sts_name}-0");
1598        self.exec(ctx, &pod_name, command).await
1599    }
1600
1601    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1602    where
1603        I: IntoIterator<Item = T> + Debug,
1604        T: Into<String>,
1605    {
1606        let first_replica_group = self
1607            .spec
1608            .replica_groups
1609            .first()
1610            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1611        let sts_name = self.statefulset_name(&first_replica_group.name);
1612        let pod_name = format!("{sts_name}-0");
1613        self.exec_with_wait(ctx, &pod_name, command).await
1614    }
1615}
1616
1617#[cfg(test)]
1618mod test {
1619    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1620    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1621
1622    use crate::controller::State;
1623    use crate::kanidm::controller::context::{Context, Stores};
1624    use crate::kanidm::crd::KanidmStatus;
1625
1626    use kaniop_k8s_util::error::Result;
1627    use kaniop_k8s_util::resources::hash_secret_data;
1628
1629    use std::collections::BTreeMap;
1630    use std::sync::Arc;
1631
1632    use http::{Request, Response};
1633    use k8s_openapi::ByteString;
1634    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1635    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1636    use k8s_openapi::api::networking::v1::Ingress;
1637    use kube::api::ObjectMeta;
1638    use kube::runtime::reflector::store::Writer;
1639    use kube::{Client, Resource, ResourceExt, client::Body};
1640    use opentelemetry::metrics::MeterProvider;
1641    use serde_json::json;
1642
1643    impl Kanidm {
1644        /// A normal test kanidm with a given status
1645        pub fn test() -> Self {
1646            let mut e = Kanidm::new(
1647                "test",
1648                serde_json::from_value(json!({
1649                    "domain": "idm.example.com",
1650                    "replicaGroups": [
1651                        {
1652                            "name": "default",
1653                            "replicas": 1
1654                        }
1655                    ]
1656                }))
1657                .unwrap(),
1658            );
1659            e.meta_mut().namespace = Some("default".into());
1660            // Pre-populate finalizer so reconcile callback runs immediately
1661            // (skips the "add finalizer + wait for next reconcile" step)
1662            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1663            e
1664        }
1665
1666        pub fn with_ingress(mut self) -> Self {
1667            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1668            self
1669        }
1670
1671        /// Modify kanidm replicas
1672        pub fn with_replicas(mut self, replicas: i32) -> Self {
1673            self.spec.replica_groups[0].replicas = replicas;
1674            self
1675        }
1676
1677        /// Modify kanidm to set a deletion timestamp
1678        pub fn needs_delete(mut self) -> Self {
1679            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1680            use k8s_openapi::jiff::Timestamp;
1681            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1682            self.meta_mut().deletion_timestamp = Some(Time(now));
1683            self
1684        }
1685
1686        /// Modify a kanidm to have an expected status
1687        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1688            self.status = Some(status);
1689            self
1690        }
1691    }
1692
1693    // We wrap tower_test::mock::Handle
1694    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1695    pub struct ApiServerVerifier(ApiServerHandle);
1696
1697    /// Scenarios we test for in ApiServerVerifier
1698    pub enum Scenario {
1699        Create(Kanidm),
1700        CreateWithTwoReplicas(Kanidm),
1701        CreateWithIngress(Kanidm),
1702        CreateWithIngressWithTwoReplicas(Kanidm),
1703        CreateWithTlsSecret(Kanidm, Box<Secret>),
1704    }
1705
1706    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1707        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1708            .await
1709            .expect("timeout on mock apiserver")
1710            .expect("scenario succeeded")
1711    }
1712
1713    impl ApiServerVerifier {
1714        /// Tests only get to run specific scenarios that has matching handlers
1715        ///
1716        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1717        ///
1718        /// NB: If the controller is making more calls than we are handling in the scenario,
1719        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1720        ///
1721        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1722        /// scenario runs to completion (i.e. all expected calls were responded to),
1723        /// using the timeout to catch missing api calls to Kubernetes.
1724        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1725            tokio::spawn(async move {
1726                // moving self => one scenario per test
1727                match scenario {
1728                    Scenario::Create(kanidm) => {
1729                        self.handle_kanidm_get(kanidm.clone())
1730                            .await
1731                            .unwrap()
1732                            .handle_kanidm_status_patch(kanidm.clone())
1733                            .await
1734                            .unwrap()
1735                            .handle_tls_secret_get_not_found(kanidm.clone())
1736                            .await
1737                            .unwrap()
1738                            .handle_statefulset_patch(kanidm.clone(), None)
1739                            .await
1740                            .unwrap()
1741                            .handle_service_patch(kanidm.clone())
1742                            .await
1743                    }
1744                    Scenario::CreateWithTwoReplicas(kanidm) => {
1745                        self.handle_kanidm_get(kanidm.clone())
1746                            .await
1747                            .unwrap()
1748                            .handle_kanidm_status_patch(kanidm.clone())
1749                            .await
1750                            .unwrap()
1751                            .handle_tls_secret_get_not_found(kanidm.clone())
1752                            .await
1753                            .unwrap()
1754                            .handle_statefulset_patch(kanidm.clone(), None)
1755                            .await
1756                            .unwrap()
1757                            .handle_service_patch(kanidm.clone())
1758                            .await
1759                    }
1760                    Scenario::CreateWithIngress(kanidm) => {
1761                        self.handle_kanidm_get(kanidm.clone())
1762                            .await
1763                            .unwrap()
1764                            .handle_kanidm_status_patch(kanidm.clone())
1765                            .await
1766                            .unwrap()
1767                            .handle_tls_secret_get_not_found(kanidm.clone())
1768                            .await
1769                            .unwrap()
1770                            .handle_statefulset_patch(kanidm.clone(), None)
1771                            .await
1772                            .unwrap()
1773                            .handle_service_patch(kanidm.clone())
1774                            .await
1775                            .unwrap()
1776                            .handle_ingress_patch(kanidm.clone())
1777                            .await
1778                    }
1779                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1780                        self.handle_kanidm_get(kanidm.clone())
1781                            .await
1782                            .unwrap()
1783                            .handle_kanidm_status_patch(kanidm.clone())
1784                            .await
1785                            .unwrap()
1786                            .handle_tls_secret_get_not_found(kanidm.clone())
1787                            .await
1788                            .unwrap()
1789                            .handle_statefulset_patch(kanidm.clone(), None)
1790                            .await
1791                            .unwrap()
1792                            .handle_service_patch(kanidm.clone())
1793                            .await
1794                            .unwrap()
1795                            .handle_ingress_patch(kanidm.clone())
1796                            .await
1797                    }
1798                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1799                        let expected_hash = hash_secret_data(&secret);
1800                        self.handle_kanidm_get(kanidm.clone())
1801                            .await
1802                            .unwrap()
1803                            .handle_kanidm_status_patch(kanidm.clone())
1804                            .await
1805                            .unwrap()
1806                            .handle_tls_secret_get(kanidm.clone(), *secret)
1807                            .await
1808                            .unwrap()
1809                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1810                            .await
1811                            .unwrap()
1812                            .handle_service_patch(kanidm.clone())
1813                            .await
1814                    }
1815                }
1816                .expect("scenario completed without errors");
1817            })
1818        }
1819
1820        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1821            let (request, send) = self.0.next_request().await.expect("service not called");
1822            assert_eq!(request.method(), http::Method::GET);
1823            let expected_uri_prefix = format!(
1824                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1825                kanidm.name_any()
1826            );
1827            let uri = request.uri().to_string();
1828            assert!(
1829                uri.starts_with(&expected_uri_prefix),
1830                "expected uri to start with {}, got {}",
1831                expected_uri_prefix,
1832                uri
1833            );
1834            let response = serde_json::to_vec(&kanidm).unwrap();
1835            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1836            Ok(self)
1837        }
1838
1839        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1840            let (request, send) = self.0.next_request().await.expect("service not called");
1841            assert_eq!(request.method(), http::Method::PATCH);
1842            assert_eq!(
1843                request.uri().to_string(),
1844                format!(
1845                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1846                    kanidm.name_any()
1847                )
1848            );
1849
1850            let req_body = request.into_body().collect_bytes().await.unwrap();
1851            let json: serde_json::Value =
1852                serde_json::from_slice(&req_body).expect("patch object is json");
1853            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1854                .expect("valid kanidm status");
1855            let mut kanidm = Kanidm::test();
1856            kanidm.status = Some(status.clone());
1857            let response = serde_json::to_vec(&kanidm).unwrap();
1858            // pass through kanidm "patch accepted"
1859            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1860            Ok(self)
1861        }
1862
1863        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1864            let (request, send) = self.0.next_request().await.expect("service not called");
1865            assert_eq!(request.method(), http::Method::GET);
1866            let expected_uri_prefix = format!(
1867                "/api/v1/namespaces/default/secrets/{}",
1868                kanidm.effective_tls_secret_name()
1869            );
1870            let uri = request.uri().to_string();
1871            assert!(
1872                uri.starts_with(&expected_uri_prefix),
1873                "expected uri to start with {expected_uri_prefix}, got {uri}"
1874            );
1875            let status = serde_json::json!({
1876                "kind": "Status",
1877                "apiVersion": "v1",
1878                "metadata": {},
1879                "status": "Failure",
1880                "message": "secret not found",
1881                "reason": "NotFound",
1882                "code": 404
1883            });
1884            send.send_response(
1885                Response::builder()
1886                    .status(404)
1887                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1888                    .unwrap(),
1889            );
1890            Ok(self)
1891        }
1892
1893        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1894            let (request, send) = self.0.next_request().await.expect("service not called");
1895            assert_eq!(request.method(), http::Method::GET);
1896            let expected_uri_prefix = format!(
1897                "/api/v1/namespaces/default/secrets/{}",
1898                kanidm.effective_tls_secret_name()
1899            );
1900            let uri = request.uri().to_string();
1901            assert!(
1902                uri.starts_with(&expected_uri_prefix),
1903                "expected uri to start with {expected_uri_prefix}, got {uri}"
1904            );
1905            let response = serde_json::to_vec(&secret).unwrap();
1906            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1907            Ok(self)
1908        }
1909
1910        async fn handle_statefulset_patch(
1911            mut self,
1912            kanidm: Kanidm,
1913            expected_tls_hash: Option<String>,
1914        ) -> Result<Self> {
1915            for rg in kanidm.spec.replica_groups.iter() {
1916                let (request, send) = self.0.next_request().await.expect("service not called");
1917                assert_eq!(request.method(), http::Method::PATCH);
1918                assert_eq!(
1919                    request.uri().to_string(),
1920                    format!(
1921                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1922                        kanidm.statefulset_name(&rg.name)
1923                    )
1924                );
1925                let req_body = request.into_body().collect_bytes().await.unwrap();
1926                let json: serde_json::Value =
1927                    serde_json::from_slice(&req_body).expect("patch object is json");
1928                let statefulset: StatefulSet =
1929                    serde_json::from_value(json).expect("valid statefulset");
1930                assert_eq!(
1931                    statefulset.clone().spec.unwrap().replicas.unwrap(),
1932                    rg.replicas
1933                );
1934                let tls_hash_annotation = statefulset
1935                    .spec
1936                    .as_ref()
1937                    .unwrap()
1938                    .template
1939                    .metadata
1940                    .as_ref()
1941                    .unwrap()
1942                    .annotations
1943                    .as_ref()
1944                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
1945                    .cloned();
1946                assert_eq!(tls_hash_annotation, expected_tls_hash);
1947                let response = serde_json::to_vec(&statefulset).unwrap();
1948                // pass through kanidm "patch accepted"
1949                send.send_response(Response::builder().body(Body::from(response)).unwrap());
1950            }
1951
1952            Ok(self)
1953        }
1954
1955        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1956            let (request, send) = self.0.next_request().await.expect("service not called");
1957            assert_eq!(request.method(), http::Method::PATCH);
1958            assert_eq!(
1959                request.uri().to_string(),
1960                format!(
1961                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1962                    kanidm.name_any()
1963                )
1964            );
1965
1966            let req_body = request.into_body().collect_bytes().await.unwrap();
1967            let json: serde_json::Value =
1968                serde_json::from_slice(&req_body).expect("patch object is json");
1969            let service: Service = serde_json::from_value(json).expect("valid service");
1970            let response = serde_json::to_vec(&service).unwrap();
1971            // pass through kanidm "patch accepted"
1972            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1973            Ok(self)
1974        }
1975
1976        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1977            let (request, send) = self.0.next_request().await.expect("service not called");
1978            assert_eq!(request.method(), http::Method::PATCH);
1979            assert_eq!(
1980                request.uri().to_string(),
1981                format!(
1982                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1983                    kanidm.name_any()
1984                )
1985            );
1986
1987            let req_body = request.into_body().collect_bytes().await.unwrap();
1988            let json: serde_json::Value =
1989                serde_json::from_slice(&req_body).expect("patch object is json");
1990            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1991            let response = serde_json::to_vec(&ingress).unwrap();
1992            // pass through kanidm "patch accepted"
1993            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1994            Ok(self)
1995        }
1996    }
1997
1998    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1999        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2000        let mock_client = Client::new(mock_service, "default");
2001        let stores = Stores {
2002            stateful_set_store: Writer::default().as_reader(),
2003            service_store: Writer::default().as_reader(),
2004            ingress_store: Writer::default().as_reader(),
2005            secret_store: Writer::default().as_reader(),
2006            http_route_store: Some(Writer::default().as_reader()),
2007            backend_tls_policy_store: Some(Writer::default().as_reader()),
2008            deployment_store: Writer::default().as_reader(),
2009            config_map_store: Writer::default().as_reader(),
2010        };
2011        let controller_id = "test";
2012
2013        // Create a test meter for metrics
2014        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2015        let meter = provider.meter("test");
2016        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2017
2018        let state = State::new(
2019            metrics,
2020            Writer::default().as_reader(),
2021            Writer::default().as_reader(),
2022            None,
2023        );
2024        let ctx = Arc::new(Context::new(
2025            state.to_context(mock_client, controller_id),
2026            stores,
2027        ));
2028        (ctx, ApiServerVerifier(handle))
2029    }
2030
2031    #[tokio::test]
2032    async fn kanidm_create() {
2033        let (testctx, fakeserver) = get_test_context();
2034        let kanidm = Kanidm::test();
2035        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2036        reconcile_kanidm(Arc::new(kanidm), testctx)
2037            .await
2038            .expect("reconciler");
2039        timeout_after_1s(mocksrv).await;
2040    }
2041
2042    #[tokio::test]
2043    async fn kanidm_create_with_two_replicas() {
2044        let (testctx, fakeserver) = get_test_context();
2045        let kanidm = Kanidm::test().with_replicas(2);
2046        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2047        reconcile_kanidm(Arc::new(kanidm), testctx)
2048            .await
2049            .expect("reconciler");
2050        timeout_after_1s(mocksrv).await;
2051    }
2052
2053    #[tokio::test]
2054    async fn kanidm_create_with_ingress() {
2055        let (testctx, fakeserver) = get_test_context();
2056        let kanidm = Kanidm::test().with_ingress();
2057        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2058        reconcile_kanidm(Arc::new(kanidm), testctx)
2059            .await
2060            .expect("reconciler");
2061        timeout_after_1s(mocksrv).await;
2062    }
2063
2064    #[tokio::test]
2065    async fn kanidm_create_with_ingress_with_two_replicas() {
2066        let (testctx, fakeserver) = get_test_context();
2067        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2068        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2069        reconcile_kanidm(Arc::new(kanidm), testctx)
2070            .await
2071            .expect("reconciler");
2072        timeout_after_1s(mocksrv).await;
2073    }
2074
2075    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2076        Arc::new(StatefulSet {
2077            metadata: ObjectMeta {
2078                name: Some(format!("{cluster}-default")),
2079                namespace: Some(namespace.to_string()),
2080                labels: Some(BTreeMap::from([(
2081                    CLUSTER_LABEL.to_string(),
2082                    cluster.to_string(),
2083                )])),
2084                ..Default::default()
2085            },
2086            spec: Some(StatefulSetSpec {
2087                template: PodTemplateSpec {
2088                    metadata: Some(ObjectMeta {
2089                        annotations: tls_hash.map(|h| {
2090                            BTreeMap::from([(
2091                                TLS_SECRET_HASH_ANNOTATION.to_string(),
2092                                h.to_string(),
2093                            )])
2094                        }),
2095                        ..Default::default()
2096                    }),
2097                    ..Default::default()
2098                },
2099                ..Default::default()
2100            }),
2101            ..Default::default()
2102        })
2103    }
2104
2105    #[test]
2106    fn test_previous_tls_secret_hash() {
2107        let kanidm = Kanidm::test();
2108
2109        // annotation carried over from an existing statefulset
2110        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2111        assert_eq!(
2112            previous_tls_secret_hash(&kanidm, stss),
2113            Some("abc123".to_string())
2114        );
2115
2116        // no annotation on the existing statefulset
2117        let stss = vec![statefulset_of("test", "default", None)];
2118        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2119
2120        // statefulsets from another cluster or namespace are ignored
2121        let stss = vec![
2122            statefulset_of("other", "default", Some("abc123")),
2123            statefulset_of("test", "other-ns", Some("abc123")),
2124        ];
2125        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2126    }
2127
2128    #[tokio::test]
2129    async fn kanidm_create_with_tls_secret() {
2130        let (testctx, fakeserver) = get_test_context();
2131        let kanidm = Kanidm::test();
2132        let secret = Secret {
2133            metadata: ObjectMeta {
2134                name: Some(kanidm.effective_tls_secret_name()),
2135                namespace: Some("default".to_string()),
2136                ..Default::default()
2137            },
2138            data: Some(BTreeMap::from([
2139                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2140                ("tls.key".to_string(), ByteString(b"key".to_vec())),
2141            ])),
2142            ..Default::default()
2143        };
2144        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2145            kanidm.clone(),
2146            Box::new(secret),
2147        ));
2148        reconcile_kanidm(Arc::new(kanidm), testctx)
2149            .await
2150            .expect("reconciler");
2151        timeout_after_1s(mocksrv).await;
2152    }
2153}