Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::{
21    StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
22    preserve_defaulted_statefulset_fields,
23};
24use self::status::StatusExt;
25use self::status::{is_kanidm_available, is_kanidm_initialized};
26
27use crate::controller::context::KubeOperations;
28use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
29use crate::kanidm::crd::{
30    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
31};
32use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
33use crate::telemetry;
34
35use kaniop_k8s_util::client::get_output;
36use kaniop_k8s_util::error::{Error, Result};
37use kaniop_k8s_util::parse::parse_semver;
38use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
39
40use std::collections::{BTreeMap, BTreeSet};
41use std::fmt::Debug;
42use std::sync::{Arc, LazyLock};
43
44use futures::future::{TryJoinAll, join_all, try_join_all};
45use futures::stream::{self, StreamExt};
46use futures::try_join;
47use k8s_openapi::NamespaceResourceScope;
48use k8s_openapi::api::apps::v1::StatefulSet;
49use k8s_openapi::api::core::v1::{Pod, Secret, Service};
50use kube::Resource;
51use kube::ResourceExt;
52use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
53use kube::runtime::controller::Action;
54use kube::runtime::events::{Event, EventType};
55use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
56use serde::{Deserialize, Serialize};
57use tokio::time::{Duration, sleep};
58use tracing::{Span, debug, field, info, instrument, trace, warn};
59
60const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
61const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
62const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
63const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
64const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
65const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
66const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
67const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
68const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
69
70pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
71const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
72pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
73
74const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
75
76static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
77    BTreeMap::from([
78        (NAME_LABEL.to_string(), "kanidm".to_string()),
79        (
80            MANAGED_BY_LABEL.to_string(),
81            format!("kaniop-{CONTROLLER_ID}"),
82        ),
83    ])
84});
85
86async fn restart_statefulsets_after_replica_secret_update(
87    ctx: Arc<Context>,
88    sts_api: &Api<StatefulSet>,
89    namespace: &str,
90    statefulset_names: &[String],
91    sequential: bool,
92) -> Result<()> {
93    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
94    // Pods must restart after cert secret changes or replication keeps stale TLS material.
95    let restart_futures = statefulset_names
96        .iter()
97        .map(|sts_name| sts_api.restart(sts_name));
98
99    if sequential {
100        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
101        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
102            return Err(Error::kube_error(
103                "restart",
104                "StatefulSet",
105                namespace,
106                "replica certificate secret update statefulsets",
107                first_err,
108            ));
109        }
110    } else {
111        let results: Vec<_> = join_all(restart_futures).await;
112        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
113            return Err(Error::kube_error(
114                "restart",
115                "StatefulSet",
116                namespace,
117                "replica certificate secret update statefulsets",
118                first_err,
119            ));
120        }
121    }
122
123    let rollout_futures = statefulset_names
124        .iter()
125        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
126    try_join_all(rollout_futures).await?;
127
128    Ok(())
129}
130
131pub async fn reconcile_admins_secret(
132    kanidm: Arc<Kanidm>,
133    ctx: Arc<Context>,
134    status: &KanidmStatus,
135) -> Result<bool> {
136    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
137        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
138        kanidm.patch(&ctx, admins_secret).await?;
139        return Ok(true);
140    }
141    Ok(false)
142}
143
144pub async fn reconcile_replication_secrets(
145    kanidm: Arc<Kanidm>,
146    ctx: Arc<Context>,
147    status: &KanidmStatus,
148) -> Result<bool> {
149    let mut changed = false;
150    let expected_secret_names = if kanidm.is_replication_enabled() {
151        status
152            .replica_statuses
153            .iter()
154            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
155            .collect::<Vec<_>>()
156    } else {
157        Vec::new()
158    };
159    let deprecated_secrets = ctx
160        .stores
161        .secret_store
162        .state()
163        .into_iter()
164        .filter(|secret| {
165            secret.namespace() == kanidm.namespace()
166                && kanidm.admins_secret_name() != secret.name_any()
167                && !expected_secret_names.contains(&secret.name_any())
168                && secret
169                    .metadata
170                    .labels
171                    .as_ref()
172                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
173        })
174        .collect::<Vec<_>>();
175    if !deprecated_secrets.is_empty() {
176        changed = true;
177    }
178    let secret_delete_future = deprecated_secrets
179        .iter()
180        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
181        .collect::<TryJoinAll<_>>();
182    try_join!(secret_delete_future)?;
183
184    if kanidm.is_replication_enabled() {
185        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
186        let statefulset_names = status
187            .replica_statuses
188            .iter()
189            .map(|rs| rs.statefulset_name.clone())
190            .collect::<BTreeSet<_>>()
191            .into_iter()
192            .collect::<Vec<_>>();
193
194        let has_pending = status
195            .replica_statuses
196            .iter()
197            .any(|rs| rs.state == KanidmReplicaState::Pending);
198
199        let sts_api =
200            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
201
202        if has_pending {
203            changed = true;
204            stream::iter(
205                status
206                    .replica_statuses
207                    .iter()
208                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
209            )
210            .then(|rs| async {
211                let secret = kanidm
212                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
213                    .await?;
214                kanidm.patch(&ctx, secret).await
215            })
216            .collect::<Vec<_>>()
217            .await
218            .into_iter()
219            .collect::<Result<Vec<_>>>()?;
220
221            let restart_futures = status
222                .replica_statuses
223                .iter()
224                .map(|rs| sts_api.restart(&rs.statefulset_name));
225
226            if has_single_replica {
227                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
228                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
229                    return Err(Error::kube_error(
230                        "restart",
231                        "StatefulSet",
232                        kanidm.get_namespace(),
233                        "pending replica statefulsets",
234                        first_err,
235                    ));
236                }
237            } else {
238                let results: Vec<_> = join_all(restart_futures).await;
239                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
240                    return Err(Error::kube_error(
241                        "restart",
242                        "StatefulSet",
243                        kanidm.get_namespace(),
244                        "pending replica statefulsets",
245                        first_err,
246                    ));
247                }
248            }
249        }
250
251        let has_certificate_host_invalid = status
252            .replica_statuses
253            .iter()
254            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
255
256        let has_certificate_expiring = status
257            .replica_statuses
258            .iter()
259            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
260
261        let has_non_ready_replicas = status
262            .replica_statuses
263            .iter()
264            .any(|rs| rs.state != KanidmReplicaState::Ready);
265
266        let host_invalid_replicas: Vec<_> = status
267            .replica_statuses
268            .iter()
269            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
270            .collect();
271
272        let cert_expiring_replicas: Vec<_> = status
273            .replica_statuses
274            .iter()
275            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
276            .collect();
277
278        if !host_invalid_replicas.is_empty() {
279            changed = true;
280            let restart_futures = host_invalid_replicas
281                .iter()
282                .map(|rs| sts_api.restart(&rs.statefulset_name));
283
284            if has_single_replica {
285                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
286                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
287                    return Err(Error::kube_error(
288                        "restart",
289                        "StatefulSet",
290                        kanidm.get_namespace(),
291                        "certificate host invalid statefulsets",
292                        first_err,
293                    ));
294                }
295            } else {
296                let results: Vec<_> = join_all(restart_futures).await;
297                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
298                    return Err(Error::kube_error(
299                        "restart",
300                        "StatefulSet",
301                        kanidm.get_namespace(),
302                        "certificate host invalid statefulsets",
303                        first_err,
304                    ));
305                }
306            }
307
308            let restarted_sts_names: Vec<_> = host_invalid_replicas
309                .iter()
310                .map(|rs| rs.statefulset_name.clone())
311                .collect::<BTreeSet<_>>()
312                .into_iter()
313                .collect();
314
315            let namespace = kanidm.get_namespace();
316            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
317                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
318            });
319            try_join_all(rollout_futures).await?;
320
321            let renew_futures = host_invalid_replicas
322                .iter()
323                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
324            let results: Vec<_> = join_all(renew_futures).await;
325            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
326                return Err(first_err);
327            }
328
329            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
330
331            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
332                let cert = show_replica_cert_with_retries(
333                    &kanidm,
334                    ctx.clone(),
335                    &rs.pod_name,
336                    CERT_SHOW_RETRY_ATTEMPTS,
337                )
338                .await?;
339                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
340                kanidm.patch(&ctx, secret.clone()).await
341            });
342            try_join_all(cert_futures).await?;
343
344            restart_statefulsets_after_replica_secret_update(
345                ctx.clone(),
346                &sts_api,
347                &kanidm.get_namespace(),
348                &statefulset_names,
349                has_single_replica,
350            )
351            .await?;
352
353            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
354        }
355
356        if !cert_expiring_replicas.is_empty() {
357            changed = true;
358            let renew_futures = cert_expiring_replicas
359                .iter()
360                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
361            let results: Vec<_> = join_all(renew_futures).await;
362            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
363                return Err(first_err);
364            }
365
366            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
367
368            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
369                let cert = show_replica_cert_with_retries(
370                    &kanidm,
371                    ctx.clone(),
372                    &rs.pod_name,
373                    CERT_SHOW_RETRY_ATTEMPTS,
374                )
375                .await?;
376                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
377                kanidm.patch(&ctx, secret.clone()).await
378            });
379            try_join_all(cert_futures).await?;
380
381            restart_statefulsets_after_replica_secret_update(
382                ctx.clone(),
383                &sts_api,
384                &kanidm.get_namespace(),
385                &statefulset_names,
386                has_single_replica,
387            )
388            .await?;
389
390            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
391        }
392
393        if has_non_ready_replicas
394            && !has_pending
395            && !has_certificate_host_invalid
396            && !has_certificate_expiring
397        {
398            changed = true;
399            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
400
401            let restart_futures = status
402                .replica_statuses
403                .iter()
404                .map(|rs| sts_api.restart(&rs.statefulset_name));
405
406            if has_single_replica {
407                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
408                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
409                    return Err(Error::kube_error(
410                        "restart",
411                        "StatefulSet",
412                        kanidm.get_namespace(),
413                        "replica statefulsets",
414                        first_err,
415                    ));
416                }
417            } else {
418                let results: Vec<_> = join_all(restart_futures).await;
419                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
420                    return Err(Error::kube_error(
421                        "restart",
422                        "StatefulSet",
423                        kanidm.get_namespace(),
424                        "replica statefulsets",
425                        first_err,
426                    ));
427                }
428            }
429        }
430    }
431
432    Ok(changed)
433}
434
435#[instrument(skip(ctx, kanidm))]
436pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
437    let trace_id = telemetry::get_trace_id();
438    Span::current().record("trace_id", field::display(&trace_id));
439    let _timer = ctx
440        .kaniop_ctx
441        .metrics
442        .reconcile_count_and_measure(&trace_id);
443    info!(msg = "reconciling Kanidm");
444
445    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
446        debug!(msg = "failed to reconcile status", %e);
447        ctx.kaniop_ctx.metrics.status_update_errors_inc();
448        e
449    })?;
450    let kanidm_api: Api<Kanidm> =
451        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
452    let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
453    let outcome_clone = outcome.clone();
454    let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
455        let outcome = outcome_clone.clone();
456        let ctx = ctx.clone();
457        let status = status.clone();
458        async move {
459            match event {
460                Finalizer::Apply(kanidm) => {
461                    let (action, changed) = reconcile(kanidm, ctx, status).await?;
462                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
463                    Ok(action)
464                }
465                Finalizer::Cleanup(kanidm) => {
466                    let (action, changed) = cleanup(kanidm, ctx).await?;
467                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
468                    Ok(action)
469                }
470            }
471        }
472    })
473    .await
474    .or_else(|e| match e {
475        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
476            debug!(msg = "resource already removed during finalizer cleanup");
477            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
478        }
479        _ => Err(Error::FinalizerError(
480            "failed on kanidm account finalizer".to_string(),
481            Box::new(e),
482        )),
483    })?;
484    let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
485    Ok((action, changed))
486}
487
488/// Hash of the TLS secret content, injected as a pod template annotation so
489/// certificate renewals roll the statefulsets. `None` if the secret does not
490/// exist yet.
491async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
492    let secret_name = kanidm.effective_tls_secret_name();
493    let secret_api =
494        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
495    let secret = secret_api
496        .get_opt(&secret_name)
497        .await
498        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
499    Ok(secret.as_ref().map(hash_secret_data))
500}
501
502/// Previous TLS secret hash annotation from the existing statefulsets.
503///
504/// Used when the TLS secret is missing: dropping the annotation would roll
505/// the pods onto a missing volume and leave them stuck in ContainerCreating.
506fn previous_tls_secret_hash(
507    kanidm: &Kanidm,
508    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
509) -> Option<String> {
510    stateful_sets
511        .into_iter()
512        .filter(|sts| {
513            sts.namespace() == kanidm.namespace()
514                && sts
515                    .metadata
516                    .labels
517                    .as_ref()
518                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
519        })
520        .find_map(|sts| {
521            sts.spec
522                .as_ref()?
523                .template
524                .metadata
525                .as_ref()?
526                .annotations
527                .as_ref()?
528                .get(TLS_SECRET_HASH_ANNOTATION)
529                .cloned()
530        })
531}
532
533fn is_headless_service(service: &Service) -> bool {
534    service
535        .spec
536        .as_ref()
537        .and_then(|spec| spec.cluster_ip.as_deref())
538        == Some("None")
539}
540
541fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
542    if is_headless_service(desired) != is_headless_service(current) {
543        return;
544    }
545    let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
546    else {
547        return;
548    };
549    if desired_spec.cluster_ip.is_none() {
550        desired_spec.cluster_ip = current_spec.cluster_ip.clone();
551    }
552    if desired_spec.cluster_ips.is_none() {
553        desired_spec.cluster_ips = current_spec.cluster_ips.clone();
554    }
555}
556
557async fn wait_for_statefulset_deletion(
558    client: kube::Client,
559    namespace: &str,
560    name: &str,
561) -> Result<()> {
562    let api = Api::<StatefulSet>::namespaced(client, namespace);
563    let start = std::time::Instant::now();
564    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
565    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
566
567    loop {
568        match api.get_opt(name).await {
569            Ok(None) => return Ok(()),
570            Ok(Some(_)) => {}
571            Err(e) => {
572                return Err(Error::kube_error(
573                    "wait for deletion of",
574                    "StatefulSet",
575                    namespace,
576                    name,
577                    e,
578                ));
579            }
580        }
581
582        if start.elapsed() >= timeout {
583            return Err(Error::ReceiveOutput(format!(
584                "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
585            )));
586        }
587        sleep(poll_interval).await;
588    }
589}
590
591async fn reconcile_service(
592    kanidm: &Kanidm,
593    ctx: &Context,
594    mut desired: Service,
595) -> Result<Service> {
596    let namespace = kanidm.get_namespace();
597    let name = desired.name_any();
598    let desired_headless = is_headless_service(&desired);
599
600    if let Some(cached) = ctx
601        .stores
602        .service_store
603        .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
604    {
605        if is_headless_service(cached.as_ref()) == desired_headless {
606            preserve_defaulted_service_fields(&mut desired, cached.as_ref());
607        } else {
608            let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
609            let current = service_api
610                .get_opt(&name)
611                .await
612                .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
613
614            if let Some(current) = current {
615                let current_headless = is_headless_service(&current);
616                if current_headless != desired_headless {
617                    info!(
618                        msg = "recreating Service to change headless mode",
619                        resource.name = &name,
620                        resource.namespace = &namespace,
621                        current_headless,
622                        desired_headless,
623                    );
624                    kanidm.delete(ctx, &current).await?;
625                    ctx.kaniop_ctx
626                        .metrics
627                        .reconcile_deploy_delete_create_inc("Service", "headless_transition");
628                    wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
629                        .await?;
630                } else {
631                    preserve_defaulted_service_fields(&mut desired, &current);
632                }
633            }
634        }
635    }
636
637    kanidm.patch(ctx, desired).await
638}
639
640async fn wait_for_service_deletion(
641    client: kube::Client,
642    namespace: &str,
643    name: &str,
644) -> Result<()> {
645    let api = Api::<Service>::namespaced(client, namespace);
646    let start = std::time::Instant::now();
647    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
648    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
649
650    loop {
651        match api.get_opt(name).await {
652            Ok(None) => return Ok(()),
653            Ok(Some(_)) => {}
654            Err(e) => {
655                return Err(Error::kube_error(
656                    "wait for deletion of",
657                    "Service",
658                    namespace,
659                    name,
660                    e,
661                ));
662            }
663        }
664
665        if start.elapsed() >= timeout {
666            return Err(Error::ReceiveOutput(format!(
667                "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
668            )));
669        }
670        sleep(poll_interval).await;
671    }
672}
673
674async fn validate_statefulset_replacement(
675    api: &Api<StatefulSet>,
676    desired: &StatefulSet,
677    namespace: &str,
678    name: &str,
679) -> Result<()> {
680    let mut candidate = desired.clone();
681    candidate.metadata.name = None;
682    candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
683    candidate.metadata.namespace = None;
684    candidate.metadata.resource_version = None;
685    candidate.metadata.uid = None;
686    candidate.metadata.creation_timestamp = None;
687    candidate.metadata.generation = None;
688    candidate.metadata.managed_fields = None;
689    candidate.metadata.deletion_timestamp = None;
690    candidate.metadata.deletion_grace_period_seconds = None;
691    candidate.status = None;
692
693    api.create(
694        &PostParams {
695            dry_run: true,
696            ..Default::default()
697        },
698        &candidate,
699    )
700    .await
701    .map(|_| ())
702    .map_err(|e| {
703        Error::kube_error(
704            "validate replacement for",
705            "StatefulSet",
706            namespace,
707            name,
708            e,
709        )
710    })
711}
712
713async fn reconcile_statefulset(
714    kanidm: Arc<Kanidm>,
715    ctx: Arc<Context>,
716    desired: StatefulSet,
717) -> Result<StatefulSet> {
718    let namespace = kanidm.namespace().unwrap();
719    let name = desired.name_any();
720    let cached_current = ctx.stores.stateful_set_store.find(|current| {
721        current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
722    });
723
724    // On a cold reflector cache, keep the normal create/update path GET-free. If the optimistic
725    // apply fails with HTTP 422, the object may already exist with an immutable conflict that the
726    // cache has not observed yet. Only 422 errors enter the live classification/recreate recovery
727    // path; 429, 5xx, 403, and transport errors are returned immediately and non-destructively.
728    let cache_miss_422_error = if let Some(cached_current) = cached_current {
729        let mut cached_desired = desired.clone();
730        preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
731        if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
732            == StatefulSetApplyStrategy::Apply
733        {
734            return kanidm.apply(&ctx, cached_desired).await;
735        }
736        None
737    } else {
738        match kanidm.apply(&ctx, desired.clone()).await {
739            Ok(applied) => return Ok(applied),
740            Err(error) => {
741                // Only HTTP 422 can indicate an immutable-field conflict that is safe to
742                // recover from via delete/recreate. Arbitrary failures must not trigger
743                // destruction.
744                if is_kube_422(&error) {
745                    Some(error)
746                } else {
747                    return Err(error);
748                }
749            }
750        }
751    };
752
753    // Never make a destructive decision from reflector state or an SSA error alone. Re-read the
754    // live object and classify it immediately before deletion; another actor or an earlier
755    // reconcile may already have converged the immutable fields while the cache was catching up.
756    let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
757    let Some(live_current) = api
758        .get_opt(&name)
759        .await
760        .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
761    else {
762        return match cache_miss_422_error {
763            Some(error) => Err(error),
764            None => kanidm.apply(&ctx, desired).await,
765        };
766    };
767
768    // Start from the original desired object again. Defaults copied from a stale reflector entry
769    // must not survive into the live re-check and become the reason for a recreation.
770    let mut live_desired = desired.clone();
771    preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
772    match classify_statefulset_change(&live_current, &live_desired) {
773        StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
774            Some(error) => Err(error),
775            None => kanidm.apply(&ctx, live_desired).await,
776        },
777        StatefulSetApplyStrategy::Recreate { .. } => {
778            // An update 422 can combine an immutable conflict with independently-invalid desired
779            // state. Validate the replacement as a dry-run CREATE before deleting the live object,
780            // so create-time validation runs without the existing object's immutable-update check.
781            validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
782
783            // Re-read and reclassify immediately before destruction. Another actor or an earlier
784            // reconcile may have changed the live immutable fields or PVC-retention semantics.
785            let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
786                Error::kube_error(
787                    "re-read before deletion",
788                    "StatefulSet",
789                    &namespace,
790                    &name,
791                    e,
792                )
793            })?
794            else {
795                return kanidm.apply(&ctx, live_desired).await;
796            };
797            let mut latest_desired = live_desired.clone();
798            preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
799
800            match classify_statefulset_change(&latest_current, &latest_desired) {
801                StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
802                StatefulSetApplyStrategy::Recreate { immutable_fields } => {
803                    // Validate the replacement StatefulSet via a dry-run CREATE before deleting
804                    // the live object. This prevents deleting on a mixed 422 (immutable conflict
805                    // + independently-invalid desired state) where the replacement would still
806                    // be rejected by the API.
807                    validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
808                        .await?;
809
810                    info!(
811                        msg = "recreating StatefulSet because immutable fields changed",
812                        resource.name = %name,
813                        resource.namespace = %namespace,
814                        ?immutable_fields,
815                    );
816                    kanidm.delete(&ctx, &latest_current).await?;
817                    ctx.kaniop_ctx
818                        .metrics
819                        .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
820                    wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
821                        .await?;
822                    kanidm.apply(&ctx, latest_desired).await
823                }
824            }
825        }
826    }
827}
828
829/// Return true only when the error structurally wraps a Kubernetes HTTP 422 response.
830fn is_kube_422(error: &Error) -> bool {
831    matches!(
832        error,
833        Error::KubeError(_, cause)
834            if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
835    )
836}
837
838#[cfg(test)]
839mod statefulset_recreate_recovery_tests {
840    use super::{Error, is_kube_422};
841
842    fn kube_api_error(code: u16) -> Error {
843        Error::KubeError(
844            "test error".to_string(),
845            Box::new(kube::Error::Api(Box::new(kube::error::Status {
846                code,
847                message: "test".to_string(),
848                reason: "test".to_string(),
849                ..Default::default()
850            }))),
851        )
852    }
853
854    #[test]
855    fn recreate_recovery_accepts_only_422() {
856        assert!(is_kube_422(&kube_api_error(422)));
857        assert!(!is_kube_422(&kube_api_error(403)));
858        assert!(!is_kube_422(&kube_api_error(429)));
859        assert!(!is_kube_422(&kube_api_error(500)));
860    }
861
862    #[test]
863    fn recreate_recovery_rejects_429_too_many_requests() {
864        assert!(!is_kube_422(&kube_api_error(429)));
865    }
866
867    #[test]
868    fn recreate_recovery_rejects_5xx_server_errors() {
869        assert!(!is_kube_422(&kube_api_error(500)));
870        assert!(!is_kube_422(&kube_api_error(502)));
871        assert!(!is_kube_422(&kube_api_error(503)));
872        assert!(!is_kube_422(&kube_api_error(504)));
873    }
874
875    #[test]
876    fn recreate_recovery_rejects_400_and_403() {
877        assert!(!is_kube_422(&kube_api_error(400)));
878        assert!(!is_kube_422(&kube_api_error(403)));
879    }
880}
881
882async fn reconcile(
883    kanidm: Arc<Kanidm>,
884    ctx: Arc<Context>,
885    status: KanidmStatus,
886) -> Result<(Action, bool)> {
887    if kanidm
888        .annotations()
889        .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
890    {
891        ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
892        return Ok((Action::requeue(Duration::from_secs(5)), false));
893    }
894
895    let mut changed = false;
896    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
897    let replication_secret_futures =
898        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
899
900    let sts_to_delete = {
901        let expected_sts_names = kanidm
902            .spec
903            .replica_groups
904            .iter()
905            .map(|rg| kanidm.statefulset_name(&rg.name))
906            .collect::<Vec<_>>();
907        ctx.stores
908            .stateful_set_store
909            .state()
910            .into_iter()
911            .filter(|sts| {
912                sts.namespace() == kanidm.namespace()
913                    && sts
914                        .metadata
915                        .labels
916                        .as_ref()
917                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
918                    && !expected_sts_names.contains(&sts.name_any())
919            })
920            .collect::<Vec<_>>()
921    };
922    if !sts_to_delete.is_empty() {
923        changed = true;
924    }
925    let sts_delete_futures = sts_to_delete
926        .iter()
927        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
928        .collect::<TryJoinAll<_>>();
929
930    let sts_futures = match kanidm.is_updatable(&status) {
931        true => {
932            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
933                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
934            });
935            kanidm
936                .spec
937                .replica_groups
938                .iter()
939                .map(|rg| {
940                    let sts = kanidm.create_statefulset(rg, tls_secret_hash.as_deref())?;
941                    Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
942                })
943                .collect::<Result<TryJoinAll<_>, _>>()?
944        }
945        false => {
946            let note = match status.version.as_ref() {
947                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
948                    format!(
949                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
950                        v.image_tag,
951                        crate::version::KANIDM_CLIENT_VERSION
952                    )
953                }
954                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
955            };
956            let _ignore_error = ctx
957                .kaniop_ctx
958                .recorder
959                .publish(
960                    &Event {
961                        type_: EventType::Warning,
962                        reason: "UpgradeBlocked".to_string(),
963                        note: Some(note),
964                        action: "ReconcileStatefulSet".to_string(),
965                        secondary: None,
966                    },
967                    &kanidm.object_ref(&()),
968                )
969                .await
970                .map_err(|e| {
971                    warn!(msg = "failed to publish KanidmError event", %e);
972                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
973                });
974            try_join_all([])
975        }
976    };
977    let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
978
979    let deprecated_rg_svcs = {
980        let expected_rg_svcs_names = kanidm
981            .spec
982            .replica_groups
983            .iter()
984            .filter(|rg| rg.services.is_some())
985            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
986            .collect::<Vec<_>>();
987        ctx.stores
988            .service_store
989            .state()
990            .into_iter()
991            .filter(|svc| {
992                let labels = svc.metadata.labels.as_ref();
993                svc.namespace() == kanidm.namespace()
994                    && !expected_rg_svcs_names.contains(&svc.name_any())
995                    && labels.is_some_and(|l| {
996                        l.contains_key(REPLICA_LABEL)
997                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
998                    })
999            })
1000            .collect::<Vec<_>>()
1001    };
1002    if !deprecated_rg_svcs.is_empty() {
1003        changed = true;
1004    }
1005    let rg_svcs_delete_futures = deprecated_rg_svcs
1006        .iter()
1007        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1008        .collect::<TryJoinAll<_>>();
1009
1010    let rg_services_futures = kanidm
1011        .spec
1012        .replica_groups
1013        .iter()
1014        .filter(|rg| rg.services.is_some())
1015        .flat_map(|rg| {
1016            (0..rg.replicas).map(|i| {
1017                let mut svc = kanidm.create_replica_group_service(rg, i);
1018                if let Some(current) = ctx.stores.service_store.find(|current| {
1019                    current.namespace() == kanidm.namespace()
1020                        && current.name_any() == svc.name_any()
1021                }) {
1022                    preserve_defaulted_service_fields(&mut svc, current.as_ref());
1023                }
1024                kanidm.patch(&ctx, svc)
1025            })
1026        })
1027        .collect::<TryJoinAll<_>>();
1028
1029    let deprecated_ingresses = {
1030        let names = [
1031            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1032            kanidm.generate_region_ingress_name(),
1033        ]
1034        .into_iter()
1035        .flatten()
1036        .collect::<Vec<_>>();
1037        ctx.stores
1038            .ingress_store
1039            .state()
1040            .into_iter()
1041            .filter(|ing| {
1042                ing.namespace() == kanidm.namespace()
1043                    && !names.contains(&ing.name_any())
1044                    && ing.metadata.labels == Some(kanidm.generate_labels())
1045            })
1046            .collect::<Vec<_>>()
1047    };
1048    if !deprecated_ingresses.is_empty() {
1049        changed = true;
1050    }
1051    let ingress_delete_futures = deprecated_ingresses
1052        .iter()
1053        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1054        .collect::<TryJoinAll<_>>();
1055
1056    let ingress_futures = kanidm
1057        .create_ingress()
1058        .into_iter()
1059        .chain(kanidm.create_region_ingress())
1060        .map(|ingress| kanidm.patch(&ctx, ingress))
1061        .collect::<TryJoinAll<_>>();
1062
1063    let deprecated_http_routes = match &ctx.stores.http_route_store {
1064        Some(store) => {
1065            let expected_names = kanidm
1066                .spec
1067                .gateway
1068                .as_ref()
1069                .map(|_| vec![kanidm.name_any()])
1070                .unwrap_or_default();
1071            let routes: Vec<_> = store
1072                .state()
1073                .into_iter()
1074                .filter(|route| {
1075                    route.namespace() == kanidm.namespace()
1076                        && !expected_names.contains(&route.name_any())
1077                        && route.metadata.labels == Some(kanidm.generate_labels())
1078                })
1079                .collect();
1080            if !routes.is_empty() {
1081                changed = true;
1082            }
1083            routes
1084        }
1085        None => Vec::new(),
1086    };
1087    let http_route_delete_futures = deprecated_http_routes
1088        .iter()
1089        .map(|route| kanidm.delete(&ctx, route.as_ref()))
1090        .collect::<TryJoinAll<_>>();
1091
1092    let http_route_futures = if ctx.stores.http_route_store.is_some() {
1093        kanidm
1094            .create_http_route()
1095            .into_iter()
1096            .map(|route| kanidm.patch(&ctx, route))
1097            .collect::<TryJoinAll<_>>()
1098    } else {
1099        try_join_all(Vec::new())
1100    };
1101
1102    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1103        Some(store) => {
1104            let expected_names = kanidm
1105                .spec
1106                .gateway
1107                .as_ref()
1108                .and_then(|g| g.backend_tls_policy.as_ref())
1109                .map(|_| vec![kanidm.name_any()])
1110                .unwrap_or_default();
1111            let policies: Vec<_> = store
1112                .state()
1113                .into_iter()
1114                .filter(|policy| {
1115                    policy.namespace() == kanidm.namespace()
1116                        && !expected_names.contains(&policy.name_any())
1117                        && policy.metadata.labels == Some(kanidm.generate_labels())
1118                })
1119                .collect();
1120            if !policies.is_empty() {
1121                changed = true;
1122            }
1123            policies
1124        }
1125        None => Vec::new(),
1126    };
1127    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1128        .iter()
1129        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1130        .collect::<TryJoinAll<_>>();
1131
1132    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1133        kanidm
1134            .create_backend_tls_policy()
1135            .into_iter()
1136            .map(|policy| kanidm.patch(&ctx, policy))
1137            .collect::<TryJoinAll<_>>()
1138    } else {
1139        try_join_all(Vec::new())
1140    };
1141
1142    let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1143        sts_delete_futures,
1144        admin_secret_future,
1145        replication_secret_futures,
1146        sts_futures,
1147        service_future,
1148        rg_svcs_delete_futures,
1149        rg_services_futures,
1150        ingress_delete_futures,
1151        ingress_futures,
1152        http_route_delete_futures,
1153        http_route_futures,
1154        backend_tls_policy_delete_futures,
1155        backend_tls_policy_futures
1156    )?;
1157
1158    if admin_secret_changed || replication_secrets_changed {
1159        changed = true;
1160    }
1161
1162    if is_kanidm_available(status.clone()) {
1163        let namespace = kanidm.namespace().unwrap();
1164        let name = kanidm.name_any();
1165        let system_client = crate::controller::kanidm::KanidmClients::create_client(
1166            &namespace,
1167            &name,
1168            crate::controller::kanidm::KanidmUser::Admin,
1169            ctx.kaniop_ctx.client.clone(),
1170        )
1171        .await?;
1172        reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
1173
1174        let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
1175            &namespace,
1176            &name,
1177            crate::controller::kanidm::KanidmUser::IdmAdmin,
1178            ctx.kaniop_ctx.client.clone(),
1179        )
1180        .await?;
1181        let (mail_sender_status, mail_sender_mutated) =
1182            reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
1183        if mail_sender_mutated {
1184            changed = true;
1185        }
1186        let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1187        if mail_sender_status != current_mail_sender_status {
1188            let kanidm_api: Api<Kanidm> =
1189                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1190            let status_patch = serde_json::json!({
1191                "status": {
1192                    "mailSender": mail_sender_status
1193                }
1194            });
1195            kanidm_api
1196                .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1197                .await
1198                .map_err(|e| {
1199                    Error::KubeError(
1200                        format!("failed to patch Kanidm/status {namespace}/{name}"),
1201                        Box::new(e),
1202                    )
1203                })?;
1204        }
1205    }
1206
1207    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1208}
1209
1210async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1211    debug!(msg = "cleanup");
1212
1213    let mut changed = false;
1214    let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1215    if mail_resources_cleaned {
1216        changed = true;
1217    }
1218
1219    let namespace = kanidm.namespace().unwrap();
1220    let name = kanidm.name_any();
1221
1222    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1223        &namespace,
1224        &name,
1225        crate::controller::kanidm::KanidmUser::IdmAdmin,
1226        ctx.kaniop_ctx.client.clone(),
1227    )
1228    .await
1229    {
1230        let kanidm_cleanup_done =
1231            cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1232        if kanidm_cleanup_done {
1233            changed = true;
1234        }
1235    }
1236
1237    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1238    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1239}
1240
1241async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1242    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1243    let start = std::time::Instant::now();
1244    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1245    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1246
1247    loop {
1248        let sts = sts_api.get(sts_name).await.map_err(|e| {
1249            Error::KubeError(
1250                format!("failed to get StatefulSet {namespace}/{sts_name}"),
1251                Box::new(e),
1252            )
1253        })?;
1254
1255        if sts.metadata.deletion_timestamp.is_some() {
1256            return Err(Error::ReceiveOutput(format!(
1257                "StatefulSet {namespace}/{sts_name} is being deleted"
1258            )));
1259        }
1260
1261        let spec = sts.spec.as_ref().ok_or_else(|| {
1262            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1263        })?;
1264
1265        let status = match sts.status {
1266            Some(ref s) => s,
1267            None => {
1268                if start.elapsed() >= timeout {
1269                    return Err(Error::ReceiveOutput(format!(
1270                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1271                    )));
1272                }
1273                sleep(poll_interval).await;
1274                continue;
1275            }
1276        };
1277
1278        let replicas = spec.replicas.unwrap_or(0);
1279
1280        if replicas == 0 {
1281            return Ok(());
1282        }
1283
1284        let updated_replicas = status.updated_replicas.unwrap_or(0);
1285        let current_revision = status.current_revision.as_deref().unwrap_or("");
1286        let update_revision = status.update_revision.as_deref().unwrap_or("");
1287        let ready_replicas = status.ready_replicas.unwrap_or(0);
1288
1289        if updated_replicas == replicas
1290            && current_revision == update_revision
1291            && ready_replicas == replicas
1292        {
1293            return Ok(());
1294        }
1295
1296        if start.elapsed() >= timeout {
1297            return Err(Error::ReceiveOutput(format!(
1298                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1299            )));
1300        }
1301
1302        sleep(poll_interval).await;
1303    }
1304}
1305
1306async fn show_replica_cert_with_retries(
1307    kanidm: &Kanidm,
1308    ctx: Arc<Context>,
1309    pod_name: &str,
1310    max_attempts: u32,
1311) -> std::result::Result<String, Error> {
1312    let mut last_error = None;
1313    for attempt in 0..max_attempts {
1314        if attempt > 0 {
1315            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1316        }
1317        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1318            Ok(cert) => return Ok(cert),
1319            Err(e) => {
1320                info!(
1321                    msg = format!(
1322                        "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1323                        attempt + 1,
1324                        max_attempts
1325                    )
1326                );
1327                last_error = Some(e);
1328            }
1329        }
1330    }
1331    Err(last_error.unwrap_or_else(|| {
1332        Error::ReceiveOutput(format!(
1333            "failed to get certificate for {pod_name} after {max_attempts} attempts"
1334        ))
1335    }))
1336}
1337
1338impl Kanidm {
1339    // Convenience methods that handle context and operator name
1340    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1341    where
1342        K: Resource<Scope = NamespaceResourceScope>
1343            + Serialize
1344            + Clone
1345            + std::fmt::Debug
1346            + for<'de> Deserialize<'de>,
1347        <K as kube::Resource>::DynamicType: Default,
1348        <K as Resource>::Scope: std::marker::Sized,
1349    {
1350        self.kube_delete(
1351            ctx.kaniop_ctx.client.clone(),
1352            &ctx.kaniop_ctx.metrics,
1353            resource,
1354        )
1355        .await
1356    }
1357
1358    /// Server-side apply without any generic delete/recreate fallback.
1359    /// Resource-specific reconcilers use this when they own lifecycle decisions.
1360    pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1361    where
1362        K: Resource<Scope = NamespaceResourceScope>
1363            + Serialize
1364            + Clone
1365            + std::fmt::Debug
1366            + for<'de> Deserialize<'de>,
1367        <K as kube::Resource>::DynamicType: Default,
1368        <K as Resource>::Scope: std::marker::Sized,
1369    {
1370        self.kube_apply(
1371            ctx.kaniop_ctx.client.clone(),
1372            resource,
1373            KANIDM_OPERATOR_NAME,
1374        )
1375        .await
1376    }
1377
1378    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1379    where
1380        K: Resource<Scope = NamespaceResourceScope>
1381            + Serialize
1382            + Clone
1383            + std::fmt::Debug
1384            + for<'de> Deserialize<'de>,
1385        <K as kube::Resource>::DynamicType: Default,
1386        <K as Resource>::Scope: std::marker::Sized,
1387    {
1388        self.kube_patch(
1389            ctx.kaniop_ctx.client.clone(),
1390            &ctx.kaniop_ctx.metrics,
1391            resource,
1392            KANIDM_OPERATOR_NAME,
1393        )
1394        .await
1395    }
1396
1397    #[inline]
1398    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1399        LABELS
1400            .clone()
1401            .into_iter()
1402            .chain([
1403                (INSTANCE_LABEL.to_string(), self.name_any()),
1404                (CLUSTER_LABEL.to_string(), self.name_any()),
1405            ])
1406            .collect()
1407    }
1408
1409    #[inline]
1410    fn generate_labels(&self) -> BTreeMap<String, String> {
1411        self.generate_resource_labels()
1412    }
1413
1414    #[inline]
1415    fn get_tls_secret_name(&self) -> String {
1416        format!("{}-tls", self.name_any())
1417    }
1418
1419    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
1420    /// falling back to the ingress TLS secret and then to the default name.
1421    pub(crate) fn effective_tls_secret_name(&self) -> String {
1422        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1423            self.spec
1424                .ingress
1425                .as_ref()
1426                .and_then(|i| i.tls_secret_name.clone())
1427                .unwrap_or_else(|| self.get_tls_secret_name())
1428        })
1429    }
1430
1431    #[inline]
1432    fn get_namespace(&self) -> String {
1433        // safe unwrap: Kanidm is namespaced scoped
1434        self.namespace().unwrap()
1435    }
1436
1437    #[inline]
1438    fn is_replication_enabled(&self) -> bool {
1439        self.spec.replica_groups.len() > 1
1440            || self
1441                .spec
1442                .replica_groups
1443                .first()
1444                .is_some_and(|rg| rg.replicas > 1)
1445            || !self.spec.external_replication_nodes.is_empty()
1446    }
1447
1448    #[inline]
1449    fn is_updatable(&self, status: &KanidmStatus) -> bool {
1450        status
1451            .version
1452            .as_ref()
1453            .map(|v| {
1454                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1455                    return false;
1456                }
1457                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1458                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1459                match v.upgrade_check_result {
1460                    KanidmUpgradeCheckResult::Passed => {
1461                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1462                            minor <= v_minor + 1
1463                        } else {
1464                            true
1465                        }
1466                    }
1467                    KanidmUpgradeCheckResult::Failed => {
1468                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1469                            versions
1470                        {
1471                            major == v_major && minor == v_minor && patch >= v_patch
1472                        } else {
1473                            v.image_tag == current_tag
1474                        }
1475                    }
1476                }
1477            })
1478            .unwrap_or(true)
1479    }
1480
1481    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1482        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1483        let pod = pod_api.get(pod_name).await.map_err(|e| {
1484            Error::KubeError(
1485                format!("failed to get pod {namespace}/{pod_name}"),
1486                Box::new(e),
1487            )
1488        })?;
1489        Ok(pod
1490            .status
1491            .as_ref()
1492            .and_then(|s| s.conditions.as_ref())
1493            .is_some_and(|conditions| {
1494                conditions
1495                    .iter()
1496                    .any(|c| c.type_ == "Ready" && c.status == "True")
1497            }))
1498    }
1499
1500    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1501        let start = std::time::Instant::now();
1502        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1503        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1504
1505        loop {
1506            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1507                Ok(true) => return Ok(()),
1508                Ok(false) => {}
1509                Err(e) => {
1510                    if !e.is_retryable() {
1511                        return Err(e);
1512                    }
1513                    trace!(msg = "pod readiness check failed, retrying", %e);
1514                }
1515            }
1516
1517            if start.elapsed() >= timeout {
1518                return Err(Error::ReceiveOutput(format!(
1519                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1520                )));
1521            }
1522
1523            sleep(poll_interval).await;
1524        }
1525    }
1526
1527    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1528    where
1529        I: IntoIterator<Item = T> + Debug,
1530        T: Into<String>,
1531    {
1532        let namespace = &self.get_namespace();
1533        trace!(
1534            msg = "pod exec",
1535            resource.name = &pod_name,
1536            resource.namespace = &namespace,
1537            ?command
1538        );
1539        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1540        let attached = pod
1541            .exec(
1542                pod_name,
1543                command,
1544                &AttachParams::default().container("kanidm"),
1545            )
1546            .await
1547            .map_err(|e| {
1548                Error::KubeError(
1549                    format!("failed to exec pod {namespace}/{pod_name}"),
1550                    Box::new(e),
1551                )
1552            })?;
1553        get_output(attached).await
1554    }
1555
1556    async fn exec_with_wait<I, T>(
1557        &self,
1558        ctx: Arc<Context>,
1559        pod_name: &str,
1560        command: I,
1561    ) -> Result<String>
1562    where
1563        I: IntoIterator<Item = T> + Debug,
1564        T: Into<String>,
1565    {
1566        let namespace = &self.get_namespace();
1567        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1568        self.exec(ctx, pod_name, command).await
1569    }
1570
1571    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1572    where
1573        I: IntoIterator<Item = T> + Debug,
1574        T: Into<String>,
1575    {
1576        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1577        let first_replica_group = self
1578            .spec
1579            .replica_groups
1580            .first()
1581            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1582        let sts_name = self.statefulset_name(&first_replica_group.name);
1583        let pod_name = format!("{sts_name}-0");
1584        self.exec(ctx, &pod_name, command).await
1585    }
1586
1587    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1588    where
1589        I: IntoIterator<Item = T> + Debug,
1590        T: Into<String>,
1591    {
1592        let first_replica_group = self
1593            .spec
1594            .replica_groups
1595            .first()
1596            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1597        let sts_name = self.statefulset_name(&first_replica_group.name);
1598        let pod_name = format!("{sts_name}-0");
1599        self.exec_with_wait(ctx, &pod_name, command).await
1600    }
1601}
1602
1603#[cfg(test)]
1604mod test {
1605    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1606    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1607
1608    use crate::controller::State;
1609    use crate::kanidm::controller::context::{Context, Stores};
1610    use crate::kanidm::crd::KanidmStatus;
1611
1612    use kaniop_k8s_util::error::Result;
1613    use kaniop_k8s_util::resources::hash_secret_data;
1614
1615    use std::collections::BTreeMap;
1616    use std::sync::Arc;
1617
1618    use http::{Request, Response};
1619    use k8s_openapi::ByteString;
1620    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1621    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1622    use k8s_openapi::api::networking::v1::Ingress;
1623    use kube::api::ObjectMeta;
1624    use kube::runtime::reflector::store::Writer;
1625    use kube::{Client, Resource, ResourceExt, client::Body};
1626    use opentelemetry::metrics::MeterProvider;
1627    use serde_json::json;
1628
1629    impl Kanidm {
1630        /// A normal test kanidm with a given status
1631        pub fn test() -> Self {
1632            let mut e = Kanidm::new(
1633                "test",
1634                serde_json::from_value(json!({
1635                    "domain": "idm.example.com",
1636                    "replicaGroups": [
1637                        {
1638                            "name": "default",
1639                            "replicas": 1
1640                        }
1641                    ]
1642                }))
1643                .unwrap(),
1644            );
1645            e.meta_mut().namespace = Some("default".into());
1646            // Pre-populate finalizer so reconcile callback runs immediately
1647            // (skips the "add finalizer + wait for next reconcile" step)
1648            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1649            e
1650        }
1651
1652        pub fn with_ingress(mut self) -> Self {
1653            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1654            self
1655        }
1656
1657        /// Modify kanidm replicas
1658        pub fn with_replicas(mut self, replicas: i32) -> Self {
1659            self.spec.replica_groups[0].replicas = replicas;
1660            self
1661        }
1662
1663        /// Modify kanidm to set a deletion timestamp
1664        pub fn needs_delete(mut self) -> Self {
1665            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1666            use k8s_openapi::jiff::Timestamp;
1667            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1668            self.meta_mut().deletion_timestamp = Some(Time(now));
1669            self
1670        }
1671
1672        /// Modify a kanidm to have an expected status
1673        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1674            self.status = Some(status);
1675            self
1676        }
1677    }
1678
1679    // We wrap tower_test::mock::Handle
1680    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1681    pub struct ApiServerVerifier(ApiServerHandle);
1682
1683    /// Scenarios we test for in ApiServerVerifier
1684    pub enum Scenario {
1685        Create(Kanidm),
1686        CreateWithTwoReplicas(Kanidm),
1687        CreateWithIngress(Kanidm),
1688        CreateWithIngressWithTwoReplicas(Kanidm),
1689        CreateWithTlsSecret(Kanidm, Box<Secret>),
1690    }
1691
1692    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1693        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1694            .await
1695            .expect("timeout on mock apiserver")
1696            .expect("scenario succeeded")
1697    }
1698
1699    impl ApiServerVerifier {
1700        /// Tests only get to run specific scenarios that has matching handlers
1701        ///
1702        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1703        ///
1704        /// NB: If the controller is making more calls than we are handling in the scenario,
1705        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1706        ///
1707        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1708        /// scenario runs to completion (i.e. all expected calls were responded to),
1709        /// using the timeout to catch missing api calls to Kubernetes.
1710        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1711            tokio::spawn(async move {
1712                // moving self => one scenario per test
1713                match scenario {
1714                    Scenario::Create(kanidm) => {
1715                        self.handle_kanidm_get(kanidm.clone())
1716                            .await
1717                            .unwrap()
1718                            .handle_kanidm_status_patch(kanidm.clone())
1719                            .await
1720                            .unwrap()
1721                            .handle_tls_secret_get_not_found(kanidm.clone())
1722                            .await
1723                            .unwrap()
1724                            .handle_statefulset_patch(kanidm.clone(), None)
1725                            .await
1726                            .unwrap()
1727                            .handle_service_patch(kanidm.clone())
1728                            .await
1729                    }
1730                    Scenario::CreateWithTwoReplicas(kanidm) => {
1731                        self.handle_kanidm_get(kanidm.clone())
1732                            .await
1733                            .unwrap()
1734                            .handle_kanidm_status_patch(kanidm.clone())
1735                            .await
1736                            .unwrap()
1737                            .handle_tls_secret_get_not_found(kanidm.clone())
1738                            .await
1739                            .unwrap()
1740                            .handle_statefulset_patch(kanidm.clone(), None)
1741                            .await
1742                            .unwrap()
1743                            .handle_service_patch(kanidm.clone())
1744                            .await
1745                    }
1746                    Scenario::CreateWithIngress(kanidm) => {
1747                        self.handle_kanidm_get(kanidm.clone())
1748                            .await
1749                            .unwrap()
1750                            .handle_kanidm_status_patch(kanidm.clone())
1751                            .await
1752                            .unwrap()
1753                            .handle_tls_secret_get_not_found(kanidm.clone())
1754                            .await
1755                            .unwrap()
1756                            .handle_statefulset_patch(kanidm.clone(), None)
1757                            .await
1758                            .unwrap()
1759                            .handle_service_patch(kanidm.clone())
1760                            .await
1761                            .unwrap()
1762                            .handle_ingress_patch(kanidm.clone())
1763                            .await
1764                    }
1765                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1766                        self.handle_kanidm_get(kanidm.clone())
1767                            .await
1768                            .unwrap()
1769                            .handle_kanidm_status_patch(kanidm.clone())
1770                            .await
1771                            .unwrap()
1772                            .handle_tls_secret_get_not_found(kanidm.clone())
1773                            .await
1774                            .unwrap()
1775                            .handle_statefulset_patch(kanidm.clone(), None)
1776                            .await
1777                            .unwrap()
1778                            .handle_service_patch(kanidm.clone())
1779                            .await
1780                            .unwrap()
1781                            .handle_ingress_patch(kanidm.clone())
1782                            .await
1783                    }
1784                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1785                        let expected_hash = hash_secret_data(&secret);
1786                        self.handle_kanidm_get(kanidm.clone())
1787                            .await
1788                            .unwrap()
1789                            .handle_kanidm_status_patch(kanidm.clone())
1790                            .await
1791                            .unwrap()
1792                            .handle_tls_secret_get(kanidm.clone(), *secret)
1793                            .await
1794                            .unwrap()
1795                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1796                            .await
1797                            .unwrap()
1798                            .handle_service_patch(kanidm.clone())
1799                            .await
1800                    }
1801                }
1802                .expect("scenario completed without errors");
1803            })
1804        }
1805
1806        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1807            let (request, send) = self.0.next_request().await.expect("service not called");
1808            assert_eq!(request.method(), http::Method::GET);
1809            let expected_uri_prefix = format!(
1810                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1811                kanidm.name_any()
1812            );
1813            let uri = request.uri().to_string();
1814            assert!(
1815                uri.starts_with(&expected_uri_prefix),
1816                "expected uri to start with {}, got {}",
1817                expected_uri_prefix,
1818                uri
1819            );
1820            let response = serde_json::to_vec(&kanidm).unwrap();
1821            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1822            Ok(self)
1823        }
1824
1825        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1826            let (request, send) = self.0.next_request().await.expect("service not called");
1827            assert_eq!(request.method(), http::Method::PATCH);
1828            assert_eq!(
1829                request.uri().to_string(),
1830                format!(
1831                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1832                    kanidm.name_any()
1833                )
1834            );
1835
1836            let req_body = request.into_body().collect_bytes().await.unwrap();
1837            let json: serde_json::Value =
1838                serde_json::from_slice(&req_body).expect("patch object is json");
1839            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1840                .expect("valid kanidm status");
1841            let mut kanidm = Kanidm::test();
1842            kanidm.status = Some(status.clone());
1843            let response = serde_json::to_vec(&kanidm).unwrap();
1844            // pass through kanidm "patch accepted"
1845            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1846            Ok(self)
1847        }
1848
1849        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1850            let (request, send) = self.0.next_request().await.expect("service not called");
1851            assert_eq!(request.method(), http::Method::GET);
1852            let expected_uri_prefix = format!(
1853                "/api/v1/namespaces/default/secrets/{}",
1854                kanidm.effective_tls_secret_name()
1855            );
1856            let uri = request.uri().to_string();
1857            assert!(
1858                uri.starts_with(&expected_uri_prefix),
1859                "expected uri to start with {expected_uri_prefix}, got {uri}"
1860            );
1861            let status = serde_json::json!({
1862                "kind": "Status",
1863                "apiVersion": "v1",
1864                "metadata": {},
1865                "status": "Failure",
1866                "message": "secret not found",
1867                "reason": "NotFound",
1868                "code": 404
1869            });
1870            send.send_response(
1871                Response::builder()
1872                    .status(404)
1873                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1874                    .unwrap(),
1875            );
1876            Ok(self)
1877        }
1878
1879        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1880            let (request, send) = self.0.next_request().await.expect("service not called");
1881            assert_eq!(request.method(), http::Method::GET);
1882            let expected_uri_prefix = format!(
1883                "/api/v1/namespaces/default/secrets/{}",
1884                kanidm.effective_tls_secret_name()
1885            );
1886            let uri = request.uri().to_string();
1887            assert!(
1888                uri.starts_with(&expected_uri_prefix),
1889                "expected uri to start with {expected_uri_prefix}, got {uri}"
1890            );
1891            let response = serde_json::to_vec(&secret).unwrap();
1892            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1893            Ok(self)
1894        }
1895
1896        async fn handle_statefulset_patch(
1897            mut self,
1898            kanidm: Kanidm,
1899            expected_tls_hash: Option<String>,
1900        ) -> Result<Self> {
1901            for rg in kanidm.spec.replica_groups.iter() {
1902                let (request, send) = self.0.next_request().await.expect("service not called");
1903                assert_eq!(request.method(), http::Method::PATCH);
1904                assert_eq!(
1905                    request.uri().to_string(),
1906                    format!(
1907                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1908                        kanidm.statefulset_name(&rg.name)
1909                    )
1910                );
1911                let req_body = request.into_body().collect_bytes().await.unwrap();
1912                let json: serde_json::Value =
1913                    serde_json::from_slice(&req_body).expect("patch object is json");
1914                let statefulset: StatefulSet =
1915                    serde_json::from_value(json).expect("valid statefulset");
1916                assert_eq!(
1917                    statefulset.clone().spec.unwrap().replicas.unwrap(),
1918                    rg.replicas
1919                );
1920                let tls_hash_annotation = statefulset
1921                    .spec
1922                    .as_ref()
1923                    .unwrap()
1924                    .template
1925                    .metadata
1926                    .as_ref()
1927                    .unwrap()
1928                    .annotations
1929                    .as_ref()
1930                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
1931                    .cloned();
1932                assert_eq!(tls_hash_annotation, expected_tls_hash);
1933                let response = serde_json::to_vec(&statefulset).unwrap();
1934                // pass through kanidm "patch accepted"
1935                send.send_response(Response::builder().body(Body::from(response)).unwrap());
1936            }
1937
1938            Ok(self)
1939        }
1940
1941        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1942            let (request, send) = self.0.next_request().await.expect("service not called");
1943            assert_eq!(request.method(), http::Method::PATCH);
1944            assert_eq!(
1945                request.uri().to_string(),
1946                format!(
1947                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1948                    kanidm.name_any()
1949                )
1950            );
1951
1952            let req_body = request.into_body().collect_bytes().await.unwrap();
1953            let json: serde_json::Value =
1954                serde_json::from_slice(&req_body).expect("patch object is json");
1955            let service: Service = serde_json::from_value(json).expect("valid service");
1956            let response = serde_json::to_vec(&service).unwrap();
1957            // pass through kanidm "patch accepted"
1958            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1959            Ok(self)
1960        }
1961
1962        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1963            let (request, send) = self.0.next_request().await.expect("service not called");
1964            assert_eq!(request.method(), http::Method::PATCH);
1965            assert_eq!(
1966                request.uri().to_string(),
1967                format!(
1968                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1969                    kanidm.name_any()
1970                )
1971            );
1972
1973            let req_body = request.into_body().collect_bytes().await.unwrap();
1974            let json: serde_json::Value =
1975                serde_json::from_slice(&req_body).expect("patch object is json");
1976            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1977            let response = serde_json::to_vec(&ingress).unwrap();
1978            // pass through kanidm "patch accepted"
1979            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1980            Ok(self)
1981        }
1982    }
1983
1984    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1985        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1986        let mock_client = Client::new(mock_service, "default");
1987        let stores = Stores {
1988            stateful_set_store: Writer::default().as_reader(),
1989            service_store: Writer::default().as_reader(),
1990            ingress_store: Writer::default().as_reader(),
1991            secret_store: Writer::default().as_reader(),
1992            http_route_store: Some(Writer::default().as_reader()),
1993            backend_tls_policy_store: Some(Writer::default().as_reader()),
1994            deployment_store: Writer::default().as_reader(),
1995            config_map_store: Writer::default().as_reader(),
1996        };
1997        let controller_id = "test";
1998
1999        // Create a test meter for metrics
2000        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2001        let meter = provider.meter("test");
2002        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2003
2004        let state = State::new(
2005            metrics,
2006            Writer::default().as_reader(),
2007            Writer::default().as_reader(),
2008            None,
2009        );
2010        let ctx = Arc::new(Context::new(
2011            state.to_context(mock_client, controller_id),
2012            stores,
2013        ));
2014        (ctx, ApiServerVerifier(handle))
2015    }
2016
2017    #[tokio::test]
2018    async fn kanidm_create() {
2019        let (testctx, fakeserver) = get_test_context();
2020        let kanidm = Kanidm::test();
2021        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2022        reconcile_kanidm(Arc::new(kanidm), testctx)
2023            .await
2024            .expect("reconciler");
2025        timeout_after_1s(mocksrv).await;
2026    }
2027
2028    #[tokio::test]
2029    async fn kanidm_create_with_two_replicas() {
2030        let (testctx, fakeserver) = get_test_context();
2031        let kanidm = Kanidm::test().with_replicas(2);
2032        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2033        reconcile_kanidm(Arc::new(kanidm), testctx)
2034            .await
2035            .expect("reconciler");
2036        timeout_after_1s(mocksrv).await;
2037    }
2038
2039    #[tokio::test]
2040    async fn kanidm_create_with_ingress() {
2041        let (testctx, fakeserver) = get_test_context();
2042        let kanidm = Kanidm::test().with_ingress();
2043        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2044        reconcile_kanidm(Arc::new(kanidm), testctx)
2045            .await
2046            .expect("reconciler");
2047        timeout_after_1s(mocksrv).await;
2048    }
2049
2050    #[tokio::test]
2051    async fn kanidm_create_with_ingress_with_two_replicas() {
2052        let (testctx, fakeserver) = get_test_context();
2053        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2054        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2055        reconcile_kanidm(Arc::new(kanidm), testctx)
2056            .await
2057            .expect("reconciler");
2058        timeout_after_1s(mocksrv).await;
2059    }
2060
2061    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2062        Arc::new(StatefulSet {
2063            metadata: ObjectMeta {
2064                name: Some(format!("{cluster}-default")),
2065                namespace: Some(namespace.to_string()),
2066                labels: Some(BTreeMap::from([(
2067                    CLUSTER_LABEL.to_string(),
2068                    cluster.to_string(),
2069                )])),
2070                ..Default::default()
2071            },
2072            spec: Some(StatefulSetSpec {
2073                template: PodTemplateSpec {
2074                    metadata: Some(ObjectMeta {
2075                        annotations: tls_hash.map(|h| {
2076                            BTreeMap::from([(
2077                                TLS_SECRET_HASH_ANNOTATION.to_string(),
2078                                h.to_string(),
2079                            )])
2080                        }),
2081                        ..Default::default()
2082                    }),
2083                    ..Default::default()
2084                },
2085                ..Default::default()
2086            }),
2087            ..Default::default()
2088        })
2089    }
2090
2091    #[test]
2092    fn test_previous_tls_secret_hash() {
2093        let kanidm = Kanidm::test();
2094
2095        // annotation carried over from an existing statefulset
2096        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2097        assert_eq!(
2098            previous_tls_secret_hash(&kanidm, stss),
2099            Some("abc123".to_string())
2100        );
2101
2102        // no annotation on the existing statefulset
2103        let stss = vec![statefulset_of("test", "default", None)];
2104        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2105
2106        // statefulsets from another cluster or namespace are ignored
2107        let stss = vec![
2108            statefulset_of("other", "default", Some("abc123")),
2109            statefulset_of("test", "other-ns", Some("abc123")),
2110        ];
2111        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2112    }
2113
2114    #[tokio::test]
2115    async fn kanidm_create_with_tls_secret() {
2116        let (testctx, fakeserver) = get_test_context();
2117        let kanidm = Kanidm::test();
2118        let secret = Secret {
2119            metadata: ObjectMeta {
2120                name: Some(kanidm.effective_tls_secret_name()),
2121                namespace: Some("default".to_string()),
2122                ..Default::default()
2123            },
2124            data: Some(BTreeMap::from([
2125                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2126                ("tls.key".to_string(), ByteString(b"key".to_vec())),
2127            ])),
2128            ..Default::default()
2129        };
2130        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2131            kanidm.clone(),
2132            Box::new(secret),
2133        ));
2134        reconcile_kanidm(Arc::new(kanidm), testctx)
2135            .await
2136            .expect("reconciler");
2137        timeout_after_1s(mocksrv).await;
2138    }
2139}