Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::SecretExt;
20use self::service::ServiceExt;
21use self::statefulset::{
22    StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23    preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_backup_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
31use crate::kanidm::crd::{
32    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
33};
34use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
35use crate::telemetry;
36
37use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
38use kaniop_k8s_util::client::get_output;
39use kaniop_k8s_util::error::{Error, Result};
40use kaniop_k8s_util::parse::parse_semver;
41use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::fmt::Debug;
45use std::sync::{Arc, LazyLock};
46
47use futures::future::{TryJoinAll, join_all, try_join_all};
48use futures::stream::{self, StreamExt};
49use futures::try_join;
50use k8s_openapi::NamespaceResourceScope;
51use k8s_openapi::api::apps::v1::StatefulSet;
52use k8s_openapi::api::core::v1::{Pod, Secret, Service};
53use kube::Resource;
54use kube::ResourceExt;
55use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
56use kube::runtime::controller::Action;
57use kube::runtime::events::{Event, EventType};
58use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
59use serde::{Deserialize, Serialize};
60use tokio::time::{Duration, sleep};
61use tracing::{Span, debug, field, info, instrument, trace, warn};
62
63const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
64const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
65const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
66const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
67const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
68const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
69const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
70const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
71const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
72
73pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
74const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
75pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
76
77const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
78
79static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
80    BTreeMap::from([
81        (NAME_LABEL.to_string(), "kanidm".to_string()),
82        (
83            MANAGED_BY_LABEL.to_string(),
84            format!("kaniop-{CONTROLLER_ID}"),
85        ),
86    ])
87});
88
89async fn restart_statefulsets_after_replica_secret_update(
90    ctx: Arc<Context>,
91    sts_api: &Api<StatefulSet>,
92    namespace: &str,
93    statefulset_names: &[String],
94    sequential: bool,
95) -> Result<()> {
96    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
97    // Pods must restart after cert secret changes or replication keeps stale TLS material.
98    let restart_futures = statefulset_names
99        .iter()
100        .map(|sts_name| sts_api.restart(sts_name));
101
102    if sequential {
103        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
104        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
105            return Err(Error::kube_error(
106                "restart",
107                "StatefulSet",
108                namespace,
109                "replica certificate secret update statefulsets",
110                first_err,
111            ));
112        }
113    } else {
114        let results: Vec<_> = join_all(restart_futures).await;
115        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
116            return Err(Error::kube_error(
117                "restart",
118                "StatefulSet",
119                namespace,
120                "replica certificate secret update statefulsets",
121                first_err,
122            ));
123        }
124    }
125
126    let rollout_futures = statefulset_names
127        .iter()
128        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
129    try_join_all(rollout_futures).await?;
130
131    Ok(())
132}
133
134pub async fn reconcile_admins_secret(
135    kanidm: Arc<Kanidm>,
136    ctx: Arc<Context>,
137    status: &KanidmStatus,
138) -> Result<bool> {
139    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
140        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
141        kanidm.patch(&ctx, admins_secret).await?;
142        return Ok(true);
143    }
144    Ok(false)
145}
146
147pub async fn reconcile_replication_secrets(
148    kanidm: Arc<Kanidm>,
149    ctx: Arc<Context>,
150    status: &KanidmStatus,
151) -> Result<bool> {
152    let mut changed = false;
153    let expected_secret_names = if kanidm.is_replication_enabled() {
154        status
155            .replica_statuses
156            .iter()
157            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
158            .collect::<Vec<_>>()
159    } else {
160        Vec::new()
161    };
162    let deprecated_secrets = ctx
163        .stores
164        .secret_store
165        .state()
166        .into_iter()
167        .filter(|secret| {
168            secret.namespace() == kanidm.namespace()
169                && kanidm.admins_secret_name() != secret.name_any()
170                && !expected_secret_names.contains(&secret.name_any())
171                && secret
172                    .metadata
173                    .labels
174                    .as_ref()
175                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
176        })
177        .collect::<Vec<_>>();
178    if !deprecated_secrets.is_empty() {
179        changed = true;
180    }
181    let secret_delete_future = deprecated_secrets
182        .iter()
183        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
184        .collect::<TryJoinAll<_>>();
185    try_join!(secret_delete_future)?;
186
187    if kanidm.is_replication_enabled() {
188        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
189        let statefulset_names = status
190            .replica_statuses
191            .iter()
192            .map(|rs| rs.statefulset_name.clone())
193            .collect::<BTreeSet<_>>()
194            .into_iter()
195            .collect::<Vec<_>>();
196
197        let has_pending = status
198            .replica_statuses
199            .iter()
200            .any(|rs| rs.state == KanidmReplicaState::Pending);
201
202        let sts_api =
203            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
204
205        if has_pending {
206            changed = true;
207            stream::iter(
208                status
209                    .replica_statuses
210                    .iter()
211                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
212            )
213            .then(|rs| async {
214                let secret = kanidm
215                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
216                    .await?;
217                kanidm.patch(&ctx, secret).await
218            })
219            .collect::<Vec<_>>()
220            .await
221            .into_iter()
222            .collect::<Result<Vec<_>>>()?;
223
224            let restart_futures = status
225                .replica_statuses
226                .iter()
227                .map(|rs| sts_api.restart(&rs.statefulset_name));
228
229            if has_single_replica {
230                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
231                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
232                    return Err(Error::kube_error(
233                        "restart",
234                        "StatefulSet",
235                        kanidm.get_namespace(),
236                        "pending replica statefulsets",
237                        first_err,
238                    ));
239                }
240            } else {
241                let results: Vec<_> = join_all(restart_futures).await;
242                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
243                    return Err(Error::kube_error(
244                        "restart",
245                        "StatefulSet",
246                        kanidm.get_namespace(),
247                        "pending replica statefulsets",
248                        first_err,
249                    ));
250                }
251            }
252        }
253
254        let has_certificate_host_invalid = status
255            .replica_statuses
256            .iter()
257            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
258
259        let has_certificate_expiring = status
260            .replica_statuses
261            .iter()
262            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
263
264        let has_non_ready_replicas = status
265            .replica_statuses
266            .iter()
267            .any(|rs| rs.state != KanidmReplicaState::Ready);
268
269        let host_invalid_replicas: Vec<_> = status
270            .replica_statuses
271            .iter()
272            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
273            .collect();
274
275        let cert_expiring_replicas: Vec<_> = status
276            .replica_statuses
277            .iter()
278            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
279            .collect();
280
281        if !host_invalid_replicas.is_empty() {
282            changed = true;
283            let restart_futures = host_invalid_replicas
284                .iter()
285                .map(|rs| sts_api.restart(&rs.statefulset_name));
286
287            if has_single_replica {
288                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
289                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
290                    return Err(Error::kube_error(
291                        "restart",
292                        "StatefulSet",
293                        kanidm.get_namespace(),
294                        "certificate host invalid statefulsets",
295                        first_err,
296                    ));
297                }
298            } else {
299                let results: Vec<_> = join_all(restart_futures).await;
300                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
301                    return Err(Error::kube_error(
302                        "restart",
303                        "StatefulSet",
304                        kanidm.get_namespace(),
305                        "certificate host invalid statefulsets",
306                        first_err,
307                    ));
308                }
309            }
310
311            let restarted_sts_names: Vec<_> = host_invalid_replicas
312                .iter()
313                .map(|rs| rs.statefulset_name.clone())
314                .collect::<BTreeSet<_>>()
315                .into_iter()
316                .collect();
317
318            let namespace = kanidm.get_namespace();
319            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
320                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
321            });
322            try_join_all(rollout_futures).await?;
323
324            let renew_futures = host_invalid_replicas
325                .iter()
326                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
327            let results: Vec<_> = join_all(renew_futures).await;
328            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
329                return Err(first_err);
330            }
331
332            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
333
334            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
335                let cert = show_replica_cert_with_retries(
336                    &kanidm,
337                    ctx.clone(),
338                    &rs.pod_name,
339                    CERT_SHOW_RETRY_ATTEMPTS,
340                )
341                .await?;
342                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
343                kanidm.patch(&ctx, secret.clone()).await
344            });
345            try_join_all(cert_futures).await?;
346
347            restart_statefulsets_after_replica_secret_update(
348                ctx.clone(),
349                &sts_api,
350                &kanidm.get_namespace(),
351                &statefulset_names,
352                has_single_replica,
353            )
354            .await?;
355
356            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
357        }
358
359        if !cert_expiring_replicas.is_empty() {
360            changed = true;
361            let renew_futures = cert_expiring_replicas
362                .iter()
363                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
364            let results: Vec<_> = join_all(renew_futures).await;
365            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
366                return Err(first_err);
367            }
368
369            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
370
371            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
372                let cert = show_replica_cert_with_retries(
373                    &kanidm,
374                    ctx.clone(),
375                    &rs.pod_name,
376                    CERT_SHOW_RETRY_ATTEMPTS,
377                )
378                .await?;
379                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
380                kanidm.patch(&ctx, secret.clone()).await
381            });
382            try_join_all(cert_futures).await?;
383
384            restart_statefulsets_after_replica_secret_update(
385                ctx.clone(),
386                &sts_api,
387                &kanidm.get_namespace(),
388                &statefulset_names,
389                has_single_replica,
390            )
391            .await?;
392
393            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
394        }
395
396        if has_non_ready_replicas
397            && !has_pending
398            && !has_certificate_host_invalid
399            && !has_certificate_expiring
400        {
401            changed = true;
402            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
403
404            let restart_futures = status
405                .replica_statuses
406                .iter()
407                .map(|rs| sts_api.restart(&rs.statefulset_name));
408
409            if has_single_replica {
410                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
411                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
412                    return Err(Error::kube_error(
413                        "restart",
414                        "StatefulSet",
415                        kanidm.get_namespace(),
416                        "replica statefulsets",
417                        first_err,
418                    ));
419                }
420            } else {
421                let results: Vec<_> = join_all(restart_futures).await;
422                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
423                    return Err(Error::kube_error(
424                        "restart",
425                        "StatefulSet",
426                        kanidm.get_namespace(),
427                        "replica statefulsets",
428                        first_err,
429                    ));
430                }
431            }
432        }
433    }
434
435    Ok(changed)
436}
437
438#[instrument(skip(ctx, kanidm))]
439pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
440    let trace_id = telemetry::get_trace_id();
441    Span::current().record("trace_id", field::display(&trace_id));
442    let _timer = ctx
443        .kaniop_ctx
444        .metrics
445        .reconcile_count_and_measure(&trace_id);
446    info!("reconciling Kanidm");
447
448    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
449        debug!(%e, "failed to reconcile status");
450        ctx.kaniop_ctx.metrics.status_update_errors_inc();
451        e
452    })?;
453    let kanidm_api: Api<Kanidm> =
454        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
455    let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
456    let outcome_clone = outcome.clone();
457    let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
458        let outcome = outcome_clone.clone();
459        let ctx = ctx.clone();
460        let status = status.clone();
461        async move {
462            match event {
463                Finalizer::Apply(kanidm) => {
464                    let (action, changed) = reconcile(kanidm, ctx, status).await?;
465                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
466                    Ok(action)
467                }
468                Finalizer::Cleanup(kanidm) => {
469                    let (action, changed) = cleanup(kanidm, ctx).await?;
470                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
471                    Ok(action)
472                }
473            }
474        }
475    })
476    .await
477    .or_else(|e| match e {
478        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
479            debug!("resource already removed during finalizer cleanup");
480            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
481        }
482        _ => Err(Error::FinalizerError(
483            "failed on kanidm account finalizer".to_string(),
484            Box::new(e),
485        )),
486    })?;
487    let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
488    Ok((action, changed))
489}
490
491/// Hash of the TLS secret content, injected as a pod template annotation so
492/// certificate renewals roll the statefulsets. `None` if the secret does not
493/// exist yet.
494async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
495    let secret_name = kanidm.effective_tls_secret_name();
496    let secret_api =
497        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
498    let secret = secret_api
499        .get_opt(&secret_name)
500        .await
501        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
502    Ok(secret.as_ref().map(hash_secret_data))
503}
504
505/// Previous TLS secret hash annotation from the existing statefulsets.
506///
507/// Used when the TLS secret is missing: dropping the annotation would roll
508/// the pods onto a missing volume and leave them stuck in ContainerCreating.
509fn previous_tls_secret_hash(
510    kanidm: &Kanidm,
511    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
512) -> Option<String> {
513    stateful_sets
514        .into_iter()
515        .filter(|sts| {
516            sts.namespace() == kanidm.namespace()
517                && sts
518                    .metadata
519                    .labels
520                    .as_ref()
521                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
522        })
523        .find_map(|sts| {
524            sts.spec
525                .as_ref()?
526                .template
527                .metadata
528                .as_ref()?
529                .annotations
530                .as_ref()?
531                .get(TLS_SECRET_HASH_ANNOTATION)
532                .cloned()
533        })
534}
535
536fn is_headless_service(service: &Service) -> bool {
537    service
538        .spec
539        .as_ref()
540        .and_then(|spec| spec.cluster_ip.as_deref())
541        == Some("None")
542}
543
544fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
545    if is_headless_service(desired) != is_headless_service(current) {
546        return;
547    }
548    let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
549    else {
550        return;
551    };
552    if desired_spec.cluster_ip.is_none() {
553        desired_spec.cluster_ip = current_spec.cluster_ip.clone();
554    }
555    if desired_spec.cluster_ips.is_none() {
556        desired_spec.cluster_ips = current_spec.cluster_ips.clone();
557    }
558}
559
560async fn wait_for_statefulset_deletion(
561    client: kube::Client,
562    namespace: &str,
563    name: &str,
564) -> Result<()> {
565    let api = Api::<StatefulSet>::namespaced(client, namespace);
566    let start = std::time::Instant::now();
567    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
568    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
569
570    loop {
571        match api.get_opt(name).await {
572            Ok(None) => return Ok(()),
573            Ok(Some(_)) => {}
574            Err(e) => {
575                return Err(Error::kube_error(
576                    "wait for deletion of",
577                    "StatefulSet",
578                    namespace,
579                    name,
580                    e,
581                ));
582            }
583        }
584
585        if start.elapsed() >= timeout {
586            return Err(Error::ReceiveOutput(format!(
587                "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
588            )));
589        }
590        sleep(poll_interval).await;
591    }
592}
593
594async fn reconcile_service(
595    kanidm: &Kanidm,
596    ctx: &Context,
597    mut desired: Service,
598) -> Result<Service> {
599    let namespace = kanidm.get_namespace();
600    let name = desired.name_any();
601    let desired_headless = is_headless_service(&desired);
602
603    if let Some(cached) = ctx
604        .stores
605        .service_store
606        .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
607    {
608        if is_headless_service(cached.as_ref()) == desired_headless {
609            preserve_defaulted_service_fields(&mut desired, cached.as_ref());
610        } else {
611            let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
612            let current = service_api
613                .get_opt(&name)
614                .await
615                .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
616
617            if let Some(current) = current {
618                let current_headless = is_headless_service(&current);
619                if current_headless != desired_headless {
620                    info!(
621                        resource.name = &name,
622                        resource.namespace = &namespace,
623                        current_headless,
624                        desired_headless,
625                        "recreating Service to change headless mode"
626                    );
627                    kanidm.delete(ctx, &current).await?;
628                    ctx.kaniop_ctx
629                        .metrics
630                        .reconcile_deploy_delete_create_inc("Service", "headless_transition");
631                    wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
632                        .await?;
633                } else {
634                    preserve_defaulted_service_fields(&mut desired, &current);
635                }
636            }
637        }
638    }
639
640    kanidm.patch(ctx, desired).await
641}
642
643async fn wait_for_service_deletion(
644    client: kube::Client,
645    namespace: &str,
646    name: &str,
647) -> Result<()> {
648    let api = Api::<Service>::namespaced(client, namespace);
649    let start = std::time::Instant::now();
650    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
651    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
652
653    loop {
654        match api.get_opt(name).await {
655            Ok(None) => return Ok(()),
656            Ok(Some(_)) => {}
657            Err(e) => {
658                return Err(Error::kube_error(
659                    "wait for deletion of",
660                    "Service",
661                    namespace,
662                    name,
663                    e,
664                ));
665            }
666        }
667
668        if start.elapsed() >= timeout {
669            return Err(Error::ReceiveOutput(format!(
670                "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
671            )));
672        }
673        sleep(poll_interval).await;
674    }
675}
676
677async fn validate_statefulset_replacement(
678    api: &Api<StatefulSet>,
679    desired: &StatefulSet,
680    namespace: &str,
681    name: &str,
682) -> Result<()> {
683    let mut candidate = desired.clone();
684    candidate.metadata.name = None;
685    candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
686    candidate.metadata.namespace = None;
687    candidate.metadata.resource_version = None;
688    candidate.metadata.uid = None;
689    candidate.metadata.creation_timestamp = None;
690    candidate.metadata.generation = None;
691    candidate.metadata.managed_fields = None;
692    candidate.metadata.deletion_timestamp = None;
693    candidate.metadata.deletion_grace_period_seconds = None;
694    candidate.status = None;
695
696    api.create(
697        &PostParams {
698            dry_run: true,
699            ..Default::default()
700        },
701        &candidate,
702    )
703    .await
704    .map(|_| ())
705    .map_err(|e| {
706        Error::kube_error(
707            "validate replacement for",
708            "StatefulSet",
709            namespace,
710            name,
711            e,
712        )
713    })
714}
715
716async fn reconcile_statefulset(
717    kanidm: Arc<Kanidm>,
718    ctx: Arc<Context>,
719    desired: StatefulSet,
720) -> Result<StatefulSet> {
721    let namespace = kanidm.namespace().unwrap();
722    let name = desired.name_any();
723    let cached_current = ctx.stores.stateful_set_store.find(|current| {
724        current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
725    });
726
727    // On a cold reflector cache, keep the normal create/update path GET-free. If the optimistic
728    // apply fails with HTTP 422, the object may already exist with an immutable conflict that the
729    // cache has not observed yet. Only 422 errors enter the live classification/recreate recovery
730    // path; 429, 5xx, 403, and transport errors are returned immediately and non-destructively.
731    let cache_miss_422_error = if let Some(cached_current) = cached_current {
732        let mut cached_desired = desired.clone();
733        preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
734        if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
735            == StatefulSetApplyStrategy::Apply
736        {
737            return kanidm.apply(&ctx, cached_desired).await;
738        }
739        None
740    } else {
741        match kanidm.apply(&ctx, desired.clone()).await {
742            Ok(applied) => return Ok(applied),
743            Err(error) => {
744                // Only HTTP 422 can indicate an immutable-field conflict that is safe to
745                // recover from via delete/recreate. Arbitrary failures must not trigger
746                // destruction.
747                if is_kube_422(&error) {
748                    Some(error)
749                } else {
750                    return Err(error);
751                }
752            }
753        }
754    };
755
756    // Never make a destructive decision from reflector state or an SSA error alone. Re-read the
757    // live object and classify it immediately before deletion; another actor or an earlier
758    // reconcile may already have converged the immutable fields while the cache was catching up.
759    let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
760    let Some(live_current) = api
761        .get_opt(&name)
762        .await
763        .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
764    else {
765        return match cache_miss_422_error {
766            Some(error) => Err(error),
767            None => kanidm.apply(&ctx, desired).await,
768        };
769    };
770
771    // Start from the original desired object again. Defaults copied from a stale reflector entry
772    // must not survive into the live re-check and become the reason for a recreation.
773    let mut live_desired = desired.clone();
774    preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
775    match classify_statefulset_change(&live_current, &live_desired) {
776        StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
777            Some(error) => Err(error),
778            None => kanidm.apply(&ctx, live_desired).await,
779        },
780        StatefulSetApplyStrategy::Recreate { .. } => {
781            // An update 422 can combine an immutable conflict with independently-invalid desired
782            // state. Validate the replacement as a dry-run CREATE before deleting the live object,
783            // so create-time validation runs without the existing object's immutable-update check.
784            validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
785
786            // Re-read and reclassify immediately before destruction. Another actor or an earlier
787            // reconcile may have changed the live immutable fields or PVC-retention semantics.
788            let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
789                Error::kube_error(
790                    "re-read before deletion",
791                    "StatefulSet",
792                    &namespace,
793                    &name,
794                    e,
795                )
796            })?
797            else {
798                return kanidm.apply(&ctx, live_desired).await;
799            };
800            let mut latest_desired = live_desired.clone();
801            preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
802
803            match classify_statefulset_change(&latest_current, &latest_desired) {
804                StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
805                StatefulSetApplyStrategy::Recreate { immutable_fields } => {
806                    // Validate the replacement StatefulSet via a dry-run CREATE before deleting
807                    // the live object. This prevents deleting on a mixed 422 (immutable conflict
808                    // + independently-invalid desired state) where the replacement would still
809                    // be rejected by the API.
810                    validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
811                        .await?;
812
813                    info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
814                    kanidm.delete(&ctx, &latest_current).await?;
815                    ctx.kaniop_ctx
816                        .metrics
817                        .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
818                    wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
819                        .await?;
820                    kanidm.apply(&ctx, latest_desired).await
821                }
822            }
823        }
824    }
825}
826
827/// Return true only when the error structurally wraps a Kubernetes HTTP 422 response.
828fn is_kube_422(error: &Error) -> bool {
829    matches!(
830        error,
831        Error::KubeError(_, cause)
832            if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
833    )
834}
835
836#[cfg(test)]
837mod statefulset_recreate_recovery_tests {
838    use super::{Error, is_kube_422};
839
840    fn kube_api_error(code: u16) -> Error {
841        Error::KubeError(
842            "test error".to_string(),
843            Box::new(kube::Error::Api(Box::new(kube::error::Status {
844                code,
845                message: "test".to_string(),
846                reason: "test".to_string(),
847                ..Default::default()
848            }))),
849        )
850    }
851
852    #[test]
853    fn recreate_recovery_accepts_only_422() {
854        assert!(is_kube_422(&kube_api_error(422)));
855        assert!(!is_kube_422(&kube_api_error(403)));
856        assert!(!is_kube_422(&kube_api_error(429)));
857        assert!(!is_kube_422(&kube_api_error(500)));
858    }
859
860    #[test]
861    fn recreate_recovery_rejects_429_too_many_requests() {
862        assert!(!is_kube_422(&kube_api_error(429)));
863    }
864
865    #[test]
866    fn recreate_recovery_rejects_5xx_server_errors() {
867        assert!(!is_kube_422(&kube_api_error(500)));
868        assert!(!is_kube_422(&kube_api_error(502)));
869        assert!(!is_kube_422(&kube_api_error(503)));
870        assert!(!is_kube_422(&kube_api_error(504)));
871    }
872
873    #[test]
874    fn recreate_recovery_rejects_400_and_403() {
875        assert!(!is_kube_422(&kube_api_error(400)));
876        assert!(!is_kube_422(&kube_api_error(403)));
877    }
878}
879
880async fn reconcile(
881    kanidm: Arc<Kanidm>,
882    ctx: Arc<Context>,
883    status: KanidmStatus,
884) -> Result<(Action, bool)> {
885    if kanidm
886        .annotations()
887        .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
888    {
889        ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
890        return Ok((Action::requeue(Duration::from_secs(5)), false));
891    }
892
893    let mut changed = false;
894    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
895    let replication_secret_futures =
896        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
897
898    let sts_to_delete = {
899        let expected_sts_names = kanidm
900            .spec
901            .replica_groups
902            .iter()
903            .map(|rg| kanidm.statefulset_name(&rg.name))
904            .collect::<Vec<_>>();
905        ctx.stores
906            .stateful_set_store
907            .state()
908            .into_iter()
909            .filter(|sts| {
910                sts.namespace() == kanidm.namespace()
911                    && sts
912                        .metadata
913                        .labels
914                        .as_ref()
915                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
916                    && !expected_sts_names.contains(&sts.name_any())
917            })
918            .collect::<Vec<_>>()
919    };
920    if !sts_to_delete.is_empty() {
921        changed = true;
922    }
923    let sts_delete_futures = sts_to_delete
924        .iter()
925        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
926        .collect::<TryJoinAll<_>>();
927
928    let sts_futures = match kanidm.is_updatable(&status) {
929        true => {
930            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
931                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
932            });
933
934            let namespace = kanidm.get_namespace();
935            let schedule_api: Api<KanidmBackupSchedule> =
936                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
937            let schedules = schedule_api
938                .list(&Default::default())
939                .await
940                .map_err(|e| {
941                    Error::KubeError(
942                        "failed to list KanidmBackupSchedules for backup configuration resolution"
943                            .to_string(),
944                        Box::new(e),
945                    )
946                })?
947                .items;
948
949            let repo_api: Api<KanidmBackupRepository> =
950                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
951            let repositories = repo_api
952                .list(&Default::default())
953                .await
954                .map_err(|e| {
955                    Error::KubeError(
956                        "failed to list KanidmBackupRepositories for backup configuration resolution"
957                            .to_string(),
958                        Box::new(e),
959                    )
960                })?
961                .items;
962            let backup_config = resolve_backup_config(&kanidm, &schedules, &repositories);
963
964            kanidm
965                .spec
966                .replica_groups
967                .iter()
968                .map(|rg| {
969                    let sts = kanidm.create_statefulset(
970                        rg,
971                        tls_secret_hash.as_deref(),
972                        backup_config.as_ref(),
973                    )?;
974                    Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
975                })
976                .collect::<Result<TryJoinAll<_>, _>>()?
977        }
978        false => {
979            let note = match status.version.as_ref() {
980                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
981                    format!(
982                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
983                        v.image_tag,
984                        crate::version::KANIDM_CLIENT_VERSION
985                    )
986                }
987                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
988            };
989            let _ignore_error = ctx
990                .kaniop_ctx
991                .recorder
992                .publish(
993                    &Event {
994                        type_: EventType::Warning,
995                        reason: "UpgradeBlocked".to_string(),
996                        note: Some(note),
997                        action: "ReconcileStatefulSet".to_string(),
998                        secondary: None,
999                    },
1000                    &kanidm.object_ref(&()),
1001                )
1002                .await
1003                .map_err(|e| {
1004                    warn!(%e, "failed to publish KanidmError event");
1005                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
1006                });
1007            try_join_all([])
1008        }
1009    };
1010    let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1011
1012    let deprecated_rg_svcs = {
1013        let expected_rg_svcs_names = kanidm
1014            .spec
1015            .replica_groups
1016            .iter()
1017            .filter(|rg| rg.services.is_some())
1018            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1019            .collect::<Vec<_>>();
1020        ctx.stores
1021            .service_store
1022            .state()
1023            .into_iter()
1024            .filter(|svc| {
1025                let labels = svc.metadata.labels.as_ref();
1026                svc.namespace() == kanidm.namespace()
1027                    && !expected_rg_svcs_names.contains(&svc.name_any())
1028                    && labels.is_some_and(|l| {
1029                        l.contains_key(REPLICA_LABEL)
1030                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1031                    })
1032            })
1033            .collect::<Vec<_>>()
1034    };
1035    if !deprecated_rg_svcs.is_empty() {
1036        changed = true;
1037    }
1038    let rg_svcs_delete_futures = deprecated_rg_svcs
1039        .iter()
1040        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1041        .collect::<TryJoinAll<_>>();
1042
1043    let rg_services_futures = kanidm
1044        .spec
1045        .replica_groups
1046        .iter()
1047        .filter(|rg| rg.services.is_some())
1048        .flat_map(|rg| {
1049            (0..rg.replicas).map(|i| {
1050                let mut svc = kanidm.create_replica_group_service(rg, i);
1051                if let Some(current) = ctx.stores.service_store.find(|current| {
1052                    current.namespace() == kanidm.namespace()
1053                        && current.name_any() == svc.name_any()
1054                }) {
1055                    preserve_defaulted_service_fields(&mut svc, current.as_ref());
1056                }
1057                kanidm.patch(&ctx, svc)
1058            })
1059        })
1060        .collect::<TryJoinAll<_>>();
1061
1062    let deprecated_ingresses = {
1063        let names = [
1064            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1065            kanidm.generate_region_ingress_name(),
1066        ]
1067        .into_iter()
1068        .flatten()
1069        .collect::<Vec<_>>();
1070        ctx.stores
1071            .ingress_store
1072            .state()
1073            .into_iter()
1074            .filter(|ing| {
1075                ing.namespace() == kanidm.namespace()
1076                    && !names.contains(&ing.name_any())
1077                    && ing.metadata.labels == Some(kanidm.generate_labels())
1078            })
1079            .collect::<Vec<_>>()
1080    };
1081    if !deprecated_ingresses.is_empty() {
1082        changed = true;
1083    }
1084    let ingress_delete_futures = deprecated_ingresses
1085        .iter()
1086        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1087        .collect::<TryJoinAll<_>>();
1088
1089    let ingress_futures = kanidm
1090        .create_ingress()
1091        .into_iter()
1092        .chain(kanidm.create_region_ingress())
1093        .map(|ingress| kanidm.patch(&ctx, ingress))
1094        .collect::<TryJoinAll<_>>();
1095
1096    let deprecated_http_routes = match &ctx.stores.http_route_store {
1097        Some(store) => {
1098            let expected_names = kanidm
1099                .spec
1100                .gateway
1101                .as_ref()
1102                .map(|_| vec![kanidm.name_any()])
1103                .unwrap_or_default();
1104            let routes: Vec<_> = store
1105                .state()
1106                .into_iter()
1107                .filter(|route| {
1108                    route.namespace() == kanidm.namespace()
1109                        && !expected_names.contains(&route.name_any())
1110                        && route.metadata.labels == Some(kanidm.generate_labels())
1111                })
1112                .collect();
1113            if !routes.is_empty() {
1114                changed = true;
1115            }
1116            routes
1117        }
1118        None => Vec::new(),
1119    };
1120    let http_route_delete_futures = deprecated_http_routes
1121        .iter()
1122        .map(|route| kanidm.delete(&ctx, route.as_ref()))
1123        .collect::<TryJoinAll<_>>();
1124
1125    let http_route_futures = if ctx.stores.http_route_store.is_some() {
1126        kanidm
1127            .create_http_route()
1128            .into_iter()
1129            .map(|route| kanidm.patch(&ctx, route))
1130            .collect::<TryJoinAll<_>>()
1131    } else {
1132        try_join_all(Vec::new())
1133    };
1134
1135    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1136        Some(store) => {
1137            let expected_names = kanidm
1138                .spec
1139                .gateway
1140                .as_ref()
1141                .and_then(|g| g.backend_tls_policy.as_ref())
1142                .map(|_| vec![kanidm.name_any()])
1143                .unwrap_or_default();
1144            let policies: Vec<_> = store
1145                .state()
1146                .into_iter()
1147                .filter(|policy| {
1148                    policy.namespace() == kanidm.namespace()
1149                        && !expected_names.contains(&policy.name_any())
1150                        && policy.metadata.labels == Some(kanidm.generate_labels())
1151                })
1152                .collect();
1153            if !policies.is_empty() {
1154                changed = true;
1155            }
1156            policies
1157        }
1158        None => Vec::new(),
1159    };
1160    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1161        .iter()
1162        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1163        .collect::<TryJoinAll<_>>();
1164
1165    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1166        kanidm
1167            .create_backend_tls_policy()
1168            .into_iter()
1169            .map(|policy| kanidm.patch(&ctx, policy))
1170            .collect::<TryJoinAll<_>>()
1171    } else {
1172        try_join_all(Vec::new())
1173    };
1174
1175    let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1176        sts_delete_futures,
1177        admin_secret_future,
1178        replication_secret_futures,
1179        sts_futures,
1180        service_future,
1181        rg_svcs_delete_futures,
1182        rg_services_futures,
1183        ingress_delete_futures,
1184        ingress_futures,
1185        http_route_delete_futures,
1186        http_route_futures,
1187        backend_tls_policy_delete_futures,
1188        backend_tls_policy_futures
1189    )?;
1190
1191    if admin_secret_changed || replication_secrets_changed {
1192        changed = true;
1193    }
1194
1195    if is_kanidm_available(status.clone()) {
1196        let namespace = kanidm.namespace().unwrap();
1197        let name = kanidm.name_any();
1198        match crate::controller::kanidm::KanidmClients::create_client(
1199            &namespace,
1200            &name,
1201            crate::controller::kanidm::KanidmUser::Admin,
1202            ctx.kaniop_ctx.client.clone(),
1203        )
1204        .await
1205        {
1206            Ok(system_client) => {
1207                if let Err(e) =
1208                    reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1209                {
1210                    warn!(%e, "failed to reconcile domain appearance");
1211                }
1212            }
1213            Err(e) => {
1214                warn!(%e, "failed to create admin client for domain appearance reconciliation");
1215            }
1216        }
1217
1218        match crate::controller::kanidm::KanidmClients::create_client(
1219            &namespace,
1220            &name,
1221            crate::controller::kanidm::KanidmUser::IdmAdmin,
1222            ctx.kaniop_ctx.client.clone(),
1223        )
1224        .await
1225        {
1226            Ok(kanidm_client) => {
1227                let (mail_sender_status, mail_sender_mutated) =
1228                    reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1229                        .await
1230                        .unwrap_or_else(|e| {
1231                            warn!(%e, "failed to reconcile mail sender");
1232                            (None, false)
1233                        });
1234                if mail_sender_mutated {
1235                    changed = true;
1236                }
1237                let current_mail_sender_status =
1238                    kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1239                if mail_sender_status != current_mail_sender_status {
1240                    let kanidm_api: Api<Kanidm> =
1241                        Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1242                    let status_patch = serde_json::json!({
1243                        "status": {
1244                            "mailSender": mail_sender_status
1245                        }
1246                    });
1247                    if let Err(e) = kanidm_api
1248                        .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1249                        .await
1250                    {
1251                        warn!(%e, "failed to patch Kanidm/status for mail sender");
1252                    }
1253                }
1254            }
1255            Err(e) => {
1256                warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1257            }
1258        }
1259    }
1260
1261    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1262}
1263
1264async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1265    debug!("cleanup");
1266
1267    let mut changed = false;
1268    let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1269    if mail_resources_cleaned {
1270        changed = true;
1271    }
1272
1273    let namespace = kanidm.namespace().unwrap();
1274    let name = kanidm.name_any();
1275
1276    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1277        &namespace,
1278        &name,
1279        crate::controller::kanidm::KanidmUser::IdmAdmin,
1280        ctx.kaniop_ctx.client.clone(),
1281    )
1282    .await
1283    {
1284        let kanidm_cleanup_done =
1285            cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1286        if kanidm_cleanup_done {
1287            changed = true;
1288        }
1289    }
1290
1291    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1292    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1293}
1294
1295async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1296    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1297    let start = std::time::Instant::now();
1298    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1299    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1300
1301    loop {
1302        let sts = sts_api.get(sts_name).await.map_err(|e| {
1303            Error::KubeError(
1304                format!("failed to get StatefulSet {namespace}/{sts_name}"),
1305                Box::new(e),
1306            )
1307        })?;
1308
1309        if sts.metadata.deletion_timestamp.is_some() {
1310            return Err(Error::ReceiveOutput(format!(
1311                "StatefulSet {namespace}/{sts_name} is being deleted"
1312            )));
1313        }
1314
1315        let spec = sts.spec.as_ref().ok_or_else(|| {
1316            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1317        })?;
1318
1319        let status = match sts.status {
1320            Some(ref s) => s,
1321            None => {
1322                if start.elapsed() >= timeout {
1323                    return Err(Error::ReceiveOutput(format!(
1324                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1325                    )));
1326                }
1327                sleep(poll_interval).await;
1328                continue;
1329            }
1330        };
1331
1332        let replicas = spec.replicas.unwrap_or(0);
1333
1334        if replicas == 0 {
1335            return Ok(());
1336        }
1337
1338        let updated_replicas = status.updated_replicas.unwrap_or(0);
1339        let current_revision = status.current_revision.as_deref().unwrap_or("");
1340        let update_revision = status.update_revision.as_deref().unwrap_or("");
1341        let ready_replicas = status.ready_replicas.unwrap_or(0);
1342
1343        if updated_replicas == replicas
1344            && current_revision == update_revision
1345            && ready_replicas == replicas
1346        {
1347            return Ok(());
1348        }
1349
1350        if start.elapsed() >= timeout {
1351            return Err(Error::ReceiveOutput(format!(
1352                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1353            )));
1354        }
1355
1356        sleep(poll_interval).await;
1357    }
1358}
1359
1360async fn show_replica_cert_with_retries(
1361    kanidm: &Kanidm,
1362    ctx: Arc<Context>,
1363    pod_name: &str,
1364    max_attempts: u32,
1365) -> std::result::Result<String, Error> {
1366    let mut last_error = None;
1367    for attempt in 0..max_attempts {
1368        if attempt > 0 {
1369            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1370        }
1371        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1372            Ok(cert) => return Ok(cert),
1373            Err(e) => {
1374                info!(
1375                    "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1376                    attempt + 1,
1377                    max_attempts
1378                );
1379                last_error = Some(e);
1380            }
1381        }
1382    }
1383    Err(last_error.unwrap_or_else(|| {
1384        Error::ReceiveOutput(format!(
1385            "failed to get certificate for {pod_name} after {max_attempts} attempts"
1386        ))
1387    }))
1388}
1389
1390impl Kanidm {
1391    // Convenience methods that handle context and operator name
1392    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1393    where
1394        K: Resource<Scope = NamespaceResourceScope>
1395            + Serialize
1396            + Clone
1397            + std::fmt::Debug
1398            + for<'de> Deserialize<'de>,
1399        <K as kube::Resource>::DynamicType: Default,
1400        <K as Resource>::Scope: std::marker::Sized,
1401    {
1402        self.kube_delete(
1403            ctx.kaniop_ctx.client.clone(),
1404            &ctx.kaniop_ctx.metrics,
1405            resource,
1406        )
1407        .await
1408    }
1409
1410    /// Server-side apply without any generic delete/recreate fallback.
1411    /// Resource-specific reconcilers use this when they own lifecycle decisions.
1412    pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1413    where
1414        K: Resource<Scope = NamespaceResourceScope>
1415            + Serialize
1416            + Clone
1417            + std::fmt::Debug
1418            + for<'de> Deserialize<'de>,
1419        <K as kube::Resource>::DynamicType: Default,
1420        <K as Resource>::Scope: std::marker::Sized,
1421    {
1422        self.kube_apply(
1423            ctx.kaniop_ctx.client.clone(),
1424            resource,
1425            KANIDM_OPERATOR_NAME,
1426        )
1427        .await
1428    }
1429
1430    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1431    where
1432        K: Resource<Scope = NamespaceResourceScope>
1433            + Serialize
1434            + Clone
1435            + std::fmt::Debug
1436            + for<'de> Deserialize<'de>,
1437        <K as kube::Resource>::DynamicType: Default,
1438        <K as Resource>::Scope: std::marker::Sized,
1439    {
1440        self.kube_patch(
1441            ctx.kaniop_ctx.client.clone(),
1442            &ctx.kaniop_ctx.metrics,
1443            resource,
1444            KANIDM_OPERATOR_NAME,
1445        )
1446        .await
1447    }
1448
1449    #[inline]
1450    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1451        LABELS
1452            .clone()
1453            .into_iter()
1454            .chain([
1455                (INSTANCE_LABEL.to_string(), self.name_any()),
1456                (CLUSTER_LABEL.to_string(), self.name_any()),
1457            ])
1458            .collect()
1459    }
1460
1461    #[inline]
1462    fn generate_labels(&self) -> BTreeMap<String, String> {
1463        self.generate_resource_labels()
1464    }
1465
1466    #[inline]
1467    fn get_tls_secret_name(&self) -> String {
1468        format!("{}-tls", self.name_any())
1469    }
1470
1471    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
1472    /// falling back to the ingress TLS secret and then to the default name.
1473    pub(crate) fn effective_tls_secret_name(&self) -> String {
1474        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1475            self.spec
1476                .ingress
1477                .as_ref()
1478                .and_then(|i| i.tls_secret_name.clone())
1479                .unwrap_or_else(|| self.get_tls_secret_name())
1480        })
1481    }
1482
1483    #[inline]
1484    fn get_namespace(&self) -> String {
1485        // safe unwrap: Kanidm is namespaced scoped
1486        self.namespace().unwrap()
1487    }
1488
1489    #[inline]
1490    fn is_replication_enabled(&self) -> bool {
1491        self.spec.replica_groups.len() > 1
1492            || self
1493                .spec
1494                .replica_groups
1495                .first()
1496                .is_some_and(|rg| rg.replicas > 1)
1497            || !self.spec.external_replication_nodes.is_empty()
1498    }
1499
1500    #[inline]
1501    fn is_updatable(&self, status: &KanidmStatus) -> bool {
1502        status
1503            .version
1504            .as_ref()
1505            .map(|v| {
1506                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1507                    return false;
1508                }
1509                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1510                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1511                match v.upgrade_check_result {
1512                    KanidmUpgradeCheckResult::Passed => {
1513                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1514                            minor <= v_minor + 1
1515                        } else {
1516                            true
1517                        }
1518                    }
1519                    KanidmUpgradeCheckResult::Failed => {
1520                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1521                            versions
1522                        {
1523                            major == v_major && minor == v_minor && patch >= v_patch
1524                        } else {
1525                            v.image_tag == current_tag
1526                        }
1527                    }
1528                }
1529            })
1530            .unwrap_or(true)
1531    }
1532
1533    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1534        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1535        let pod = pod_api.get(pod_name).await.map_err(|e| {
1536            Error::KubeError(
1537                format!("failed to get pod {namespace}/{pod_name}"),
1538                Box::new(e),
1539            )
1540        })?;
1541        Ok(pod
1542            .status
1543            .as_ref()
1544            .and_then(|s| s.conditions.as_ref())
1545            .is_some_and(|conditions| {
1546                conditions
1547                    .iter()
1548                    .any(|c| c.type_ == "Ready" && c.status == "True")
1549            }))
1550    }
1551
1552    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1553        let start = std::time::Instant::now();
1554        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1555        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1556
1557        loop {
1558            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1559                Ok(true) => return Ok(()),
1560                Ok(false) => {}
1561                Err(e) => {
1562                    if !e.is_retryable() {
1563                        return Err(e);
1564                    }
1565                    trace!(%e, "pod readiness check failed, retrying");
1566                }
1567            }
1568
1569            if start.elapsed() >= timeout {
1570                return Err(Error::ReceiveOutput(format!(
1571                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1572                )));
1573            }
1574
1575            sleep(poll_interval).await;
1576        }
1577    }
1578
1579    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1580    where
1581        I: IntoIterator<Item = T> + Debug,
1582        T: Into<String>,
1583    {
1584        let namespace = &self.get_namespace();
1585        trace!(
1586            resource.name = &pod_name,
1587            resource.namespace = &namespace,
1588            ?command,
1589            "pod exec"
1590        );
1591        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1592        let attached = pod
1593            .exec(
1594                pod_name,
1595                command,
1596                &AttachParams::default().container("kanidm"),
1597            )
1598            .await
1599            .map_err(|e| {
1600                Error::KubeError(
1601                    format!("failed to exec pod {namespace}/{pod_name}"),
1602                    Box::new(e),
1603                )
1604            })?;
1605        get_output(attached).await
1606    }
1607
1608    async fn exec_with_wait<I, T>(
1609        &self,
1610        ctx: Arc<Context>,
1611        pod_name: &str,
1612        command: I,
1613    ) -> Result<String>
1614    where
1615        I: IntoIterator<Item = T> + Debug,
1616        T: Into<String>,
1617    {
1618        let namespace = &self.get_namespace();
1619        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1620        self.exec(ctx, pod_name, command).await
1621    }
1622
1623    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1624    where
1625        I: IntoIterator<Item = T> + Debug,
1626        T: Into<String>,
1627    {
1628        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1629        let first_replica_group = self
1630            .spec
1631            .replica_groups
1632            .first()
1633            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1634        let sts_name = self.statefulset_name(&first_replica_group.name);
1635        let pod_name = format!("{sts_name}-0");
1636        self.exec(ctx, &pod_name, command).await
1637    }
1638
1639    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1640    where
1641        I: IntoIterator<Item = T> + Debug,
1642        T: Into<String>,
1643    {
1644        let first_replica_group = self
1645            .spec
1646            .replica_groups
1647            .first()
1648            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1649        let sts_name = self.statefulset_name(&first_replica_group.name);
1650        let pod_name = format!("{sts_name}-0");
1651        self.exec_with_wait(ctx, &pod_name, command).await
1652    }
1653}
1654
1655#[cfg(test)]
1656mod test {
1657    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1658    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1659
1660    use crate::controller::State;
1661    use crate::kanidm::controller::context::{Context, Stores};
1662    use crate::kanidm::crd::KanidmStatus;
1663
1664    use kaniop_k8s_util::error::Result;
1665    use kaniop_k8s_util::resources::hash_secret_data;
1666
1667    use std::collections::BTreeMap;
1668    use std::sync::Arc;
1669
1670    use http::{Request, Response};
1671    use k8s_openapi::ByteString;
1672    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1673    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1674    use k8s_openapi::api::networking::v1::Ingress;
1675    use kube::api::ObjectMeta;
1676    use kube::runtime::reflector::store::Writer;
1677    use kube::{Client, Resource, ResourceExt, client::Body};
1678    use opentelemetry::metrics::MeterProvider;
1679    use serde_json::json;
1680
1681    impl Kanidm {
1682        /// A normal test kanidm with a given status
1683        pub fn test() -> Self {
1684            let mut e = Kanidm::new(
1685                "test",
1686                serde_json::from_value(json!({
1687                    "domain": "idm.example.com",
1688                    "replicaGroups": [
1689                        {
1690                            "name": "default",
1691                            "replicas": 1
1692                        }
1693                    ]
1694                }))
1695                .unwrap(),
1696            );
1697            e.meta_mut().namespace = Some("default".into());
1698            // Pre-populate finalizer so reconcile callback runs immediately
1699            // (skips the "add finalizer + wait for next reconcile" step)
1700            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1701            e
1702        }
1703
1704        pub fn with_ingress(mut self) -> Self {
1705            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1706            self
1707        }
1708
1709        /// Modify kanidm replicas
1710        pub fn with_replicas(mut self, replicas: i32) -> Self {
1711            self.spec.replica_groups[0].replicas = replicas;
1712            self
1713        }
1714
1715        /// Modify kanidm to set a deletion timestamp
1716        pub fn needs_delete(mut self) -> Self {
1717            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1718            use k8s_openapi::jiff::Timestamp;
1719            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1720            self.meta_mut().deletion_timestamp = Some(Time(now));
1721            self
1722        }
1723
1724        /// Modify a kanidm to have an expected status
1725        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1726            self.status = Some(status);
1727            self
1728        }
1729    }
1730
1731    // We wrap tower_test::mock::Handle
1732    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1733    pub struct ApiServerVerifier(ApiServerHandle);
1734
1735    /// Scenarios we test for in ApiServerVerifier
1736    pub enum Scenario {
1737        Create(Kanidm),
1738        CreateWithTwoReplicas(Kanidm),
1739        CreateWithIngress(Kanidm),
1740        CreateWithIngressWithTwoReplicas(Kanidm),
1741        CreateWithTlsSecret(Kanidm, Box<Secret>),
1742    }
1743
1744    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1745        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1746            .await
1747            .expect("timeout on mock apiserver")
1748            .expect("scenario succeeded")
1749    }
1750
1751    impl ApiServerVerifier {
1752        /// Tests only get to run specific scenarios that has matching handlers
1753        ///
1754        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1755        ///
1756        /// NB: If the controller is making more calls than we are handling in the scenario,
1757        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1758        ///
1759        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1760        /// scenario runs to completion (i.e. all expected calls were responded to),
1761        /// using the timeout to catch missing api calls to Kubernetes.
1762        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1763            tokio::spawn(async move {
1764                // moving self => one scenario per test
1765                match scenario {
1766                    Scenario::Create(kanidm) => {
1767                        self.handle_kanidm_get(kanidm.clone())
1768                            .await
1769                            .unwrap()
1770                            .handle_kanidm_status_patch(kanidm.clone())
1771                            .await
1772                            .unwrap()
1773                            .handle_tls_secret_get_not_found(kanidm.clone())
1774                            .await
1775                            .unwrap()
1776                            .handle_backup_schedule_list()
1777                            .await
1778                            .unwrap()
1779                            .handle_backup_repository_list()
1780                            .await
1781                            .unwrap()
1782                            .handle_statefulset_patch(kanidm.clone(), None)
1783                            .await
1784                            .unwrap()
1785                            .handle_service_patch(kanidm.clone())
1786                            .await
1787                    }
1788                    Scenario::CreateWithTwoReplicas(kanidm) => {
1789                        self.handle_kanidm_get(kanidm.clone())
1790                            .await
1791                            .unwrap()
1792                            .handle_kanidm_status_patch(kanidm.clone())
1793                            .await
1794                            .unwrap()
1795                            .handle_tls_secret_get_not_found(kanidm.clone())
1796                            .await
1797                            .unwrap()
1798                            .handle_backup_schedule_list()
1799                            .await
1800                            .unwrap()
1801                            .handle_backup_repository_list()
1802                            .await
1803                            .unwrap()
1804                            .handle_statefulset_patch(kanidm.clone(), None)
1805                            .await
1806                            .unwrap()
1807                            .handle_service_patch(kanidm.clone())
1808                            .await
1809                    }
1810                    Scenario::CreateWithIngress(kanidm) => {
1811                        self.handle_kanidm_get(kanidm.clone())
1812                            .await
1813                            .unwrap()
1814                            .handle_kanidm_status_patch(kanidm.clone())
1815                            .await
1816                            .unwrap()
1817                            .handle_tls_secret_get_not_found(kanidm.clone())
1818                            .await
1819                            .unwrap()
1820                            .handle_backup_schedule_list()
1821                            .await
1822                            .unwrap()
1823                            .handle_backup_repository_list()
1824                            .await
1825                            .unwrap()
1826                            .handle_statefulset_patch(kanidm.clone(), None)
1827                            .await
1828                            .unwrap()
1829                            .handle_service_patch(kanidm.clone())
1830                            .await
1831                            .unwrap()
1832                            .handle_ingress_patch(kanidm.clone())
1833                            .await
1834                    }
1835                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1836                        self.handle_kanidm_get(kanidm.clone())
1837                            .await
1838                            .unwrap()
1839                            .handle_kanidm_status_patch(kanidm.clone())
1840                            .await
1841                            .unwrap()
1842                            .handle_tls_secret_get_not_found(kanidm.clone())
1843                            .await
1844                            .unwrap()
1845                            .handle_backup_schedule_list()
1846                            .await
1847                            .unwrap()
1848                            .handle_backup_repository_list()
1849                            .await
1850                            .unwrap()
1851                            .handle_statefulset_patch(kanidm.clone(), None)
1852                            .await
1853                            .unwrap()
1854                            .handle_service_patch(kanidm.clone())
1855                            .await
1856                            .unwrap()
1857                            .handle_ingress_patch(kanidm.clone())
1858                            .await
1859                    }
1860                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1861                        let expected_hash = hash_secret_data(&secret);
1862                        self.handle_kanidm_get(kanidm.clone())
1863                            .await
1864                            .unwrap()
1865                            .handle_kanidm_status_patch(kanidm.clone())
1866                            .await
1867                            .unwrap()
1868                            .handle_tls_secret_get(kanidm.clone(), *secret)
1869                            .await
1870                            .unwrap()
1871                            .handle_backup_schedule_list()
1872                            .await
1873                            .unwrap()
1874                            .handle_backup_repository_list()
1875                            .await
1876                            .unwrap()
1877                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1878                            .await
1879                            .unwrap()
1880                            .handle_service_patch(kanidm.clone())
1881                            .await
1882                    }
1883                }
1884                .expect("scenario completed without errors");
1885            })
1886        }
1887
1888        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1889            let (request, send) = self.0.next_request().await.expect("service not called");
1890            assert_eq!(request.method(), http::Method::GET);
1891            let expected_uri_prefix = format!(
1892                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1893                kanidm.name_any()
1894            );
1895            let uri = request.uri().to_string();
1896            assert!(
1897                uri.starts_with(&expected_uri_prefix),
1898                "expected uri to start with {}, got {}",
1899                expected_uri_prefix,
1900                uri
1901            );
1902            let response = serde_json::to_vec(&kanidm).unwrap();
1903            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1904            Ok(self)
1905        }
1906
1907        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1908            let (request, send) = self.0.next_request().await.expect("service not called");
1909            assert_eq!(request.method(), http::Method::PATCH);
1910            assert_eq!(
1911                request.uri().to_string(),
1912                format!(
1913                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1914                    kanidm.name_any()
1915                )
1916            );
1917
1918            let req_body = request.into_body().collect_bytes().await.unwrap();
1919            let json: serde_json::Value =
1920                serde_json::from_slice(&req_body).expect("patch object is json");
1921            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1922                .expect("valid kanidm status");
1923            let mut kanidm = Kanidm::test();
1924            kanidm.status = Some(status.clone());
1925            let response = serde_json::to_vec(&kanidm).unwrap();
1926            // pass through kanidm "patch accepted"
1927            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1928            Ok(self)
1929        }
1930
1931        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1932            let (request, send) = self.0.next_request().await.expect("service not called");
1933            assert_eq!(request.method(), http::Method::GET);
1934            let expected_uri_prefix = format!(
1935                "/api/v1/namespaces/default/secrets/{}",
1936                kanidm.effective_tls_secret_name()
1937            );
1938            let uri = request.uri().to_string();
1939            assert!(
1940                uri.starts_with(&expected_uri_prefix),
1941                "expected uri to start with {expected_uri_prefix}, got {uri}"
1942            );
1943            let status = serde_json::json!({
1944                "kind": "Status",
1945                "apiVersion": "v1",
1946                "metadata": {},
1947                "status": "Failure",
1948                "message": "secret not found",
1949                "reason": "NotFound",
1950                "code": 404
1951            });
1952            send.send_response(
1953                Response::builder()
1954                    .status(404)
1955                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1956                    .unwrap(),
1957            );
1958            Ok(self)
1959        }
1960
1961        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1962            let (request, send) = self.0.next_request().await.expect("service not called");
1963            assert_eq!(request.method(), http::Method::GET);
1964            let expected_uri_prefix = format!(
1965                "/api/v1/namespaces/default/secrets/{}",
1966                kanidm.effective_tls_secret_name()
1967            );
1968            let uri = request.uri().to_string();
1969            assert!(
1970                uri.starts_with(&expected_uri_prefix),
1971                "expected uri to start with {expected_uri_prefix}, got {uri}"
1972            );
1973            let response = serde_json::to_vec(&secret).unwrap();
1974            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1975            Ok(self)
1976        }
1977
1978        async fn handle_backup_schedule_list(mut self) -> Result<Self> {
1979            let (request, send) = self.0.next_request().await.expect("service not called");
1980            assert_eq!(request.method(), http::Method::GET);
1981            assert!(
1982                request
1983                    .uri()
1984                    .to_string()
1985                    .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
1986                "expected backup schedule list request, got {}",
1987                request.uri()
1988            );
1989            let list = json!({
1990                "apiVersion": "kaniop.rs/v1alpha1",
1991                "kind": "KanidmBackupScheduleList",
1992                "metadata": {},
1993                "items": []
1994            });
1995            let response = serde_json::to_vec(&list).unwrap();
1996            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1997            Ok(self)
1998        }
1999
2000        async fn handle_backup_repository_list(mut self) -> Result<Self> {
2001            let (request, send) = self.0.next_request().await.expect("service not called");
2002            assert_eq!(request.method(), http::Method::GET);
2003            assert!(
2004                request.uri().to_string().contains(
2005                    "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2006                ),
2007                "expected backup repository list request, got {}",
2008                request.uri()
2009            );
2010            let list = json!({
2011                "apiVersion": "kaniop.rs/v1alpha1",
2012                "kind": "KanidmBackupRepositoryList",
2013                "metadata": {},
2014                "items": []
2015            });
2016            let response = serde_json::to_vec(&list).unwrap();
2017            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2018            Ok(self)
2019        }
2020
2021        async fn handle_statefulset_patch(
2022            mut self,
2023            kanidm: Kanidm,
2024            expected_tls_hash: Option<String>,
2025        ) -> Result<Self> {
2026            for rg in kanidm.spec.replica_groups.iter() {
2027                let (request, send) = self.0.next_request().await.expect("service not called");
2028                assert_eq!(request.method(), http::Method::PATCH);
2029                assert_eq!(
2030                    request.uri().to_string(),
2031                    format!(
2032                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2033                        kanidm.statefulset_name(&rg.name)
2034                    )
2035                );
2036                let req_body = request.into_body().collect_bytes().await.unwrap();
2037                let json: serde_json::Value =
2038                    serde_json::from_slice(&req_body).expect("patch object is json");
2039                let statefulset: StatefulSet =
2040                    serde_json::from_value(json).expect("valid statefulset");
2041                assert_eq!(
2042                    statefulset.clone().spec.unwrap().replicas.unwrap(),
2043                    rg.replicas
2044                );
2045                let tls_hash_annotation = statefulset
2046                    .spec
2047                    .as_ref()
2048                    .unwrap()
2049                    .template
2050                    .metadata
2051                    .as_ref()
2052                    .unwrap()
2053                    .annotations
2054                    .as_ref()
2055                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2056                    .cloned();
2057                assert_eq!(tls_hash_annotation, expected_tls_hash);
2058                let response = serde_json::to_vec(&statefulset).unwrap();
2059                // pass through kanidm "patch accepted"
2060                send.send_response(Response::builder().body(Body::from(response)).unwrap());
2061            }
2062
2063            Ok(self)
2064        }
2065
2066        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2067            let (request, send) = self.0.next_request().await.expect("service not called");
2068            assert_eq!(request.method(), http::Method::PATCH);
2069            assert_eq!(
2070                request.uri().to_string(),
2071                format!(
2072                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2073                    kanidm.name_any()
2074                )
2075            );
2076
2077            let req_body = request.into_body().collect_bytes().await.unwrap();
2078            let json: serde_json::Value =
2079                serde_json::from_slice(&req_body).expect("patch object is json");
2080            let service: Service = serde_json::from_value(json).expect("valid service");
2081            let response = serde_json::to_vec(&service).unwrap();
2082            // pass through kanidm "patch accepted"
2083            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2084            Ok(self)
2085        }
2086
2087        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2088            let (request, send) = self.0.next_request().await.expect("service not called");
2089            assert_eq!(request.method(), http::Method::PATCH);
2090            assert_eq!(
2091                request.uri().to_string(),
2092                format!(
2093                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2094                    kanidm.name_any()
2095                )
2096            );
2097
2098            let req_body = request.into_body().collect_bytes().await.unwrap();
2099            let json: serde_json::Value =
2100                serde_json::from_slice(&req_body).expect("patch object is json");
2101            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2102            let response = serde_json::to_vec(&ingress).unwrap();
2103            // pass through kanidm "patch accepted"
2104            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2105            Ok(self)
2106        }
2107    }
2108
2109    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2110        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2111        let mock_client = Client::new(mock_service, "default");
2112        let stores = Stores {
2113            stateful_set_store: Writer::default().as_reader(),
2114            service_store: Writer::default().as_reader(),
2115            ingress_store: Writer::default().as_reader(),
2116            secret_store: Writer::default().as_reader(),
2117            http_route_store: Some(Writer::default().as_reader()),
2118            backend_tls_policy_store: Some(Writer::default().as_reader()),
2119            deployment_store: Writer::default().as_reader(),
2120            config_map_store: Writer::default().as_reader(),
2121        };
2122        let controller_id = "test";
2123
2124        // Create a test meter for metrics
2125        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2126        let meter = provider.meter("test");
2127        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2128
2129        let state = State::new(
2130            metrics,
2131            Writer::default().as_reader(),
2132            Writer::default().as_reader(),
2133            None,
2134        );
2135        let ctx = Arc::new(Context::new(
2136            state.to_context(mock_client, controller_id),
2137            stores,
2138        ));
2139        (ctx, ApiServerVerifier(handle))
2140    }
2141
2142    #[tokio::test]
2143    async fn kanidm_create() {
2144        let (testctx, fakeserver) = get_test_context();
2145        let kanidm = Kanidm::test();
2146        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2147        reconcile_kanidm(Arc::new(kanidm), testctx)
2148            .await
2149            .expect("reconciler");
2150        timeout_after_1s(mocksrv).await;
2151    }
2152
2153    #[tokio::test]
2154    async fn kanidm_create_with_two_replicas() {
2155        let (testctx, fakeserver) = get_test_context();
2156        let kanidm = Kanidm::test().with_replicas(2);
2157        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2158        reconcile_kanidm(Arc::new(kanidm), testctx)
2159            .await
2160            .expect("reconciler");
2161        timeout_after_1s(mocksrv).await;
2162    }
2163
2164    #[tokio::test]
2165    async fn kanidm_create_with_ingress() {
2166        let (testctx, fakeserver) = get_test_context();
2167        let kanidm = Kanidm::test().with_ingress();
2168        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2169        reconcile_kanidm(Arc::new(kanidm), testctx)
2170            .await
2171            .expect("reconciler");
2172        timeout_after_1s(mocksrv).await;
2173    }
2174
2175    #[tokio::test]
2176    async fn kanidm_create_with_ingress_with_two_replicas() {
2177        let (testctx, fakeserver) = get_test_context();
2178        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2179        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2180        reconcile_kanidm(Arc::new(kanidm), testctx)
2181            .await
2182            .expect("reconciler");
2183        timeout_after_1s(mocksrv).await;
2184    }
2185
2186    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2187        Arc::new(StatefulSet {
2188            metadata: ObjectMeta {
2189                name: Some(format!("{cluster}-default")),
2190                namespace: Some(namespace.to_string()),
2191                labels: Some(BTreeMap::from([(
2192                    CLUSTER_LABEL.to_string(),
2193                    cluster.to_string(),
2194                )])),
2195                ..Default::default()
2196            },
2197            spec: Some(StatefulSetSpec {
2198                template: PodTemplateSpec {
2199                    metadata: Some(ObjectMeta {
2200                        annotations: tls_hash.map(|h| {
2201                            BTreeMap::from([(
2202                                TLS_SECRET_HASH_ANNOTATION.to_string(),
2203                                h.to_string(),
2204                            )])
2205                        }),
2206                        ..Default::default()
2207                    }),
2208                    ..Default::default()
2209                },
2210                ..Default::default()
2211            }),
2212            ..Default::default()
2213        })
2214    }
2215
2216    #[test]
2217    fn test_previous_tls_secret_hash() {
2218        let kanidm = Kanidm::test();
2219
2220        // annotation carried over from an existing statefulset
2221        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2222        assert_eq!(
2223            previous_tls_secret_hash(&kanidm, stss),
2224            Some("abc123".to_string())
2225        );
2226
2227        // no annotation on the existing statefulset
2228        let stss = vec![statefulset_of("test", "default", None)];
2229        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2230
2231        // statefulsets from another cluster or namespace are ignored
2232        let stss = vec![
2233            statefulset_of("other", "default", Some("abc123")),
2234            statefulset_of("test", "other-ns", Some("abc123")),
2235        ];
2236        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2237    }
2238
2239    #[tokio::test]
2240    async fn kanidm_create_with_tls_secret() {
2241        let (testctx, fakeserver) = get_test_context();
2242        let kanidm = Kanidm::test();
2243        let secret = Secret {
2244            metadata: ObjectMeta {
2245                name: Some(kanidm.effective_tls_secret_name()),
2246                namespace: Some("default".to_string()),
2247                ..Default::default()
2248            },
2249            data: Some(BTreeMap::from([
2250                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2251                ("tls.key".to_string(), ByteString(b"key".to_vec())),
2252            ])),
2253            ..Default::default()
2254        };
2255        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2256            kanidm.clone(),
2257            Box::new(secret),
2258        ));
2259        reconcile_kanidm(Arc::new(kanidm), testctx)
2260            .await
2261            .expect("reconciler");
2262        timeout_after_1s(mocksrv).await;
2263    }
2264}