Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::{SECRET_TYPE_LABEL, SecretExt, SecretType};
20use self::service::ServiceExt;
21use self::statefulset::{
22    StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23    preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_backup_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
31use crate::kanidm::crd::{
32    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
33};
34use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
35use crate::telemetry;
36
37use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
38use kaniop_k8s_util::client::get_output;
39use kaniop_k8s_util::error::{Error, Result};
40use kaniop_k8s_util::parse::parse_semver;
41use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::fmt::Debug;
45use std::sync::{Arc, LazyLock};
46
47use futures::future::{TryJoinAll, join_all, try_join_all};
48use futures::stream::{self, StreamExt};
49use futures::try_join;
50use k8s_openapi::NamespaceResourceScope;
51use k8s_openapi::api::apps::v1::StatefulSet;
52use k8s_openapi::api::core::v1::{Pod, Secret, Service};
53use kube::Resource;
54use kube::ResourceExt;
55use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
56use kube::runtime::controller::Action;
57use kube::runtime::events::{Event, EventType};
58use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
59use serde::{Deserialize, Serialize};
60use tokio::time::{Duration, sleep};
61use tracing::{Span, debug, field, info, instrument, trace, warn};
62
63const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
64const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
65const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
66const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
67const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
68const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
69const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
70const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
71const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
72
73pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
74const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
75pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
76
77const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
78
79static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
80    BTreeMap::from([
81        (NAME_LABEL.to_string(), "kanidm".to_string()),
82        (
83            MANAGED_BY_LABEL.to_string(),
84            format!("kaniop-{CONTROLLER_ID}"),
85        ),
86    ])
87});
88
89async fn restart_statefulsets_after_replica_secret_update(
90    ctx: Arc<Context>,
91    sts_api: &Api<StatefulSet>,
92    namespace: &str,
93    statefulset_names: &[String],
94    sequential: bool,
95) -> Result<()> {
96    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
97    // Pods must restart after cert secret changes or replication keeps stale TLS material.
98    let restart_futures = statefulset_names
99        .iter()
100        .map(|sts_name| sts_api.restart(sts_name));
101
102    if sequential {
103        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
104        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
105            return Err(Error::kube_error(
106                "restart",
107                "StatefulSet",
108                namespace,
109                "replica certificate secret update statefulsets",
110                first_err,
111            ));
112        }
113    } else {
114        let results: Vec<_> = join_all(restart_futures).await;
115        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
116            return Err(Error::kube_error(
117                "restart",
118                "StatefulSet",
119                namespace,
120                "replica certificate secret update statefulsets",
121                first_err,
122            ));
123        }
124    }
125
126    let rollout_futures = statefulset_names
127        .iter()
128        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
129    try_join_all(rollout_futures).await?;
130
131    Ok(())
132}
133
134pub async fn reconcile_admins_secret(
135    kanidm: Arc<Kanidm>,
136    ctx: Arc<Context>,
137    status: &KanidmStatus,
138) -> Result<bool> {
139    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
140        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
141        kanidm.patch(&ctx, admins_secret).await?;
142        return Ok(true);
143    }
144    Ok(false)
145}
146
147pub async fn reconcile_replication_secrets(
148    kanidm: Arc<Kanidm>,
149    ctx: Arc<Context>,
150    status: &KanidmStatus,
151) -> Result<bool> {
152    let mut changed = false;
153    let expected_secret_names = if kanidm.is_replication_enabled() {
154        status
155            .replica_statuses
156            .iter()
157            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
158            .collect::<Vec<_>>()
159    } else {
160        Vec::new()
161    };
162    let replica_cert_label =
163        serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
164    let deprecated_secrets = ctx
165        .stores
166        .secret_store
167        .state()
168        .into_iter()
169        .filter(|secret| {
170            secret.namespace() == kanidm.namespace()
171                && kanidm.admins_secret_name() != secret.name_any()
172                && !expected_secret_names.contains(&secret.name_any())
173                && secret.metadata.labels.as_ref().is_some_and(|l| {
174                    l.get(SECRET_TYPE_LABEL) == Some(&replica_cert_label)
175                        && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
176                })
177        })
178        .collect::<Vec<_>>();
179    if !deprecated_secrets.is_empty() {
180        changed = true;
181    }
182    let secret_delete_future = deprecated_secrets
183        .iter()
184        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
185        .collect::<TryJoinAll<_>>();
186    try_join!(secret_delete_future)?;
187
188    if kanidm.is_replication_enabled() {
189        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
190        let statefulset_names = status
191            .replica_statuses
192            .iter()
193            .map(|rs| rs.statefulset_name.clone())
194            .collect::<BTreeSet<_>>()
195            .into_iter()
196            .collect::<Vec<_>>();
197
198        let has_pending = status
199            .replica_statuses
200            .iter()
201            .any(|rs| rs.state == KanidmReplicaState::Pending);
202
203        let sts_api =
204            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
205
206        if has_pending {
207            changed = true;
208            stream::iter(
209                status
210                    .replica_statuses
211                    .iter()
212                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
213            )
214            .then(|rs| async {
215                let secret = kanidm
216                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
217                    .await?;
218                kanidm.patch(&ctx, secret).await
219            })
220            .collect::<Vec<_>>()
221            .await
222            .into_iter()
223            .collect::<Result<Vec<_>>>()?;
224
225            let restart_futures = status
226                .replica_statuses
227                .iter()
228                .map(|rs| sts_api.restart(&rs.statefulset_name));
229
230            if has_single_replica {
231                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
232                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
233                    return Err(Error::kube_error(
234                        "restart",
235                        "StatefulSet",
236                        kanidm.get_namespace(),
237                        "pending replica statefulsets",
238                        first_err,
239                    ));
240                }
241            } else {
242                let results: Vec<_> = join_all(restart_futures).await;
243                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
244                    return Err(Error::kube_error(
245                        "restart",
246                        "StatefulSet",
247                        kanidm.get_namespace(),
248                        "pending replica statefulsets",
249                        first_err,
250                    ));
251                }
252            }
253        }
254
255        let has_certificate_host_invalid = status
256            .replica_statuses
257            .iter()
258            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
259
260        let has_certificate_expiring = status
261            .replica_statuses
262            .iter()
263            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
264
265        let has_non_ready_replicas = status
266            .replica_statuses
267            .iter()
268            .any(|rs| rs.state != KanidmReplicaState::Ready);
269
270        let host_invalid_replicas: Vec<_> = status
271            .replica_statuses
272            .iter()
273            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
274            .collect();
275
276        let cert_expiring_replicas: Vec<_> = status
277            .replica_statuses
278            .iter()
279            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
280            .collect();
281
282        if !host_invalid_replicas.is_empty() {
283            changed = true;
284            let restart_futures = host_invalid_replicas
285                .iter()
286                .map(|rs| sts_api.restart(&rs.statefulset_name));
287
288            if has_single_replica {
289                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
290                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
291                    return Err(Error::kube_error(
292                        "restart",
293                        "StatefulSet",
294                        kanidm.get_namespace(),
295                        "certificate host invalid statefulsets",
296                        first_err,
297                    ));
298                }
299            } else {
300                let results: Vec<_> = join_all(restart_futures).await;
301                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
302                    return Err(Error::kube_error(
303                        "restart",
304                        "StatefulSet",
305                        kanidm.get_namespace(),
306                        "certificate host invalid statefulsets",
307                        first_err,
308                    ));
309                }
310            }
311
312            let restarted_sts_names: Vec<_> = host_invalid_replicas
313                .iter()
314                .map(|rs| rs.statefulset_name.clone())
315                .collect::<BTreeSet<_>>()
316                .into_iter()
317                .collect();
318
319            let namespace = kanidm.get_namespace();
320            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
321                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
322            });
323            try_join_all(rollout_futures).await?;
324
325            let renew_futures = host_invalid_replicas
326                .iter()
327                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
328            let results: Vec<_> = join_all(renew_futures).await;
329            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
330                return Err(first_err);
331            }
332
333            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
334
335            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
336                let cert = show_replica_cert_with_retries(
337                    &kanidm,
338                    ctx.clone(),
339                    &rs.pod_name,
340                    CERT_SHOW_RETRY_ATTEMPTS,
341                )
342                .await?;
343                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
344                kanidm.patch(&ctx, secret.clone()).await
345            });
346            try_join_all(cert_futures).await?;
347
348            restart_statefulsets_after_replica_secret_update(
349                ctx.clone(),
350                &sts_api,
351                &kanidm.get_namespace(),
352                &statefulset_names,
353                has_single_replica,
354            )
355            .await?;
356
357            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
358        }
359
360        if !cert_expiring_replicas.is_empty() {
361            changed = true;
362            let renew_futures = cert_expiring_replicas
363                .iter()
364                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
365            let results: Vec<_> = join_all(renew_futures).await;
366            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
367                return Err(first_err);
368            }
369
370            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
371
372            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
373                let cert = show_replica_cert_with_retries(
374                    &kanidm,
375                    ctx.clone(),
376                    &rs.pod_name,
377                    CERT_SHOW_RETRY_ATTEMPTS,
378                )
379                .await?;
380                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
381                kanidm.patch(&ctx, secret.clone()).await
382            });
383            try_join_all(cert_futures).await?;
384
385            restart_statefulsets_after_replica_secret_update(
386                ctx.clone(),
387                &sts_api,
388                &kanidm.get_namespace(),
389                &statefulset_names,
390                has_single_replica,
391            )
392            .await?;
393
394            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
395        }
396
397        if has_non_ready_replicas
398            && !has_pending
399            && !has_certificate_host_invalid
400            && !has_certificate_expiring
401        {
402            changed = true;
403            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
404
405            let restart_futures = status
406                .replica_statuses
407                .iter()
408                .map(|rs| sts_api.restart(&rs.statefulset_name));
409
410            if has_single_replica {
411                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
412                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
413                    return Err(Error::kube_error(
414                        "restart",
415                        "StatefulSet",
416                        kanidm.get_namespace(),
417                        "replica statefulsets",
418                        first_err,
419                    ));
420                }
421            } else {
422                let results: Vec<_> = join_all(restart_futures).await;
423                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
424                    return Err(Error::kube_error(
425                        "restart",
426                        "StatefulSet",
427                        kanidm.get_namespace(),
428                        "replica statefulsets",
429                        first_err,
430                    ));
431                }
432            }
433        }
434    }
435
436    Ok(changed)
437}
438
439#[instrument(skip(ctx, kanidm))]
440pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
441    let trace_id = telemetry::get_trace_id();
442    Span::current().record("trace_id", field::display(&trace_id));
443    let _timer = ctx
444        .kaniop_ctx
445        .metrics
446        .reconcile_count_and_measure(&trace_id);
447    info!("reconciling Kanidm");
448
449    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
450        debug!(%e, "failed to reconcile status");
451        ctx.kaniop_ctx.metrics.status_update_errors_inc();
452        e
453    })?;
454    let kanidm_api: Api<Kanidm> =
455        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
456    let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
457    let outcome_clone = outcome.clone();
458    let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
459        let outcome = outcome_clone.clone();
460        let ctx = ctx.clone();
461        let status = status.clone();
462        async move {
463            match event {
464                Finalizer::Apply(kanidm) => {
465                    let (action, changed) = reconcile(kanidm, ctx, status).await?;
466                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
467                    Ok(action)
468                }
469                Finalizer::Cleanup(kanidm) => {
470                    let (action, changed) = cleanup(kanidm, ctx).await?;
471                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
472                    Ok(action)
473                }
474            }
475        }
476    })
477    .await
478    .or_else(|e| match e {
479        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
480            debug!("resource already removed during finalizer cleanup");
481            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
482        }
483        _ => Err(Error::FinalizerError(
484            "failed on kanidm account finalizer".to_string(),
485            Box::new(e),
486        )),
487    })?;
488    let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
489    Ok((action, changed))
490}
491
492/// Hash of the TLS secret content, injected as a pod template annotation so
493/// certificate renewals roll the statefulsets. `None` if the secret does not
494/// exist yet.
495async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
496    let secret_name = kanidm.effective_tls_secret_name();
497    let secret_api =
498        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
499    let secret = secret_api
500        .get_opt(&secret_name)
501        .await
502        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
503    Ok(secret.as_ref().map(hash_secret_data))
504}
505
506/// Previous TLS secret hash annotation from the existing statefulsets.
507///
508/// Used when the TLS secret is missing: dropping the annotation would roll
509/// the pods onto a missing volume and leave them stuck in ContainerCreating.
510fn previous_tls_secret_hash(
511    kanidm: &Kanidm,
512    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
513) -> Option<String> {
514    stateful_sets
515        .into_iter()
516        .filter(|sts| {
517            sts.namespace() == kanidm.namespace()
518                && sts
519                    .metadata
520                    .labels
521                    .as_ref()
522                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
523        })
524        .find_map(|sts| {
525            sts.spec
526                .as_ref()?
527                .template
528                .metadata
529                .as_ref()?
530                .annotations
531                .as_ref()?
532                .get(TLS_SECRET_HASH_ANNOTATION)
533                .cloned()
534        })
535}
536
537fn is_headless_service(service: &Service) -> bool {
538    service
539        .spec
540        .as_ref()
541        .and_then(|spec| spec.cluster_ip.as_deref())
542        == Some("None")
543}
544
545fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
546    if is_headless_service(desired) != is_headless_service(current) {
547        return;
548    }
549    let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
550    else {
551        return;
552    };
553    if desired_spec.cluster_ip.is_none() {
554        desired_spec.cluster_ip = current_spec.cluster_ip.clone();
555    }
556    if desired_spec.cluster_ips.is_none() {
557        desired_spec.cluster_ips = current_spec.cluster_ips.clone();
558    }
559}
560
561async fn wait_for_statefulset_deletion(
562    client: kube::Client,
563    namespace: &str,
564    name: &str,
565) -> Result<()> {
566    let api = Api::<StatefulSet>::namespaced(client, namespace);
567    let start = std::time::Instant::now();
568    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
569    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
570
571    loop {
572        match api.get_opt(name).await {
573            Ok(None) => return Ok(()),
574            Ok(Some(_)) => {}
575            Err(e) => {
576                return Err(Error::kube_error(
577                    "wait for deletion of",
578                    "StatefulSet",
579                    namespace,
580                    name,
581                    e,
582                ));
583            }
584        }
585
586        if start.elapsed() >= timeout {
587            return Err(Error::ReceiveOutput(format!(
588                "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
589            )));
590        }
591        sleep(poll_interval).await;
592    }
593}
594
595async fn reconcile_service(
596    kanidm: &Kanidm,
597    ctx: &Context,
598    mut desired: Service,
599) -> Result<Service> {
600    let namespace = kanidm.get_namespace();
601    let name = desired.name_any();
602    let desired_headless = is_headless_service(&desired);
603
604    if let Some(cached) = ctx
605        .stores
606        .service_store
607        .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
608    {
609        if is_headless_service(cached.as_ref()) == desired_headless {
610            preserve_defaulted_service_fields(&mut desired, cached.as_ref());
611        } else {
612            let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
613            let current = service_api
614                .get_opt(&name)
615                .await
616                .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
617
618            if let Some(current) = current {
619                let current_headless = is_headless_service(&current);
620                if current_headless != desired_headless {
621                    info!(
622                        resource.name = &name,
623                        resource.namespace = &namespace,
624                        current_headless,
625                        desired_headless,
626                        "recreating Service to change headless mode"
627                    );
628                    kanidm.delete(ctx, &current).await?;
629                    ctx.kaniop_ctx
630                        .metrics
631                        .reconcile_deploy_delete_create_inc("Service", "headless_transition");
632                    wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
633                        .await?;
634                } else {
635                    preserve_defaulted_service_fields(&mut desired, &current);
636                }
637            }
638        }
639    }
640
641    kanidm.patch(ctx, desired).await
642}
643
644async fn wait_for_service_deletion(
645    client: kube::Client,
646    namespace: &str,
647    name: &str,
648) -> Result<()> {
649    let api = Api::<Service>::namespaced(client, namespace);
650    let start = std::time::Instant::now();
651    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
652    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
653
654    loop {
655        match api.get_opt(name).await {
656            Ok(None) => return Ok(()),
657            Ok(Some(_)) => {}
658            Err(e) => {
659                return Err(Error::kube_error(
660                    "wait for deletion of",
661                    "Service",
662                    namespace,
663                    name,
664                    e,
665                ));
666            }
667        }
668
669        if start.elapsed() >= timeout {
670            return Err(Error::ReceiveOutput(format!(
671                "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
672            )));
673        }
674        sleep(poll_interval).await;
675    }
676}
677
678async fn validate_statefulset_replacement(
679    api: &Api<StatefulSet>,
680    desired: &StatefulSet,
681    namespace: &str,
682    name: &str,
683) -> Result<()> {
684    let mut candidate = desired.clone();
685    candidate.metadata.name = None;
686    candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
687    candidate.metadata.namespace = None;
688    candidate.metadata.resource_version = None;
689    candidate.metadata.uid = None;
690    candidate.metadata.creation_timestamp = None;
691    candidate.metadata.generation = None;
692    candidate.metadata.managed_fields = None;
693    candidate.metadata.deletion_timestamp = None;
694    candidate.metadata.deletion_grace_period_seconds = None;
695    candidate.status = None;
696
697    api.create(
698        &PostParams {
699            dry_run: true,
700            ..Default::default()
701        },
702        &candidate,
703    )
704    .await
705    .map(|_| ())
706    .map_err(|e| {
707        Error::kube_error(
708            "validate replacement for",
709            "StatefulSet",
710            namespace,
711            name,
712            e,
713        )
714    })
715}
716
717async fn reconcile_statefulset(
718    kanidm: Arc<Kanidm>,
719    ctx: Arc<Context>,
720    desired: StatefulSet,
721) -> Result<StatefulSet> {
722    let namespace = kanidm.namespace().unwrap();
723    let name = desired.name_any();
724    let cached_current = ctx.stores.stateful_set_store.find(|current| {
725        current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
726    });
727
728    // On a cold reflector cache, keep the normal create/update path GET-free. If the optimistic
729    // apply fails with HTTP 422, the object may already exist with an immutable conflict that the
730    // cache has not observed yet. Only 422 errors enter the live classification/recreate recovery
731    // path; 429, 5xx, 403, and transport errors are returned immediately and non-destructively.
732    let cache_miss_422_error = if let Some(cached_current) = cached_current {
733        let mut cached_desired = desired.clone();
734        preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
735        if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
736            == StatefulSetApplyStrategy::Apply
737        {
738            return kanidm.apply(&ctx, cached_desired).await;
739        }
740        None
741    } else {
742        match kanidm.apply(&ctx, desired.clone()).await {
743            Ok(applied) => return Ok(applied),
744            Err(error) => {
745                // Only HTTP 422 can indicate an immutable-field conflict that is safe to
746                // recover from via delete/recreate. Arbitrary failures must not trigger
747                // destruction.
748                if is_kube_422(&error) {
749                    Some(error)
750                } else {
751                    return Err(error);
752                }
753            }
754        }
755    };
756
757    // Never make a destructive decision from reflector state or an SSA error alone. Re-read the
758    // live object and classify it immediately before deletion; another actor or an earlier
759    // reconcile may already have converged the immutable fields while the cache was catching up.
760    let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
761    let Some(live_current) = api
762        .get_opt(&name)
763        .await
764        .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
765    else {
766        return match cache_miss_422_error {
767            Some(error) => Err(error),
768            None => kanidm.apply(&ctx, desired).await,
769        };
770    };
771
772    // Start from the original desired object again. Defaults copied from a stale reflector entry
773    // must not survive into the live re-check and become the reason for a recreation.
774    let mut live_desired = desired.clone();
775    preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
776    match classify_statefulset_change(&live_current, &live_desired) {
777        StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
778            Some(error) => Err(error),
779            None => kanidm.apply(&ctx, live_desired).await,
780        },
781        StatefulSetApplyStrategy::Recreate { .. } => {
782            // An update 422 can combine an immutable conflict with independently-invalid desired
783            // state. Validate the replacement as a dry-run CREATE before deleting the live object,
784            // so create-time validation runs without the existing object's immutable-update check.
785            validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
786
787            // Re-read and reclassify immediately before destruction. Another actor or an earlier
788            // reconcile may have changed the live immutable fields or PVC-retention semantics.
789            let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
790                Error::kube_error(
791                    "re-read before deletion",
792                    "StatefulSet",
793                    &namespace,
794                    &name,
795                    e,
796                )
797            })?
798            else {
799                return kanidm.apply(&ctx, live_desired).await;
800            };
801            let mut latest_desired = live_desired.clone();
802            preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
803
804            match classify_statefulset_change(&latest_current, &latest_desired) {
805                StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
806                StatefulSetApplyStrategy::Recreate { immutable_fields } => {
807                    // Validate the replacement StatefulSet via a dry-run CREATE before deleting
808                    // the live object. This prevents deleting on a mixed 422 (immutable conflict
809                    // + independently-invalid desired state) where the replacement would still
810                    // be rejected by the API.
811                    validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
812                        .await?;
813
814                    info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
815                    kanidm.delete(&ctx, &latest_current).await?;
816                    ctx.kaniop_ctx
817                        .metrics
818                        .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
819                    wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
820                        .await?;
821                    kanidm.apply(&ctx, latest_desired).await
822                }
823            }
824        }
825    }
826}
827
828/// Return true only when the error structurally wraps a Kubernetes HTTP 422 response.
829fn is_kube_422(error: &Error) -> bool {
830    matches!(
831        error,
832        Error::KubeError(_, cause)
833            if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
834    )
835}
836
837#[cfg(test)]
838mod statefulset_recreate_recovery_tests {
839    use super::{Error, is_kube_422};
840
841    fn kube_api_error(code: u16) -> Error {
842        Error::KubeError(
843            "test error".to_string(),
844            Box::new(kube::Error::Api(Box::new(kube::error::Status {
845                code,
846                message: "test".to_string(),
847                reason: "test".to_string(),
848                ..Default::default()
849            }))),
850        )
851    }
852
853    #[test]
854    fn recreate_recovery_accepts_only_422() {
855        assert!(is_kube_422(&kube_api_error(422)));
856        assert!(!is_kube_422(&kube_api_error(403)));
857        assert!(!is_kube_422(&kube_api_error(429)));
858        assert!(!is_kube_422(&kube_api_error(500)));
859    }
860
861    #[test]
862    fn recreate_recovery_rejects_429_too_many_requests() {
863        assert!(!is_kube_422(&kube_api_error(429)));
864    }
865
866    #[test]
867    fn recreate_recovery_rejects_5xx_server_errors() {
868        assert!(!is_kube_422(&kube_api_error(500)));
869        assert!(!is_kube_422(&kube_api_error(502)));
870        assert!(!is_kube_422(&kube_api_error(503)));
871        assert!(!is_kube_422(&kube_api_error(504)));
872    }
873
874    #[test]
875    fn recreate_recovery_rejects_400_and_403() {
876        assert!(!is_kube_422(&kube_api_error(400)));
877        assert!(!is_kube_422(&kube_api_error(403)));
878    }
879}
880
881async fn reconcile(
882    kanidm: Arc<Kanidm>,
883    ctx: Arc<Context>,
884    status: KanidmStatus,
885) -> Result<(Action, bool)> {
886    if kanidm
887        .annotations()
888        .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
889    {
890        ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
891        return Ok((Action::requeue(Duration::from_secs(5)), false));
892    }
893
894    let mut changed = false;
895    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
896    let replication_secret_futures =
897        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
898
899    let sts_to_delete = {
900        let expected_sts_names = kanidm
901            .spec
902            .replica_groups
903            .iter()
904            .map(|rg| kanidm.statefulset_name(&rg.name))
905            .collect::<Vec<_>>();
906        ctx.stores
907            .stateful_set_store
908            .state()
909            .into_iter()
910            .filter(|sts| {
911                sts.namespace() == kanidm.namespace()
912                    && sts
913                        .metadata
914                        .labels
915                        .as_ref()
916                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
917                    && !expected_sts_names.contains(&sts.name_any())
918            })
919            .collect::<Vec<_>>()
920    };
921    if !sts_to_delete.is_empty() {
922        changed = true;
923    }
924    let sts_delete_futures = sts_to_delete
925        .iter()
926        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
927        .collect::<TryJoinAll<_>>();
928
929    let sts_futures = match kanidm.is_updatable(&status) {
930        true => {
931            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
932                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
933            });
934
935            let namespace = kanidm.get_namespace();
936            let schedule_api: Api<KanidmBackupSchedule> =
937                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
938            let schedules = schedule_api
939                .list(&Default::default())
940                .await
941                .map_err(|e| {
942                    Error::KubeError(
943                        "failed to list KanidmBackupSchedules for backup configuration resolution"
944                            .to_string(),
945                        Box::new(e),
946                    )
947                })?
948                .items;
949
950            let repo_api: Api<KanidmBackupRepository> =
951                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
952            let repositories = repo_api
953                .list(&Default::default())
954                .await
955                .map_err(|e| {
956                    Error::KubeError(
957                        "failed to list KanidmBackupRepositories for backup configuration resolution"
958                            .to_string(),
959                        Box::new(e),
960                    )
961                })?
962                .items;
963            let backup_config = resolve_backup_config(&kanidm, &schedules, &repositories);
964
965            kanidm
966                .spec
967                .replica_groups
968                .iter()
969                .map(|rg| {
970                    let sts = kanidm.create_statefulset(
971                        rg,
972                        tls_secret_hash.as_deref(),
973                        backup_config.as_ref(),
974                    )?;
975                    Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
976                })
977                .collect::<Result<TryJoinAll<_>, _>>()?
978        }
979        false => {
980            let note = match status.version.as_ref() {
981                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
982                    format!(
983                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
984                        v.image_tag,
985                        crate::version::KANIDM_CLIENT_VERSION
986                    )
987                }
988                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
989            };
990            let _ignore_error = ctx
991                .kaniop_ctx
992                .recorder
993                .publish(
994                    &Event {
995                        type_: EventType::Warning,
996                        reason: "UpgradeBlocked".to_string(),
997                        note: Some(note),
998                        action: "ReconcileStatefulSet".to_string(),
999                        secondary: None,
1000                    },
1001                    &kanidm.object_ref(&()),
1002                )
1003                .await
1004                .map_err(|e| {
1005                    warn!(%e, "failed to publish KanidmError event");
1006                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
1007                });
1008            try_join_all([])
1009        }
1010    };
1011    let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1012
1013    let deprecated_rg_svcs = {
1014        let expected_rg_svcs_names = kanidm
1015            .spec
1016            .replica_groups
1017            .iter()
1018            .filter(|rg| rg.services.is_some())
1019            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1020            .collect::<Vec<_>>();
1021        ctx.stores
1022            .service_store
1023            .state()
1024            .into_iter()
1025            .filter(|svc| {
1026                let labels = svc.metadata.labels.as_ref();
1027                svc.namespace() == kanidm.namespace()
1028                    && !expected_rg_svcs_names.contains(&svc.name_any())
1029                    && labels.is_some_and(|l| {
1030                        l.contains_key(REPLICA_LABEL)
1031                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1032                    })
1033            })
1034            .collect::<Vec<_>>()
1035    };
1036    if !deprecated_rg_svcs.is_empty() {
1037        changed = true;
1038    }
1039    let rg_svcs_delete_futures = deprecated_rg_svcs
1040        .iter()
1041        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1042        .collect::<TryJoinAll<_>>();
1043
1044    let rg_services_futures = kanidm
1045        .spec
1046        .replica_groups
1047        .iter()
1048        .filter(|rg| rg.services.is_some())
1049        .flat_map(|rg| {
1050            (0..rg.replicas).map(|i| {
1051                let mut svc = kanidm.create_replica_group_service(rg, i);
1052                if let Some(current) = ctx.stores.service_store.find(|current| {
1053                    current.namespace() == kanidm.namespace()
1054                        && current.name_any() == svc.name_any()
1055                }) {
1056                    preserve_defaulted_service_fields(&mut svc, current.as_ref());
1057                }
1058                kanidm.patch(&ctx, svc)
1059            })
1060        })
1061        .collect::<TryJoinAll<_>>();
1062
1063    let deprecated_ingresses = {
1064        let names = [
1065            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1066            kanidm.generate_region_ingress_name(),
1067        ]
1068        .into_iter()
1069        .flatten()
1070        .collect::<Vec<_>>();
1071        ctx.stores
1072            .ingress_store
1073            .state()
1074            .into_iter()
1075            .filter(|ing| {
1076                ing.namespace() == kanidm.namespace()
1077                    && !names.contains(&ing.name_any())
1078                    && ing.metadata.labels == Some(kanidm.generate_labels())
1079            })
1080            .collect::<Vec<_>>()
1081    };
1082    if !deprecated_ingresses.is_empty() {
1083        changed = true;
1084    }
1085    let ingress_delete_futures = deprecated_ingresses
1086        .iter()
1087        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1088        .collect::<TryJoinAll<_>>();
1089
1090    let ingress_futures = kanidm
1091        .create_ingress()
1092        .into_iter()
1093        .chain(kanidm.create_region_ingress())
1094        .map(|ingress| kanidm.patch(&ctx, ingress))
1095        .collect::<TryJoinAll<_>>();
1096
1097    let deprecated_http_routes = match &ctx.stores.http_route_store {
1098        Some(store) => {
1099            let expected_names = kanidm
1100                .spec
1101                .gateway
1102                .as_ref()
1103                .map(|_| vec![kanidm.name_any()])
1104                .unwrap_or_default();
1105            let routes: Vec<_> = store
1106                .state()
1107                .into_iter()
1108                .filter(|route| {
1109                    route.namespace() == kanidm.namespace()
1110                        && !expected_names.contains(&route.name_any())
1111                        && route.metadata.labels == Some(kanidm.generate_labels())
1112                })
1113                .collect();
1114            if !routes.is_empty() {
1115                changed = true;
1116            }
1117            routes
1118        }
1119        None => Vec::new(),
1120    };
1121    let http_route_delete_futures = deprecated_http_routes
1122        .iter()
1123        .map(|route| kanidm.delete(&ctx, route.as_ref()))
1124        .collect::<TryJoinAll<_>>();
1125
1126    let http_route_futures = if ctx.stores.http_route_store.is_some() {
1127        kanidm
1128            .create_http_route()
1129            .into_iter()
1130            .map(|route| kanidm.patch(&ctx, route))
1131            .collect::<TryJoinAll<_>>()
1132    } else {
1133        try_join_all(Vec::new())
1134    };
1135
1136    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1137        Some(store) => {
1138            let expected_names = kanidm
1139                .spec
1140                .gateway
1141                .as_ref()
1142                .and_then(|g| g.backend_tls_policy.as_ref())
1143                .map(|_| vec![kanidm.name_any()])
1144                .unwrap_or_default();
1145            let policies: Vec<_> = store
1146                .state()
1147                .into_iter()
1148                .filter(|policy| {
1149                    policy.namespace() == kanidm.namespace()
1150                        && !expected_names.contains(&policy.name_any())
1151                        && policy.metadata.labels == Some(kanidm.generate_labels())
1152                })
1153                .collect();
1154            if !policies.is_empty() {
1155                changed = true;
1156            }
1157            policies
1158        }
1159        None => Vec::new(),
1160    };
1161    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1162        .iter()
1163        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1164        .collect::<TryJoinAll<_>>();
1165
1166    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1167        kanidm
1168            .create_backend_tls_policy()
1169            .into_iter()
1170            .map(|policy| kanidm.patch(&ctx, policy))
1171            .collect::<TryJoinAll<_>>()
1172    } else {
1173        try_join_all(Vec::new())
1174    };
1175
1176    let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1177        sts_delete_futures,
1178        admin_secret_future,
1179        replication_secret_futures,
1180        sts_futures,
1181        service_future,
1182        rg_svcs_delete_futures,
1183        rg_services_futures,
1184        ingress_delete_futures,
1185        ingress_futures,
1186        http_route_delete_futures,
1187        http_route_futures,
1188        backend_tls_policy_delete_futures,
1189        backend_tls_policy_futures
1190    )?;
1191
1192    if admin_secret_changed || replication_secrets_changed {
1193        changed = true;
1194    }
1195
1196    if is_kanidm_available(status.clone()) {
1197        let namespace = kanidm.namespace().unwrap();
1198        let name = kanidm.name_any();
1199        match crate::controller::kanidm::KanidmClients::create_client(
1200            &namespace,
1201            &name,
1202            crate::controller::kanidm::KanidmUser::Admin,
1203            ctx.kaniop_ctx.client.clone(),
1204        )
1205        .await
1206        {
1207            Ok(system_client) => {
1208                if let Err(e) =
1209                    reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1210                {
1211                    warn!(%e, "failed to reconcile domain appearance");
1212                }
1213            }
1214            Err(e) => {
1215                warn!(%e, "failed to create admin client for domain appearance reconciliation");
1216            }
1217        }
1218
1219        match crate::controller::kanidm::KanidmClients::create_client(
1220            &namespace,
1221            &name,
1222            crate::controller::kanidm::KanidmUser::IdmAdmin,
1223            ctx.kaniop_ctx.client.clone(),
1224        )
1225        .await
1226        {
1227            Ok(kanidm_client) => {
1228                let (mail_sender_status, mail_sender_mutated) =
1229                    reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1230                        .await
1231                        .unwrap_or_else(|e| {
1232                            warn!(%e, "failed to reconcile mail sender");
1233                            (None, false)
1234                        });
1235                if mail_sender_mutated {
1236                    changed = true;
1237                }
1238                let current_mail_sender_status =
1239                    kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1240                if mail_sender_status != current_mail_sender_status {
1241                    let kanidm_api: Api<Kanidm> =
1242                        Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1243                    let status_patch = serde_json::json!({
1244                        "status": {
1245                            "mailSender": mail_sender_status
1246                        }
1247                    });
1248                    if let Err(e) = kanidm_api
1249                        .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1250                        .await
1251                    {
1252                        warn!(%e, "failed to patch Kanidm/status for mail sender");
1253                    }
1254                }
1255            }
1256            Err(e) => {
1257                warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1258            }
1259        }
1260    }
1261
1262    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1263}
1264
1265async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1266    debug!("cleanup");
1267
1268    let mut changed = false;
1269    let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1270    if mail_resources_cleaned {
1271        changed = true;
1272    }
1273
1274    let namespace = kanidm.namespace().unwrap();
1275    let name = kanidm.name_any();
1276
1277    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1278        &namespace,
1279        &name,
1280        crate::controller::kanidm::KanidmUser::IdmAdmin,
1281        ctx.kaniop_ctx.client.clone(),
1282    )
1283    .await
1284    {
1285        let kanidm_cleanup_done =
1286            cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1287        if kanidm_cleanup_done {
1288            changed = true;
1289        }
1290    }
1291
1292    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1293    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1294}
1295
1296async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1297    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1298    let start = std::time::Instant::now();
1299    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1300    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1301
1302    loop {
1303        let sts = sts_api.get(sts_name).await.map_err(|e| {
1304            Error::KubeError(
1305                format!("failed to get StatefulSet {namespace}/{sts_name}"),
1306                Box::new(e),
1307            )
1308        })?;
1309
1310        if sts.metadata.deletion_timestamp.is_some() {
1311            return Err(Error::ReceiveOutput(format!(
1312                "StatefulSet {namespace}/{sts_name} is being deleted"
1313            )));
1314        }
1315
1316        let spec = sts.spec.as_ref().ok_or_else(|| {
1317            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1318        })?;
1319
1320        let status = match sts.status {
1321            Some(ref s) => s,
1322            None => {
1323                if start.elapsed() >= timeout {
1324                    return Err(Error::ReceiveOutput(format!(
1325                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1326                    )));
1327                }
1328                sleep(poll_interval).await;
1329                continue;
1330            }
1331        };
1332
1333        let replicas = spec.replicas.unwrap_or(0);
1334
1335        if replicas == 0 {
1336            return Ok(());
1337        }
1338
1339        let updated_replicas = status.updated_replicas.unwrap_or(0);
1340        let current_revision = status.current_revision.as_deref().unwrap_or("");
1341        let update_revision = status.update_revision.as_deref().unwrap_or("");
1342        let ready_replicas = status.ready_replicas.unwrap_or(0);
1343
1344        if updated_replicas == replicas
1345            && current_revision == update_revision
1346            && ready_replicas == replicas
1347        {
1348            return Ok(());
1349        }
1350
1351        if start.elapsed() >= timeout {
1352            return Err(Error::ReceiveOutput(format!(
1353                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1354            )));
1355        }
1356
1357        sleep(poll_interval).await;
1358    }
1359}
1360
1361async fn show_replica_cert_with_retries(
1362    kanidm: &Kanidm,
1363    ctx: Arc<Context>,
1364    pod_name: &str,
1365    max_attempts: u32,
1366) -> std::result::Result<String, Error> {
1367    let mut last_error = None;
1368    for attempt in 0..max_attempts {
1369        if attempt > 0 {
1370            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1371        }
1372        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1373            Ok(cert) => return Ok(cert),
1374            Err(e) => {
1375                info!(
1376                    "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1377                    attempt + 1,
1378                    max_attempts
1379                );
1380                last_error = Some(e);
1381            }
1382        }
1383    }
1384    Err(last_error.unwrap_or_else(|| {
1385        Error::ReceiveOutput(format!(
1386            "failed to get certificate for {pod_name} after {max_attempts} attempts"
1387        ))
1388    }))
1389}
1390
1391impl Kanidm {
1392    // Convenience methods that handle context and operator name
1393    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1394    where
1395        K: Resource<Scope = NamespaceResourceScope>
1396            + Serialize
1397            + Clone
1398            + std::fmt::Debug
1399            + for<'de> Deserialize<'de>,
1400        <K as kube::Resource>::DynamicType: Default,
1401        <K as Resource>::Scope: std::marker::Sized,
1402    {
1403        self.kube_delete(
1404            ctx.kaniop_ctx.client.clone(),
1405            &ctx.kaniop_ctx.metrics,
1406            resource,
1407        )
1408        .await
1409    }
1410
1411    /// Server-side apply without any generic delete/recreate fallback.
1412    /// Resource-specific reconcilers use this when they own lifecycle decisions.
1413    pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1414    where
1415        K: Resource<Scope = NamespaceResourceScope>
1416            + Serialize
1417            + Clone
1418            + std::fmt::Debug
1419            + for<'de> Deserialize<'de>,
1420        <K as kube::Resource>::DynamicType: Default,
1421        <K as Resource>::Scope: std::marker::Sized,
1422    {
1423        self.kube_apply(
1424            ctx.kaniop_ctx.client.clone(),
1425            resource,
1426            KANIDM_OPERATOR_NAME,
1427        )
1428        .await
1429    }
1430
1431    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1432    where
1433        K: Resource<Scope = NamespaceResourceScope>
1434            + Serialize
1435            + Clone
1436            + std::fmt::Debug
1437            + for<'de> Deserialize<'de>,
1438        <K as kube::Resource>::DynamicType: Default,
1439        <K as Resource>::Scope: std::marker::Sized,
1440    {
1441        self.kube_patch(
1442            ctx.kaniop_ctx.client.clone(),
1443            &ctx.kaniop_ctx.metrics,
1444            resource,
1445            KANIDM_OPERATOR_NAME,
1446        )
1447        .await
1448    }
1449
1450    #[inline]
1451    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1452        LABELS
1453            .clone()
1454            .into_iter()
1455            .chain([
1456                (INSTANCE_LABEL.to_string(), self.name_any()),
1457                (CLUSTER_LABEL.to_string(), self.name_any()),
1458            ])
1459            .collect()
1460    }
1461
1462    #[inline]
1463    fn generate_labels(&self) -> BTreeMap<String, String> {
1464        self.generate_resource_labels()
1465    }
1466
1467    #[inline]
1468    fn get_tls_secret_name(&self) -> String {
1469        format!("{}-tls", self.name_any())
1470    }
1471
1472    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
1473    /// falling back to the ingress TLS secret and then to the default name.
1474    pub(crate) fn effective_tls_secret_name(&self) -> String {
1475        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1476            self.spec
1477                .ingress
1478                .as_ref()
1479                .and_then(|i| i.tls_secret_name.clone())
1480                .unwrap_or_else(|| self.get_tls_secret_name())
1481        })
1482    }
1483
1484    #[inline]
1485    fn get_namespace(&self) -> String {
1486        // safe unwrap: Kanidm is namespaced scoped
1487        self.namespace().unwrap()
1488    }
1489
1490    #[inline]
1491    fn is_replication_enabled(&self) -> bool {
1492        self.spec.replica_groups.len() > 1
1493            || self
1494                .spec
1495                .replica_groups
1496                .first()
1497                .is_some_and(|rg| rg.replicas > 1)
1498            || !self.spec.external_replication_nodes.is_empty()
1499    }
1500
1501    #[inline]
1502    fn is_updatable(&self, status: &KanidmStatus) -> bool {
1503        // The explicit upgrade-check bypass must remain effective even when
1504        // version metadata is retained for backup transport.
1505        if self.spec.disable_upgrade_checks {
1506            return true;
1507        }
1508        status
1509            .version
1510            .as_ref()
1511            .map(|v| {
1512                if matches!(
1513                    v.compatibility_result,
1514                    VersionCompatibilityResult::Incompatible
1515                        | VersionCompatibilityResult::NotChecked
1516                ) {
1517                    return false;
1518                }
1519                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1520                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1521                match v.upgrade_check_result {
1522                    KanidmUpgradeCheckResult::Passed => {
1523                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1524                            minor <= v_minor + 1
1525                        } else {
1526                            true
1527                        }
1528                    }
1529                    KanidmUpgradeCheckResult::Failed => {
1530                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1531                            versions
1532                        {
1533                            major == v_major && minor == v_minor && patch >= v_patch
1534                        } else {
1535                            v.image_tag == current_tag
1536                        }
1537                    }
1538                    // Skipped checks cannot count as successful validation.
1539                    KanidmUpgradeCheckResult::Skipped => false,
1540                }
1541            })
1542            .unwrap_or(true)
1543    }
1544
1545    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1546        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1547        let pod = pod_api.get(pod_name).await.map_err(|e| {
1548            Error::KubeError(
1549                format!("failed to get pod {namespace}/{pod_name}"),
1550                Box::new(e),
1551            )
1552        })?;
1553        Ok(pod
1554            .status
1555            .as_ref()
1556            .and_then(|s| s.conditions.as_ref())
1557            .is_some_and(|conditions| {
1558                conditions
1559                    .iter()
1560                    .any(|c| c.type_ == "Ready" && c.status == "True")
1561            }))
1562    }
1563
1564    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1565        let start = std::time::Instant::now();
1566        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1567        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1568
1569        loop {
1570            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1571                Ok(true) => return Ok(()),
1572                Ok(false) => {}
1573                Err(e) => {
1574                    if !e.is_retryable() {
1575                        return Err(e);
1576                    }
1577                    trace!(%e, "pod readiness check failed, retrying");
1578                }
1579            }
1580
1581            if start.elapsed() >= timeout {
1582                return Err(Error::ReceiveOutput(format!(
1583                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1584                )));
1585            }
1586
1587            sleep(poll_interval).await;
1588        }
1589    }
1590
1591    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1592    where
1593        I: IntoIterator<Item = T> + Debug,
1594        T: Into<String>,
1595    {
1596        let namespace = &self.get_namespace();
1597        trace!(
1598            resource.name = &pod_name,
1599            resource.namespace = &namespace,
1600            ?command,
1601            "pod exec"
1602        );
1603        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1604        let attached = pod
1605            .exec(
1606                pod_name,
1607                command,
1608                &AttachParams::default().container("kanidm"),
1609            )
1610            .await
1611            .map_err(|e| {
1612                Error::KubeError(
1613                    format!("failed to exec pod {namespace}/{pod_name}"),
1614                    Box::new(e),
1615                )
1616            })?;
1617        get_output(attached).await
1618    }
1619
1620    async fn exec_with_wait<I, T>(
1621        &self,
1622        ctx: Arc<Context>,
1623        pod_name: &str,
1624        command: I,
1625    ) -> Result<String>
1626    where
1627        I: IntoIterator<Item = T> + Debug,
1628        T: Into<String>,
1629    {
1630        let namespace = &self.get_namespace();
1631        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1632        self.exec(ctx, pod_name, command).await
1633    }
1634
1635    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1636    where
1637        I: IntoIterator<Item = T> + Debug,
1638        T: Into<String>,
1639    {
1640        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1641        let first_replica_group = self
1642            .spec
1643            .replica_groups
1644            .first()
1645            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1646        let sts_name = self.statefulset_name(&first_replica_group.name);
1647        let pod_name = format!("{sts_name}-0");
1648        self.exec(ctx, &pod_name, command).await
1649    }
1650
1651    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1652    where
1653        I: IntoIterator<Item = T> + Debug,
1654        T: Into<String>,
1655    {
1656        let first_replica_group = self
1657            .spec
1658            .replica_groups
1659            .first()
1660            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1661        let sts_name = self.statefulset_name(&first_replica_group.name);
1662        let pod_name = format!("{sts_name}-0");
1663        self.exec_with_wait(ctx, &pod_name, command).await
1664    }
1665}
1666
1667#[cfg(test)]
1668mod test {
1669    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1670    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1671
1672    use crate::controller::State;
1673    use crate::kanidm::controller::context::{Context, Stores};
1674    use crate::kanidm::crd::KanidmStatus;
1675
1676    use kaniop_k8s_util::error::Result;
1677    use kaniop_k8s_util::resources::hash_secret_data;
1678
1679    use std::collections::BTreeMap;
1680    use std::sync::Arc;
1681
1682    use http::{Request, Response};
1683    use k8s_openapi::ByteString;
1684    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1685    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1686    use k8s_openapi::api::networking::v1::Ingress;
1687    use kube::api::ObjectMeta;
1688    use kube::runtime::reflector::store::Writer;
1689    use kube::{Client, Resource, ResourceExt, client::Body};
1690    use opentelemetry::metrics::MeterProvider;
1691    use serde_json::json;
1692
1693    impl Kanidm {
1694        /// A normal test kanidm with a given status
1695        pub fn test() -> Self {
1696            let mut e = Kanidm::new(
1697                "test",
1698                serde_json::from_value(json!({
1699                    "domain": "idm.example.com",
1700                    "replicaGroups": [
1701                        {
1702                            "name": "default",
1703                            "replicas": 1
1704                        }
1705                    ]
1706                }))
1707                .unwrap(),
1708            );
1709            e.meta_mut().namespace = Some("default".into());
1710            // Pre-populate finalizer so reconcile callback runs immediately
1711            // (skips the "add finalizer + wait for next reconcile" step)
1712            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1713            e
1714        }
1715
1716        pub fn with_ingress(mut self) -> Self {
1717            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1718            self
1719        }
1720
1721        /// Modify kanidm replicas
1722        pub fn with_replicas(mut self, replicas: i32) -> Self {
1723            self.spec.replica_groups[0].replicas = replicas;
1724            self
1725        }
1726
1727        /// Modify kanidm to set a deletion timestamp
1728        pub fn needs_delete(mut self) -> Self {
1729            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1730            use k8s_openapi::jiff::Timestamp;
1731            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1732            self.meta_mut().deletion_timestamp = Some(Time(now));
1733            self
1734        }
1735
1736        /// Modify a kanidm to have an expected status
1737        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1738            self.status = Some(status);
1739            self
1740        }
1741    }
1742
1743    // We wrap tower_test::mock::Handle
1744    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1745    pub struct ApiServerVerifier(ApiServerHandle);
1746
1747    /// Scenarios we test for in ApiServerVerifier
1748    pub enum Scenario {
1749        Create(Kanidm),
1750        CreateWithTwoReplicas(Kanidm),
1751        CreateWithIngress(Kanidm),
1752        CreateWithIngressWithTwoReplicas(Kanidm),
1753        CreateWithTlsSecret(Kanidm, Box<Secret>),
1754    }
1755
1756    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1757        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1758            .await
1759            .expect("timeout on mock apiserver")
1760            .expect("scenario succeeded")
1761    }
1762
1763    impl ApiServerVerifier {
1764        /// Tests only get to run specific scenarios that has matching handlers
1765        ///
1766        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1767        ///
1768        /// NB: If the controller is making more calls than we are handling in the scenario,
1769        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1770        ///
1771        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1772        /// scenario runs to completion (i.e. all expected calls were responded to),
1773        /// using the timeout to catch missing api calls to Kubernetes.
1774        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1775            tokio::spawn(async move {
1776                // moving self => one scenario per test
1777                match scenario {
1778                    Scenario::Create(kanidm) => {
1779                        self.handle_kanidm_get(kanidm.clone())
1780                            .await
1781                            .unwrap()
1782                            .handle_kanidm_status_patch(kanidm.clone())
1783                            .await
1784                            .unwrap()
1785                            .handle_tls_secret_get_not_found(kanidm.clone())
1786                            .await
1787                            .unwrap()
1788                            .handle_backup_schedule_list()
1789                            .await
1790                            .unwrap()
1791                            .handle_backup_repository_list()
1792                            .await
1793                            .unwrap()
1794                            .handle_statefulset_patch(kanidm.clone(), None)
1795                            .await
1796                            .unwrap()
1797                            .handle_service_patch(kanidm.clone())
1798                            .await
1799                    }
1800                    Scenario::CreateWithTwoReplicas(kanidm) => {
1801                        self.handle_kanidm_get(kanidm.clone())
1802                            .await
1803                            .unwrap()
1804                            .handle_kanidm_status_patch(kanidm.clone())
1805                            .await
1806                            .unwrap()
1807                            .handle_tls_secret_get_not_found(kanidm.clone())
1808                            .await
1809                            .unwrap()
1810                            .handle_backup_schedule_list()
1811                            .await
1812                            .unwrap()
1813                            .handle_backup_repository_list()
1814                            .await
1815                            .unwrap()
1816                            .handle_statefulset_patch(kanidm.clone(), None)
1817                            .await
1818                            .unwrap()
1819                            .handle_service_patch(kanidm.clone())
1820                            .await
1821                    }
1822                    Scenario::CreateWithIngress(kanidm) => {
1823                        self.handle_kanidm_get(kanidm.clone())
1824                            .await
1825                            .unwrap()
1826                            .handle_kanidm_status_patch(kanidm.clone())
1827                            .await
1828                            .unwrap()
1829                            .handle_tls_secret_get_not_found(kanidm.clone())
1830                            .await
1831                            .unwrap()
1832                            .handle_backup_schedule_list()
1833                            .await
1834                            .unwrap()
1835                            .handle_backup_repository_list()
1836                            .await
1837                            .unwrap()
1838                            .handle_statefulset_patch(kanidm.clone(), None)
1839                            .await
1840                            .unwrap()
1841                            .handle_service_patch(kanidm.clone())
1842                            .await
1843                            .unwrap()
1844                            .handle_ingress_patch(kanidm.clone())
1845                            .await
1846                    }
1847                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1848                        self.handle_kanidm_get(kanidm.clone())
1849                            .await
1850                            .unwrap()
1851                            .handle_kanidm_status_patch(kanidm.clone())
1852                            .await
1853                            .unwrap()
1854                            .handle_tls_secret_get_not_found(kanidm.clone())
1855                            .await
1856                            .unwrap()
1857                            .handle_backup_schedule_list()
1858                            .await
1859                            .unwrap()
1860                            .handle_backup_repository_list()
1861                            .await
1862                            .unwrap()
1863                            .handle_statefulset_patch(kanidm.clone(), None)
1864                            .await
1865                            .unwrap()
1866                            .handle_service_patch(kanidm.clone())
1867                            .await
1868                            .unwrap()
1869                            .handle_ingress_patch(kanidm.clone())
1870                            .await
1871                    }
1872                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1873                        let expected_hash = hash_secret_data(&secret);
1874                        self.handle_kanidm_get(kanidm.clone())
1875                            .await
1876                            .unwrap()
1877                            .handle_kanidm_status_patch(kanidm.clone())
1878                            .await
1879                            .unwrap()
1880                            .handle_tls_secret_get(kanidm.clone(), *secret)
1881                            .await
1882                            .unwrap()
1883                            .handle_backup_schedule_list()
1884                            .await
1885                            .unwrap()
1886                            .handle_backup_repository_list()
1887                            .await
1888                            .unwrap()
1889                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1890                            .await
1891                            .unwrap()
1892                            .handle_service_patch(kanidm.clone())
1893                            .await
1894                    }
1895                }
1896                .expect("scenario completed without errors");
1897            })
1898        }
1899
1900        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1901            let (request, send) = self.0.next_request().await.expect("service not called");
1902            assert_eq!(request.method(), http::Method::GET);
1903            let expected_uri_prefix = format!(
1904                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1905                kanidm.name_any()
1906            );
1907            let uri = request.uri().to_string();
1908            assert!(
1909                uri.starts_with(&expected_uri_prefix),
1910                "expected uri to start with {}, got {}",
1911                expected_uri_prefix,
1912                uri
1913            );
1914            let response = serde_json::to_vec(&kanidm).unwrap();
1915            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1916            Ok(self)
1917        }
1918
1919        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1920            let (request, send) = self.0.next_request().await.expect("service not called");
1921            assert_eq!(request.method(), http::Method::PATCH);
1922            assert_eq!(
1923                request.uri().to_string(),
1924                format!(
1925                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1926                    kanidm.name_any()
1927                )
1928            );
1929
1930            let req_body = request.into_body().collect_bytes().await.unwrap();
1931            let json: serde_json::Value =
1932                serde_json::from_slice(&req_body).expect("patch object is json");
1933            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1934                .expect("valid kanidm status");
1935            let mut kanidm = Kanidm::test();
1936            kanidm.status = Some(status.clone());
1937            let response = serde_json::to_vec(&kanidm).unwrap();
1938            // pass through kanidm "patch accepted"
1939            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1940            Ok(self)
1941        }
1942
1943        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1944            let (request, send) = self.0.next_request().await.expect("service not called");
1945            assert_eq!(request.method(), http::Method::GET);
1946            let expected_uri_prefix = format!(
1947                "/api/v1/namespaces/default/secrets/{}",
1948                kanidm.effective_tls_secret_name()
1949            );
1950            let uri = request.uri().to_string();
1951            assert!(
1952                uri.starts_with(&expected_uri_prefix),
1953                "expected uri to start with {expected_uri_prefix}, got {uri}"
1954            );
1955            let status = serde_json::json!({
1956                "kind": "Status",
1957                "apiVersion": "v1",
1958                "metadata": {},
1959                "status": "Failure",
1960                "message": "secret not found",
1961                "reason": "NotFound",
1962                "code": 404
1963            });
1964            send.send_response(
1965                Response::builder()
1966                    .status(404)
1967                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1968                    .unwrap(),
1969            );
1970            Ok(self)
1971        }
1972
1973        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1974            let (request, send) = self.0.next_request().await.expect("service not called");
1975            assert_eq!(request.method(), http::Method::GET);
1976            let expected_uri_prefix = format!(
1977                "/api/v1/namespaces/default/secrets/{}",
1978                kanidm.effective_tls_secret_name()
1979            );
1980            let uri = request.uri().to_string();
1981            assert!(
1982                uri.starts_with(&expected_uri_prefix),
1983                "expected uri to start with {expected_uri_prefix}, got {uri}"
1984            );
1985            let response = serde_json::to_vec(&secret).unwrap();
1986            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1987            Ok(self)
1988        }
1989
1990        async fn handle_backup_schedule_list(mut self) -> Result<Self> {
1991            let (request, send) = self.0.next_request().await.expect("service not called");
1992            assert_eq!(request.method(), http::Method::GET);
1993            assert!(
1994                request
1995                    .uri()
1996                    .to_string()
1997                    .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
1998                "expected backup schedule list request, got {}",
1999                request.uri()
2000            );
2001            let list = json!({
2002                "apiVersion": "kaniop.rs/v1alpha1",
2003                "kind": "KanidmBackupScheduleList",
2004                "metadata": {},
2005                "items": []
2006            });
2007            let response = serde_json::to_vec(&list).unwrap();
2008            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2009            Ok(self)
2010        }
2011
2012        async fn handle_backup_repository_list(mut self) -> Result<Self> {
2013            let (request, send) = self.0.next_request().await.expect("service not called");
2014            assert_eq!(request.method(), http::Method::GET);
2015            assert!(
2016                request.uri().to_string().contains(
2017                    "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2018                ),
2019                "expected backup repository list request, got {}",
2020                request.uri()
2021            );
2022            let list = json!({
2023                "apiVersion": "kaniop.rs/v1alpha1",
2024                "kind": "KanidmBackupRepositoryList",
2025                "metadata": {},
2026                "items": []
2027            });
2028            let response = serde_json::to_vec(&list).unwrap();
2029            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2030            Ok(self)
2031        }
2032
2033        async fn handle_statefulset_patch(
2034            mut self,
2035            kanidm: Kanidm,
2036            expected_tls_hash: Option<String>,
2037        ) -> Result<Self> {
2038            for rg in kanidm.spec.replica_groups.iter() {
2039                let (request, send) = self.0.next_request().await.expect("service not called");
2040                assert_eq!(request.method(), http::Method::PATCH);
2041                assert_eq!(
2042                    request.uri().to_string(),
2043                    format!(
2044                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2045                        kanidm.statefulset_name(&rg.name)
2046                    )
2047                );
2048                let req_body = request.into_body().collect_bytes().await.unwrap();
2049                let json: serde_json::Value =
2050                    serde_json::from_slice(&req_body).expect("patch object is json");
2051                let statefulset: StatefulSet =
2052                    serde_json::from_value(json).expect("valid statefulset");
2053                assert_eq!(
2054                    statefulset.clone().spec.unwrap().replicas.unwrap(),
2055                    rg.replicas
2056                );
2057                let tls_hash_annotation = statefulset
2058                    .spec
2059                    .as_ref()
2060                    .unwrap()
2061                    .template
2062                    .metadata
2063                    .as_ref()
2064                    .unwrap()
2065                    .annotations
2066                    .as_ref()
2067                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2068                    .cloned();
2069                assert_eq!(tls_hash_annotation, expected_tls_hash);
2070                let response = serde_json::to_vec(&statefulset).unwrap();
2071                // pass through kanidm "patch accepted"
2072                send.send_response(Response::builder().body(Body::from(response)).unwrap());
2073            }
2074
2075            Ok(self)
2076        }
2077
2078        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2079            let (request, send) = self.0.next_request().await.expect("service not called");
2080            assert_eq!(request.method(), http::Method::PATCH);
2081            assert_eq!(
2082                request.uri().to_string(),
2083                format!(
2084                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2085                    kanidm.name_any()
2086                )
2087            );
2088
2089            let req_body = request.into_body().collect_bytes().await.unwrap();
2090            let json: serde_json::Value =
2091                serde_json::from_slice(&req_body).expect("patch object is json");
2092            let service: Service = serde_json::from_value(json).expect("valid service");
2093            let response = serde_json::to_vec(&service).unwrap();
2094            // pass through kanidm "patch accepted"
2095            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2096            Ok(self)
2097        }
2098
2099        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2100            let (request, send) = self.0.next_request().await.expect("service not called");
2101            assert_eq!(request.method(), http::Method::PATCH);
2102            assert_eq!(
2103                request.uri().to_string(),
2104                format!(
2105                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2106                    kanidm.name_any()
2107                )
2108            );
2109
2110            let req_body = request.into_body().collect_bytes().await.unwrap();
2111            let json: serde_json::Value =
2112                serde_json::from_slice(&req_body).expect("patch object is json");
2113            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2114            let response = serde_json::to_vec(&ingress).unwrap();
2115            // pass through kanidm "patch accepted"
2116            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2117            Ok(self)
2118        }
2119    }
2120
2121    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2122        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2123        let mock_client = Client::new(mock_service, "default");
2124        let stores = Stores {
2125            stateful_set_store: Writer::default().as_reader(),
2126            service_store: Writer::default().as_reader(),
2127            ingress_store: Writer::default().as_reader(),
2128            secret_store: Writer::default().as_reader(),
2129            http_route_store: Some(Writer::default().as_reader()),
2130            backend_tls_policy_store: Some(Writer::default().as_reader()),
2131            deployment_store: Writer::default().as_reader(),
2132            config_map_store: Writer::default().as_reader(),
2133        };
2134        let controller_id = "test";
2135
2136        // Create a test meter for metrics
2137        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2138        let meter = provider.meter("test");
2139        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2140
2141        let state = State::new(
2142            metrics,
2143            Writer::default().as_reader(),
2144            Writer::default().as_reader(),
2145            None,
2146        );
2147        let ctx = Arc::new(Context::new(
2148            state.to_context(mock_client, controller_id),
2149            stores,
2150        ));
2151        (ctx, ApiServerVerifier(handle))
2152    }
2153
2154    #[tokio::test]
2155    async fn kanidm_create() {
2156        let (testctx, fakeserver) = get_test_context();
2157        let kanidm = Kanidm::test();
2158        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2159        reconcile_kanidm(Arc::new(kanidm), testctx)
2160            .await
2161            .expect("reconciler");
2162        timeout_after_1s(mocksrv).await;
2163    }
2164
2165    #[tokio::test]
2166    async fn kanidm_create_with_two_replicas() {
2167        let (testctx, fakeserver) = get_test_context();
2168        let kanidm = Kanidm::test().with_replicas(2);
2169        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2170        reconcile_kanidm(Arc::new(kanidm), testctx)
2171            .await
2172            .expect("reconciler");
2173        timeout_after_1s(mocksrv).await;
2174    }
2175
2176    #[tokio::test]
2177    async fn kanidm_create_with_ingress() {
2178        let (testctx, fakeserver) = get_test_context();
2179        let kanidm = Kanidm::test().with_ingress();
2180        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2181        reconcile_kanidm(Arc::new(kanidm), testctx)
2182            .await
2183            .expect("reconciler");
2184        timeout_after_1s(mocksrv).await;
2185    }
2186
2187    #[tokio::test]
2188    async fn kanidm_create_with_ingress_with_two_replicas() {
2189        let (testctx, fakeserver) = get_test_context();
2190        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2191        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2192        reconcile_kanidm(Arc::new(kanidm), testctx)
2193            .await
2194            .expect("reconciler");
2195        timeout_after_1s(mocksrv).await;
2196    }
2197
2198    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2199        Arc::new(StatefulSet {
2200            metadata: ObjectMeta {
2201                name: Some(format!("{cluster}-default")),
2202                namespace: Some(namespace.to_string()),
2203                labels: Some(BTreeMap::from([(
2204                    CLUSTER_LABEL.to_string(),
2205                    cluster.to_string(),
2206                )])),
2207                ..Default::default()
2208            },
2209            spec: Some(StatefulSetSpec {
2210                template: PodTemplateSpec {
2211                    metadata: Some(ObjectMeta {
2212                        annotations: tls_hash.map(|h| {
2213                            BTreeMap::from([(
2214                                TLS_SECRET_HASH_ANNOTATION.to_string(),
2215                                h.to_string(),
2216                            )])
2217                        }),
2218                        ..Default::default()
2219                    }),
2220                    ..Default::default()
2221                },
2222                ..Default::default()
2223            }),
2224            ..Default::default()
2225        })
2226    }
2227
2228    #[test]
2229    fn test_previous_tls_secret_hash() {
2230        let kanidm = Kanidm::test();
2231
2232        // annotation carried over from an existing statefulset
2233        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2234        assert_eq!(
2235            previous_tls_secret_hash(&kanidm, stss),
2236            Some("abc123".to_string())
2237        );
2238
2239        // no annotation on the existing statefulset
2240        let stss = vec![statefulset_of("test", "default", None)];
2241        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2242
2243        // statefulsets from another cluster or namespace are ignored
2244        let stss = vec![
2245            statefulset_of("other", "default", Some("abc123")),
2246            statefulset_of("test", "other-ns", Some("abc123")),
2247        ];
2248        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2249    }
2250
2251    #[tokio::test]
2252    async fn kanidm_create_with_tls_secret() {
2253        let (testctx, fakeserver) = get_test_context();
2254        let kanidm = Kanidm::test();
2255        let secret = Secret {
2256            metadata: ObjectMeta {
2257                name: Some(kanidm.effective_tls_secret_name()),
2258                namespace: Some("default".to_string()),
2259                ..Default::default()
2260            },
2261            data: Some(BTreeMap::from([
2262                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2263                ("tls.key".to_string(), ByteString(b"key".to_vec())),
2264            ])),
2265            ..Default::default()
2266        };
2267        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2268            kanidm.clone(),
2269            Box::new(secret),
2270        ));
2271        reconcile_kanidm(Arc::new(kanidm), testctx)
2272            .await
2273            .expect("reconciler");
2274        timeout_after_1s(mocksrv).await;
2275    }
2276
2277    #[tokio::test]
2278    async fn replication_secret_gc_skips_non_replica_secrets() {
2279        use super::reconcile_replication_secrets;
2280        use super::secret::{SECRET_TYPE_LABEL, SecretType};
2281
2282        let kanidm = Kanidm::test();
2283        let status = kanidm.status.clone().unwrap_or_default();
2284
2285        let replica_cert_label =
2286            serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
2287
2288        let replica_cert_secret = Secret {
2289            metadata: ObjectMeta {
2290                name: Some(format!("{}-default-0-cert", kanidm.name_any())),
2291                namespace: Some("default".into()),
2292                labels: Some(BTreeMap::from([
2293                    (CLUSTER_LABEL.to_string(), kanidm.name_any()),
2294                    (SECRET_TYPE_LABEL.to_string(), replica_cert_label.clone()),
2295                ])),
2296                ..Default::default()
2297            },
2298            ..Default::default()
2299        };
2300
2301        let mail_sender_secret = Secret {
2302            metadata: ObjectMeta {
2303                name: Some(format!("{}-mail-sender-config", kanidm.name_any())),
2304                namespace: Some("default".into()),
2305                labels: Some(BTreeMap::from([(
2306                    CLUSTER_LABEL.to_string(),
2307                    kanidm.name_any(),
2308                )])),
2309                ..Default::default()
2310            },
2311            ..Default::default()
2312        };
2313
2314        let (mock_service, mut handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2315        let mock_client = Client::new(mock_service, "default");
2316
2317        let mut secret_writer = Writer::default();
2318        secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2319            replica_cert_secret.clone(),
2320        ));
2321        secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2322            mail_sender_secret.clone(),
2323        ));
2324
2325        let stores = Stores {
2326            stateful_set_store: Writer::default().as_reader(),
2327            service_store: Writer::default().as_reader(),
2328            ingress_store: Writer::default().as_reader(),
2329            secret_store: secret_writer.as_reader(),
2330            http_route_store: Some(Writer::default().as_reader()),
2331            backend_tls_policy_store: Some(Writer::default().as_reader()),
2332            deployment_store: Writer::default().as_reader(),
2333            config_map_store: Writer::default().as_reader(),
2334        };
2335
2336        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2337        let meter = provider.meter("test");
2338        let metrics = crate::metrics::Metrics::new(&meter, &["test"]);
2339        let state = State::new(
2340            metrics,
2341            Writer::default().as_reader(),
2342            Writer::default().as_reader(),
2343            None,
2344        );
2345        let ctx = Arc::new(Context::new(state.to_context(mock_client, "test"), stores));
2346
2347        let mock_srv = tokio::spawn(async move {
2348            let (request, send) = handle.next_request().await.expect("expected a request");
2349            assert_eq!(request.method(), http::Method::DELETE);
2350            let uri = request.uri().to_string();
2351            assert!(
2352                uri.contains("-default-0-cert"),
2353                "expected delete for replica cert secret, got {uri}"
2354            );
2355            let not_found = serde_json::json!({
2356                "kind": "Status",
2357                "apiVersion": "v1",
2358                "metadata": {},
2359                "status": "Failure",
2360                "message": "not found",
2361                "reason": "NotFound",
2362                "code": 404
2363            });
2364            send.send_response(
2365                Response::builder()
2366                    .status(404)
2367                    .body(Body::from(serde_json::to_vec(&not_found).unwrap()))
2368                    .unwrap(),
2369            );
2370
2371            if let Ok(Some((req, _))) =
2372                tokio::time::timeout(std::time::Duration::from_millis(200), handle.next_request())
2373                    .await
2374            {
2375                panic!("unexpected second request: {} {}", req.method(), req.uri());
2376            }
2377        });
2378
2379        let changed = reconcile_replication_secrets(Arc::new(kanidm), ctx, &status)
2380            .await
2381            .expect("reconcile succeeds despite 404 on delete");
2382        assert!(
2383            changed,
2384            "expected changed=true since a deprecated secret was found"
2385        );
2386
2387        tokio::time::timeout(std::time::Duration::from_secs(1), mock_srv)
2388            .await
2389            .expect("timeout waiting for mock server")
2390            .expect("mock server task succeeded");
2391    }
2392
2393    #[test]
2394    fn disable_upgrade_checks_allows_unchecked_version_status() {
2395        let mut kanidm = Kanidm::test();
2396        kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2397        kanidm.spec.disable_upgrade_checks = true;
2398        let status = KanidmStatus {
2399            version: Some(crate::kanidm::crd::KanidmVersionStatus {
2400                image_tag: "1.11.2".to_string(),
2401                upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Skipped,
2402                compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::NotChecked,
2403            }),
2404            ..Default::default()
2405        };
2406        assert!(kanidm.is_updatable(&status));
2407    }
2408
2409    #[test]
2410    fn disable_upgrade_checks_allows_previous_incompatible_status() {
2411        let mut kanidm = Kanidm::test();
2412        kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2413        kanidm.spec.disable_upgrade_checks = true;
2414        let status = KanidmStatus {
2415            version: Some(crate::kanidm::crd::KanidmVersionStatus {
2416                image_tag: "1.11.2".to_string(),
2417                upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Passed,
2418                compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::Incompatible,
2419            }),
2420            ..Default::default()
2421        };
2422        assert!(kanidm.is_updatable(&status));
2423    }
2424
2425    #[test]
2426    fn upgrade_checks_still_block_incompatible_versions_when_enabled() {
2427        let mut kanidm = Kanidm::test();
2428        kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2429        kanidm.spec.disable_upgrade_checks = false;
2430        let status = KanidmStatus {
2431            version: Some(crate::kanidm::crd::KanidmVersionStatus {
2432                image_tag: "1.11.2".to_string(),
2433                upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Passed,
2434                compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::Incompatible,
2435            }),
2436            ..Default::default()
2437        };
2438        assert!(!kanidm.is_updatable(&status));
2439    }
2440
2441    #[test]
2442    fn upgrade_checks_do_not_treat_skipped_checks_as_passed() {
2443        let mut kanidm = Kanidm::test();
2444        kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2445        kanidm.spec.disable_upgrade_checks = false;
2446        let status = KanidmStatus {
2447            version: Some(crate::kanidm::crd::KanidmVersionStatus {
2448                image_tag: "1.11.2".to_string(),
2449                upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Skipped,
2450                compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::NotChecked,
2451            }),
2452            ..Default::default()
2453        };
2454        assert!(!kanidm.is_updatable(&status));
2455    }
2456}