Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::{SECRET_TYPE_LABEL, SecretExt, SecretType};
20use self::service::ServiceExt;
21use self::statefulset::{
22    StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23    preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_backup_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
31use crate::kanidm::crd::{
32    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
33};
34use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
35use crate::telemetry;
36
37use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
38use kaniop_k8s_util::client::get_output;
39use kaniop_k8s_util::error::{Error, Result};
40use kaniop_k8s_util::parse::parse_semver;
41use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::fmt::Debug;
45use std::sync::{Arc, LazyLock};
46
47use futures::future::{TryJoinAll, join_all, try_join_all};
48use futures::stream::{self, StreamExt};
49use futures::try_join;
50use k8s_openapi::NamespaceResourceScope;
51use k8s_openapi::api::apps::v1::StatefulSet;
52use k8s_openapi::api::core::v1::{Pod, Secret, Service};
53use kube::Resource;
54use kube::ResourceExt;
55use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
56use kube::runtime::controller::Action;
57use kube::runtime::events::{Event, EventType};
58use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
59use serde::{Deserialize, Serialize};
60use tokio::time::{Duration, sleep};
61use tracing::{Span, debug, field, info, instrument, trace, warn};
62
63const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
64const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
65const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
66const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
67const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
68const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
69const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
70const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
71const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
72
73pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
74const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
75pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
76
77const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
78
79static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
80    BTreeMap::from([
81        (NAME_LABEL.to_string(), "kanidm".to_string()),
82        (
83            MANAGED_BY_LABEL.to_string(),
84            format!("kaniop-{CONTROLLER_ID}"),
85        ),
86    ])
87});
88
89async fn restart_statefulsets_after_replica_secret_update(
90    ctx: Arc<Context>,
91    sts_api: &Api<StatefulSet>,
92    namespace: &str,
93    statefulset_names: &[String],
94    sequential: bool,
95) -> Result<()> {
96    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
97    // Pods must restart after cert secret changes or replication keeps stale TLS material.
98    let restart_futures = statefulset_names
99        .iter()
100        .map(|sts_name| sts_api.restart(sts_name));
101
102    if sequential {
103        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
104        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
105            return Err(Error::kube_error(
106                "restart",
107                "StatefulSet",
108                namespace,
109                "replica certificate secret update statefulsets",
110                first_err,
111            ));
112        }
113    } else {
114        let results: Vec<_> = join_all(restart_futures).await;
115        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
116            return Err(Error::kube_error(
117                "restart",
118                "StatefulSet",
119                namespace,
120                "replica certificate secret update statefulsets",
121                first_err,
122            ));
123        }
124    }
125
126    let rollout_futures = statefulset_names
127        .iter()
128        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
129    try_join_all(rollout_futures).await?;
130
131    Ok(())
132}
133
134pub async fn reconcile_admins_secret(
135    kanidm: Arc<Kanidm>,
136    ctx: Arc<Context>,
137    status: &KanidmStatus,
138) -> Result<bool> {
139    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
140        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
141        kanidm.patch(&ctx, admins_secret).await?;
142        return Ok(true);
143    }
144    Ok(false)
145}
146
147pub async fn reconcile_replication_secrets(
148    kanidm: Arc<Kanidm>,
149    ctx: Arc<Context>,
150    status: &KanidmStatus,
151) -> Result<bool> {
152    let mut changed = false;
153    let expected_secret_names = if kanidm.is_replication_enabled() {
154        status
155            .replica_statuses
156            .iter()
157            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
158            .collect::<Vec<_>>()
159    } else {
160        Vec::new()
161    };
162    let replica_cert_label =
163        serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
164    let deprecated_secrets = ctx
165        .stores
166        .secret_store
167        .state()
168        .into_iter()
169        .filter(|secret| {
170            secret.namespace() == kanidm.namespace()
171                && kanidm.admins_secret_name() != secret.name_any()
172                && !expected_secret_names.contains(&secret.name_any())
173                && secret.metadata.labels.as_ref().is_some_and(|l| {
174                    l.get(SECRET_TYPE_LABEL) == Some(&replica_cert_label)
175                        && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
176                })
177        })
178        .collect::<Vec<_>>();
179    if !deprecated_secrets.is_empty() {
180        changed = true;
181    }
182    let secret_delete_future = deprecated_secrets
183        .iter()
184        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
185        .collect::<TryJoinAll<_>>();
186    try_join!(secret_delete_future)?;
187
188    if kanidm.is_replication_enabled() {
189        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
190        let statefulset_names = status
191            .replica_statuses
192            .iter()
193            .map(|rs| rs.statefulset_name.clone())
194            .collect::<BTreeSet<_>>()
195            .into_iter()
196            .collect::<Vec<_>>();
197
198        let has_pending = status
199            .replica_statuses
200            .iter()
201            .any(|rs| rs.state == KanidmReplicaState::Pending);
202
203        let sts_api =
204            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
205
206        if has_pending {
207            changed = true;
208            stream::iter(
209                status
210                    .replica_statuses
211                    .iter()
212                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
213            )
214            .then(|rs| async {
215                let secret = kanidm
216                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
217                    .await?;
218                kanidm.patch(&ctx, secret).await
219            })
220            .collect::<Vec<_>>()
221            .await
222            .into_iter()
223            .collect::<Result<Vec<_>>>()?;
224
225            let restart_futures = status
226                .replica_statuses
227                .iter()
228                .map(|rs| sts_api.restart(&rs.statefulset_name));
229
230            if has_single_replica {
231                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
232                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
233                    return Err(Error::kube_error(
234                        "restart",
235                        "StatefulSet",
236                        kanidm.get_namespace(),
237                        "pending replica statefulsets",
238                        first_err,
239                    ));
240                }
241            } else {
242                let results: Vec<_> = join_all(restart_futures).await;
243                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
244                    return Err(Error::kube_error(
245                        "restart",
246                        "StatefulSet",
247                        kanidm.get_namespace(),
248                        "pending replica statefulsets",
249                        first_err,
250                    ));
251                }
252            }
253        }
254
255        let has_certificate_host_invalid = status
256            .replica_statuses
257            .iter()
258            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
259
260        let has_certificate_expiring = status
261            .replica_statuses
262            .iter()
263            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
264
265        let has_non_ready_replicas = status
266            .replica_statuses
267            .iter()
268            .any(|rs| rs.state != KanidmReplicaState::Ready);
269
270        let host_invalid_replicas: Vec<_> = status
271            .replica_statuses
272            .iter()
273            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
274            .collect();
275
276        let cert_expiring_replicas: Vec<_> = status
277            .replica_statuses
278            .iter()
279            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
280            .collect();
281
282        if !host_invalid_replicas.is_empty() {
283            changed = true;
284            let restart_futures = host_invalid_replicas
285                .iter()
286                .map(|rs| sts_api.restart(&rs.statefulset_name));
287
288            if has_single_replica {
289                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
290                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
291                    return Err(Error::kube_error(
292                        "restart",
293                        "StatefulSet",
294                        kanidm.get_namespace(),
295                        "certificate host invalid statefulsets",
296                        first_err,
297                    ));
298                }
299            } else {
300                let results: Vec<_> = join_all(restart_futures).await;
301                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
302                    return Err(Error::kube_error(
303                        "restart",
304                        "StatefulSet",
305                        kanidm.get_namespace(),
306                        "certificate host invalid statefulsets",
307                        first_err,
308                    ));
309                }
310            }
311
312            let restarted_sts_names: Vec<_> = host_invalid_replicas
313                .iter()
314                .map(|rs| rs.statefulset_name.clone())
315                .collect::<BTreeSet<_>>()
316                .into_iter()
317                .collect();
318
319            let namespace = kanidm.get_namespace();
320            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
321                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
322            });
323            try_join_all(rollout_futures).await?;
324
325            let renew_futures = host_invalid_replicas
326                .iter()
327                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
328            let results: Vec<_> = join_all(renew_futures).await;
329            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
330                return Err(first_err);
331            }
332
333            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
334
335            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
336                let cert = show_replica_cert_with_retries(
337                    &kanidm,
338                    ctx.clone(),
339                    &rs.pod_name,
340                    CERT_SHOW_RETRY_ATTEMPTS,
341                )
342                .await?;
343                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
344                kanidm.patch(&ctx, secret.clone()).await
345            });
346            try_join_all(cert_futures).await?;
347
348            restart_statefulsets_after_replica_secret_update(
349                ctx.clone(),
350                &sts_api,
351                &kanidm.get_namespace(),
352                &statefulset_names,
353                has_single_replica,
354            )
355            .await?;
356
357            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
358        }
359
360        if !cert_expiring_replicas.is_empty() {
361            changed = true;
362            let renew_futures = cert_expiring_replicas
363                .iter()
364                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
365            let results: Vec<_> = join_all(renew_futures).await;
366            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
367                return Err(first_err);
368            }
369
370            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
371
372            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
373                let cert = show_replica_cert_with_retries(
374                    &kanidm,
375                    ctx.clone(),
376                    &rs.pod_name,
377                    CERT_SHOW_RETRY_ATTEMPTS,
378                )
379                .await?;
380                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
381                kanidm.patch(&ctx, secret.clone()).await
382            });
383            try_join_all(cert_futures).await?;
384
385            restart_statefulsets_after_replica_secret_update(
386                ctx.clone(),
387                &sts_api,
388                &kanidm.get_namespace(),
389                &statefulset_names,
390                has_single_replica,
391            )
392            .await?;
393
394            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
395        }
396
397        if has_non_ready_replicas
398            && !has_pending
399            && !has_certificate_host_invalid
400            && !has_certificate_expiring
401        {
402            changed = true;
403            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
404
405            let restart_futures = status
406                .replica_statuses
407                .iter()
408                .map(|rs| sts_api.restart(&rs.statefulset_name));
409
410            if has_single_replica {
411                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
412                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
413                    return Err(Error::kube_error(
414                        "restart",
415                        "StatefulSet",
416                        kanidm.get_namespace(),
417                        "replica statefulsets",
418                        first_err,
419                    ));
420                }
421            } else {
422                let results: Vec<_> = join_all(restart_futures).await;
423                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
424                    return Err(Error::kube_error(
425                        "restart",
426                        "StatefulSet",
427                        kanidm.get_namespace(),
428                        "replica statefulsets",
429                        first_err,
430                    ));
431                }
432            }
433        }
434    }
435
436    Ok(changed)
437}
438
439#[instrument(skip(ctx, kanidm))]
440pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
441    let trace_id = telemetry::get_trace_id();
442    Span::current().record("trace_id", field::display(&trace_id));
443    let _timer = ctx
444        .kaniop_ctx
445        .metrics
446        .reconcile_count_and_measure(&trace_id);
447    info!("reconciling Kanidm");
448
449    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
450        debug!(%e, "failed to reconcile status");
451        ctx.kaniop_ctx.metrics.status_update_errors_inc();
452        e
453    })?;
454    let kanidm_api: Api<Kanidm> =
455        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
456    let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
457    let outcome_clone = outcome.clone();
458    let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
459        let outcome = outcome_clone.clone();
460        let ctx = ctx.clone();
461        let status = status.clone();
462        async move {
463            match event {
464                Finalizer::Apply(kanidm) => {
465                    let (action, changed) = reconcile(kanidm, ctx, status).await?;
466                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
467                    Ok(action)
468                }
469                Finalizer::Cleanup(kanidm) => {
470                    let (action, changed) = cleanup(kanidm, ctx).await?;
471                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
472                    Ok(action)
473                }
474            }
475        }
476    })
477    .await
478    .or_else(|e| match e {
479        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
480            debug!("resource already removed during finalizer cleanup");
481            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
482        }
483        _ => Err(Error::FinalizerError(
484            "failed on kanidm account finalizer".to_string(),
485            Box::new(e),
486        )),
487    })?;
488    let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
489    Ok((action, changed))
490}
491
492/// Hash of the TLS secret content, injected as a pod template annotation so
493/// certificate renewals roll the statefulsets. `None` if the secret does not
494/// exist yet.
495async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
496    let secret_name = kanidm.effective_tls_secret_name();
497    let secret_api =
498        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
499    let secret = secret_api
500        .get_opt(&secret_name)
501        .await
502        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
503    Ok(secret.as_ref().map(hash_secret_data))
504}
505
506/// Previous TLS secret hash annotation from the existing statefulsets.
507///
508/// Used when the TLS secret is missing: dropping the annotation would roll
509/// the pods onto a missing volume and leave them stuck in ContainerCreating.
510fn previous_tls_secret_hash(
511    kanidm: &Kanidm,
512    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
513) -> Option<String> {
514    stateful_sets
515        .into_iter()
516        .filter(|sts| {
517            sts.namespace() == kanidm.namespace()
518                && sts
519                    .metadata
520                    .labels
521                    .as_ref()
522                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
523        })
524        .find_map(|sts| {
525            sts.spec
526                .as_ref()?
527                .template
528                .metadata
529                .as_ref()?
530                .annotations
531                .as_ref()?
532                .get(TLS_SECRET_HASH_ANNOTATION)
533                .cloned()
534        })
535}
536
537fn is_headless_service(service: &Service) -> bool {
538    service
539        .spec
540        .as_ref()
541        .and_then(|spec| spec.cluster_ip.as_deref())
542        == Some("None")
543}
544
545fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
546    if is_headless_service(desired) != is_headless_service(current) {
547        return;
548    }
549    let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
550    else {
551        return;
552    };
553    if desired_spec.cluster_ip.is_none() {
554        desired_spec.cluster_ip = current_spec.cluster_ip.clone();
555    }
556    if desired_spec.cluster_ips.is_none() {
557        desired_spec.cluster_ips = current_spec.cluster_ips.clone();
558    }
559}
560
561async fn wait_for_statefulset_deletion(
562    client: kube::Client,
563    namespace: &str,
564    name: &str,
565) -> Result<()> {
566    let api = Api::<StatefulSet>::namespaced(client, namespace);
567    let start = std::time::Instant::now();
568    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
569    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
570
571    loop {
572        match api.get_opt(name).await {
573            Ok(None) => return Ok(()),
574            Ok(Some(_)) => {}
575            Err(e) => {
576                return Err(Error::kube_error(
577                    "wait for deletion of",
578                    "StatefulSet",
579                    namespace,
580                    name,
581                    e,
582                ));
583            }
584        }
585
586        if start.elapsed() >= timeout {
587            return Err(Error::ReceiveOutput(format!(
588                "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
589            )));
590        }
591        sleep(poll_interval).await;
592    }
593}
594
595async fn reconcile_service(
596    kanidm: &Kanidm,
597    ctx: &Context,
598    mut desired: Service,
599) -> Result<Service> {
600    let namespace = kanidm.get_namespace();
601    let name = desired.name_any();
602    let desired_headless = is_headless_service(&desired);
603
604    if let Some(cached) = ctx
605        .stores
606        .service_store
607        .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
608    {
609        if is_headless_service(cached.as_ref()) == desired_headless {
610            preserve_defaulted_service_fields(&mut desired, cached.as_ref());
611        } else {
612            let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
613            let current = service_api
614                .get_opt(&name)
615                .await
616                .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
617
618            if let Some(current) = current {
619                let current_headless = is_headless_service(&current);
620                if current_headless != desired_headless {
621                    info!(
622                        resource.name = &name,
623                        resource.namespace = &namespace,
624                        current_headless,
625                        desired_headless,
626                        "recreating Service to change headless mode"
627                    );
628                    kanidm.delete(ctx, &current).await?;
629                    ctx.kaniop_ctx
630                        .metrics
631                        .reconcile_deploy_delete_create_inc("Service", "headless_transition");
632                    wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
633                        .await?;
634                } else {
635                    preserve_defaulted_service_fields(&mut desired, &current);
636                }
637            }
638        }
639    }
640
641    kanidm.patch(ctx, desired).await
642}
643
644async fn wait_for_service_deletion(
645    client: kube::Client,
646    namespace: &str,
647    name: &str,
648) -> Result<()> {
649    let api = Api::<Service>::namespaced(client, namespace);
650    let start = std::time::Instant::now();
651    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
652    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
653
654    loop {
655        match api.get_opt(name).await {
656            Ok(None) => return Ok(()),
657            Ok(Some(_)) => {}
658            Err(e) => {
659                return Err(Error::kube_error(
660                    "wait for deletion of",
661                    "Service",
662                    namespace,
663                    name,
664                    e,
665                ));
666            }
667        }
668
669        if start.elapsed() >= timeout {
670            return Err(Error::ReceiveOutput(format!(
671                "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
672            )));
673        }
674        sleep(poll_interval).await;
675    }
676}
677
678async fn validate_statefulset_replacement(
679    api: &Api<StatefulSet>,
680    desired: &StatefulSet,
681    namespace: &str,
682    name: &str,
683) -> Result<()> {
684    let mut candidate = desired.clone();
685    candidate.metadata.name = None;
686    candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
687    candidate.metadata.namespace = None;
688    candidate.metadata.resource_version = None;
689    candidate.metadata.uid = None;
690    candidate.metadata.creation_timestamp = None;
691    candidate.metadata.generation = None;
692    candidate.metadata.managed_fields = None;
693    candidate.metadata.deletion_timestamp = None;
694    candidate.metadata.deletion_grace_period_seconds = None;
695    candidate.status = None;
696
697    api.create(
698        &PostParams {
699            dry_run: true,
700            ..Default::default()
701        },
702        &candidate,
703    )
704    .await
705    .map(|_| ())
706    .map_err(|e| {
707        Error::kube_error(
708            "validate replacement for",
709            "StatefulSet",
710            namespace,
711            name,
712            e,
713        )
714    })
715}
716
717async fn reconcile_statefulset(
718    kanidm: Arc<Kanidm>,
719    ctx: Arc<Context>,
720    desired: StatefulSet,
721) -> Result<StatefulSet> {
722    let namespace = kanidm.namespace().unwrap();
723    let name = desired.name_any();
724    let cached_current = ctx.stores.stateful_set_store.find(|current| {
725        current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
726    });
727
728    // On a cold reflector cache, keep the normal create/update path GET-free. If the optimistic
729    // apply fails with HTTP 422, the object may already exist with an immutable conflict that the
730    // cache has not observed yet. Only 422 errors enter the live classification/recreate recovery
731    // path; 429, 5xx, 403, and transport errors are returned immediately and non-destructively.
732    let cache_miss_422_error = if let Some(cached_current) = cached_current {
733        let mut cached_desired = desired.clone();
734        preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
735        if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
736            == StatefulSetApplyStrategy::Apply
737        {
738            return kanidm.apply(&ctx, cached_desired).await;
739        }
740        None
741    } else {
742        match kanidm.apply(&ctx, desired.clone()).await {
743            Ok(applied) => return Ok(applied),
744            Err(error) => {
745                // Only HTTP 422 can indicate an immutable-field conflict that is safe to
746                // recover from via delete/recreate. Arbitrary failures must not trigger
747                // destruction.
748                if is_kube_422(&error) {
749                    Some(error)
750                } else {
751                    return Err(error);
752                }
753            }
754        }
755    };
756
757    // Never make a destructive decision from reflector state or an SSA error alone. Re-read the
758    // live object and classify it immediately before deletion; another actor or an earlier
759    // reconcile may already have converged the immutable fields while the cache was catching up.
760    let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
761    let Some(live_current) = api
762        .get_opt(&name)
763        .await
764        .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
765    else {
766        return match cache_miss_422_error {
767            Some(error) => Err(error),
768            None => kanidm.apply(&ctx, desired).await,
769        };
770    };
771
772    // Start from the original desired object again. Defaults copied from a stale reflector entry
773    // must not survive into the live re-check and become the reason for a recreation.
774    let mut live_desired = desired.clone();
775    preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
776    match classify_statefulset_change(&live_current, &live_desired) {
777        StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
778            Some(error) => Err(error),
779            None => kanidm.apply(&ctx, live_desired).await,
780        },
781        StatefulSetApplyStrategy::Recreate { .. } => {
782            // An update 422 can combine an immutable conflict with independently-invalid desired
783            // state. Validate the replacement as a dry-run CREATE before deleting the live object,
784            // so create-time validation runs without the existing object's immutable-update check.
785            validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
786
787            // Re-read and reclassify immediately before destruction. Another actor or an earlier
788            // reconcile may have changed the live immutable fields or PVC-retention semantics.
789            let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
790                Error::kube_error(
791                    "re-read before deletion",
792                    "StatefulSet",
793                    &namespace,
794                    &name,
795                    e,
796                )
797            })?
798            else {
799                return kanidm.apply(&ctx, live_desired).await;
800            };
801            let mut latest_desired = live_desired.clone();
802            preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
803
804            match classify_statefulset_change(&latest_current, &latest_desired) {
805                StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
806                StatefulSetApplyStrategy::Recreate { immutable_fields } => {
807                    // Validate the replacement StatefulSet via a dry-run CREATE before deleting
808                    // the live object. This prevents deleting on a mixed 422 (immutable conflict
809                    // + independently-invalid desired state) where the replacement would still
810                    // be rejected by the API.
811                    validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
812                        .await?;
813
814                    info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
815                    kanidm.delete(&ctx, &latest_current).await?;
816                    ctx.kaniop_ctx
817                        .metrics
818                        .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
819                    wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
820                        .await?;
821                    kanidm.apply(&ctx, latest_desired).await
822                }
823            }
824        }
825    }
826}
827
828/// Return true only when the error structurally wraps a Kubernetes HTTP 422 response.
829fn is_kube_422(error: &Error) -> bool {
830    matches!(
831        error,
832        Error::KubeError(_, cause)
833            if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
834    )
835}
836
837#[cfg(test)]
838mod statefulset_recreate_recovery_tests {
839    use super::{Error, is_kube_422};
840
841    fn kube_api_error(code: u16) -> Error {
842        Error::KubeError(
843            "test error".to_string(),
844            Box::new(kube::Error::Api(Box::new(kube::error::Status {
845                code,
846                message: "test".to_string(),
847                reason: "test".to_string(),
848                ..Default::default()
849            }))),
850        )
851    }
852
853    #[test]
854    fn recreate_recovery_accepts_only_422() {
855        assert!(is_kube_422(&kube_api_error(422)));
856        assert!(!is_kube_422(&kube_api_error(403)));
857        assert!(!is_kube_422(&kube_api_error(429)));
858        assert!(!is_kube_422(&kube_api_error(500)));
859    }
860
861    #[test]
862    fn recreate_recovery_rejects_429_too_many_requests() {
863        assert!(!is_kube_422(&kube_api_error(429)));
864    }
865
866    #[test]
867    fn recreate_recovery_rejects_5xx_server_errors() {
868        assert!(!is_kube_422(&kube_api_error(500)));
869        assert!(!is_kube_422(&kube_api_error(502)));
870        assert!(!is_kube_422(&kube_api_error(503)));
871        assert!(!is_kube_422(&kube_api_error(504)));
872    }
873
874    #[test]
875    fn recreate_recovery_rejects_400_and_403() {
876        assert!(!is_kube_422(&kube_api_error(400)));
877        assert!(!is_kube_422(&kube_api_error(403)));
878    }
879}
880
881async fn reconcile(
882    kanidm: Arc<Kanidm>,
883    ctx: Arc<Context>,
884    status: KanidmStatus,
885) -> Result<(Action, bool)> {
886    if kanidm
887        .annotations()
888        .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
889    {
890        ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
891        return Ok((Action::requeue(Duration::from_secs(5)), false));
892    }
893
894    let mut changed = false;
895    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
896    let replication_secret_futures =
897        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
898
899    let sts_to_delete = {
900        let expected_sts_names = kanidm
901            .spec
902            .replica_groups
903            .iter()
904            .map(|rg| kanidm.statefulset_name(&rg.name))
905            .collect::<Vec<_>>();
906        ctx.stores
907            .stateful_set_store
908            .state()
909            .into_iter()
910            .filter(|sts| {
911                sts.namespace() == kanidm.namespace()
912                    && sts
913                        .metadata
914                        .labels
915                        .as_ref()
916                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
917                    && !expected_sts_names.contains(&sts.name_any())
918            })
919            .collect::<Vec<_>>()
920    };
921    if !sts_to_delete.is_empty() {
922        changed = true;
923    }
924    let sts_delete_futures = sts_to_delete
925        .iter()
926        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
927        .collect::<TryJoinAll<_>>();
928
929    let sts_futures = match kanidm.is_updatable(&status) {
930        true => {
931            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
932                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
933            });
934
935            let namespace = kanidm.get_namespace();
936            let schedule_api: Api<KanidmBackupSchedule> =
937                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
938            let schedules = schedule_api
939                .list(&Default::default())
940                .await
941                .map_err(|e| {
942                    Error::KubeError(
943                        "failed to list KanidmBackupSchedules for backup configuration resolution"
944                            .to_string(),
945                        Box::new(e),
946                    )
947                })?
948                .items;
949
950            let repo_api: Api<KanidmBackupRepository> =
951                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
952            let repositories = repo_api
953                .list(&Default::default())
954                .await
955                .map_err(|e| {
956                    Error::KubeError(
957                        "failed to list KanidmBackupRepositories for backup configuration resolution"
958                            .to_string(),
959                        Box::new(e),
960                    )
961                })?
962                .items;
963            let backup_config = resolve_backup_config(&kanidm, &schedules, &repositories);
964
965            kanidm
966                .spec
967                .replica_groups
968                .iter()
969                .map(|rg| {
970                    let sts = kanidm.create_statefulset(
971                        rg,
972                        tls_secret_hash.as_deref(),
973                        backup_config.as_ref(),
974                    )?;
975                    Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
976                })
977                .collect::<Result<TryJoinAll<_>, _>>()?
978        }
979        false => {
980            let note = match status.version.as_ref() {
981                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
982                    format!(
983                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
984                        v.image_tag,
985                        crate::version::KANIDM_CLIENT_VERSION
986                    )
987                }
988                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
989            };
990            let _ignore_error = ctx
991                .kaniop_ctx
992                .recorder
993                .publish(
994                    &Event {
995                        type_: EventType::Warning,
996                        reason: "UpgradeBlocked".to_string(),
997                        note: Some(note),
998                        action: "ReconcileStatefulSet".to_string(),
999                        secondary: None,
1000                    },
1001                    &kanidm.object_ref(&()),
1002                )
1003                .await
1004                .map_err(|e| {
1005                    warn!(%e, "failed to publish KanidmError event");
1006                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
1007                });
1008            try_join_all([])
1009        }
1010    };
1011    let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1012
1013    let deprecated_rg_svcs = {
1014        let expected_rg_svcs_names = kanidm
1015            .spec
1016            .replica_groups
1017            .iter()
1018            .filter(|rg| rg.services.is_some())
1019            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1020            .collect::<Vec<_>>();
1021        ctx.stores
1022            .service_store
1023            .state()
1024            .into_iter()
1025            .filter(|svc| {
1026                let labels = svc.metadata.labels.as_ref();
1027                svc.namespace() == kanidm.namespace()
1028                    && !expected_rg_svcs_names.contains(&svc.name_any())
1029                    && labels.is_some_and(|l| {
1030                        l.contains_key(REPLICA_LABEL)
1031                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1032                    })
1033            })
1034            .collect::<Vec<_>>()
1035    };
1036    if !deprecated_rg_svcs.is_empty() {
1037        changed = true;
1038    }
1039    let rg_svcs_delete_futures = deprecated_rg_svcs
1040        .iter()
1041        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1042        .collect::<TryJoinAll<_>>();
1043
1044    let rg_services_futures = kanidm
1045        .spec
1046        .replica_groups
1047        .iter()
1048        .filter(|rg| rg.services.is_some())
1049        .flat_map(|rg| {
1050            (0..rg.replicas).map(|i| {
1051                let mut svc = kanidm.create_replica_group_service(rg, i);
1052                if let Some(current) = ctx.stores.service_store.find(|current| {
1053                    current.namespace() == kanidm.namespace()
1054                        && current.name_any() == svc.name_any()
1055                }) {
1056                    preserve_defaulted_service_fields(&mut svc, current.as_ref());
1057                }
1058                kanidm.patch(&ctx, svc)
1059            })
1060        })
1061        .collect::<TryJoinAll<_>>();
1062
1063    let deprecated_ingresses = {
1064        let names = [
1065            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1066            kanidm.generate_region_ingress_name(),
1067        ]
1068        .into_iter()
1069        .flatten()
1070        .collect::<Vec<_>>();
1071        ctx.stores
1072            .ingress_store
1073            .state()
1074            .into_iter()
1075            .filter(|ing| {
1076                ing.namespace() == kanidm.namespace()
1077                    && !names.contains(&ing.name_any())
1078                    && ing.metadata.labels == Some(kanidm.generate_labels())
1079            })
1080            .collect::<Vec<_>>()
1081    };
1082    if !deprecated_ingresses.is_empty() {
1083        changed = true;
1084    }
1085    let ingress_delete_futures = deprecated_ingresses
1086        .iter()
1087        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1088        .collect::<TryJoinAll<_>>();
1089
1090    let ingress_futures = kanidm
1091        .create_ingress()
1092        .into_iter()
1093        .chain(kanidm.create_region_ingress())
1094        .map(|ingress| kanidm.patch(&ctx, ingress))
1095        .collect::<TryJoinAll<_>>();
1096
1097    let deprecated_http_routes = match &ctx.stores.http_route_store {
1098        Some(store) => {
1099            let expected_names = kanidm
1100                .spec
1101                .gateway
1102                .as_ref()
1103                .map(|_| vec![kanidm.name_any()])
1104                .unwrap_or_default();
1105            let routes: Vec<_> = store
1106                .state()
1107                .into_iter()
1108                .filter(|route| {
1109                    route.namespace() == kanidm.namespace()
1110                        && !expected_names.contains(&route.name_any())
1111                        && route.metadata.labels == Some(kanidm.generate_labels())
1112                })
1113                .collect();
1114            if !routes.is_empty() {
1115                changed = true;
1116            }
1117            routes
1118        }
1119        None => Vec::new(),
1120    };
1121    let http_route_delete_futures = deprecated_http_routes
1122        .iter()
1123        .map(|route| kanidm.delete(&ctx, route.as_ref()))
1124        .collect::<TryJoinAll<_>>();
1125
1126    let http_route_futures = if ctx.stores.http_route_store.is_some() {
1127        kanidm
1128            .create_http_route()
1129            .into_iter()
1130            .map(|route| kanidm.patch(&ctx, route))
1131            .collect::<TryJoinAll<_>>()
1132    } else {
1133        try_join_all(Vec::new())
1134    };
1135
1136    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1137        Some(store) => {
1138            let expected_names = kanidm
1139                .spec
1140                .gateway
1141                .as_ref()
1142                .and_then(|g| g.backend_tls_policy.as_ref())
1143                .map(|_| vec![kanidm.name_any()])
1144                .unwrap_or_default();
1145            let policies: Vec<_> = store
1146                .state()
1147                .into_iter()
1148                .filter(|policy| {
1149                    policy.namespace() == kanidm.namespace()
1150                        && !expected_names.contains(&policy.name_any())
1151                        && policy.metadata.labels == Some(kanidm.generate_labels())
1152                })
1153                .collect();
1154            if !policies.is_empty() {
1155                changed = true;
1156            }
1157            policies
1158        }
1159        None => Vec::new(),
1160    };
1161    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1162        .iter()
1163        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1164        .collect::<TryJoinAll<_>>();
1165
1166    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1167        kanidm
1168            .create_backend_tls_policy()
1169            .into_iter()
1170            .map(|policy| kanidm.patch(&ctx, policy))
1171            .collect::<TryJoinAll<_>>()
1172    } else {
1173        try_join_all(Vec::new())
1174    };
1175
1176    let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1177        sts_delete_futures,
1178        admin_secret_future,
1179        replication_secret_futures,
1180        sts_futures,
1181        service_future,
1182        rg_svcs_delete_futures,
1183        rg_services_futures,
1184        ingress_delete_futures,
1185        ingress_futures,
1186        http_route_delete_futures,
1187        http_route_futures,
1188        backend_tls_policy_delete_futures,
1189        backend_tls_policy_futures
1190    )?;
1191
1192    if admin_secret_changed || replication_secrets_changed {
1193        changed = true;
1194    }
1195
1196    if is_kanidm_available(status.clone()) {
1197        let namespace = kanidm.namespace().unwrap();
1198        let name = kanidm.name_any();
1199        match crate::controller::kanidm::KanidmClients::create_client(
1200            &namespace,
1201            &name,
1202            crate::controller::kanidm::KanidmUser::Admin,
1203            ctx.kaniop_ctx.client.clone(),
1204        )
1205        .await
1206        {
1207            Ok(system_client) => {
1208                if let Err(e) =
1209                    reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1210                {
1211                    warn!(%e, "failed to reconcile domain appearance");
1212                }
1213            }
1214            Err(e) => {
1215                warn!(%e, "failed to create admin client for domain appearance reconciliation");
1216            }
1217        }
1218
1219        match crate::controller::kanidm::KanidmClients::create_client(
1220            &namespace,
1221            &name,
1222            crate::controller::kanidm::KanidmUser::IdmAdmin,
1223            ctx.kaniop_ctx.client.clone(),
1224        )
1225        .await
1226        {
1227            Ok(kanidm_client) => {
1228                let (mail_sender_status, mail_sender_mutated) =
1229                    reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1230                        .await
1231                        .unwrap_or_else(|e| {
1232                            warn!(%e, "failed to reconcile mail sender");
1233                            (None, false)
1234                        });
1235                if mail_sender_mutated {
1236                    changed = true;
1237                }
1238                let current_mail_sender_status =
1239                    kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1240                if mail_sender_status != current_mail_sender_status {
1241                    let kanidm_api: Api<Kanidm> =
1242                        Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1243                    let status_patch = serde_json::json!({
1244                        "status": {
1245                            "mailSender": mail_sender_status
1246                        }
1247                    });
1248                    if let Err(e) = kanidm_api
1249                        .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1250                        .await
1251                    {
1252                        warn!(%e, "failed to patch Kanidm/status for mail sender");
1253                    }
1254                }
1255            }
1256            Err(e) => {
1257                warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1258            }
1259        }
1260    }
1261
1262    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1263}
1264
1265async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1266    debug!("cleanup");
1267
1268    let mut changed = false;
1269    let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1270    if mail_resources_cleaned {
1271        changed = true;
1272    }
1273
1274    let namespace = kanidm.namespace().unwrap();
1275    let name = kanidm.name_any();
1276
1277    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1278        &namespace,
1279        &name,
1280        crate::controller::kanidm::KanidmUser::IdmAdmin,
1281        ctx.kaniop_ctx.client.clone(),
1282    )
1283    .await
1284    {
1285        let kanidm_cleanup_done =
1286            cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1287        if kanidm_cleanup_done {
1288            changed = true;
1289        }
1290    }
1291
1292    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1293    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1294}
1295
1296async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1297    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1298    let start = std::time::Instant::now();
1299    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1300    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1301
1302    loop {
1303        let sts = sts_api.get(sts_name).await.map_err(|e| {
1304            Error::KubeError(
1305                format!("failed to get StatefulSet {namespace}/{sts_name}"),
1306                Box::new(e),
1307            )
1308        })?;
1309
1310        if sts.metadata.deletion_timestamp.is_some() {
1311            return Err(Error::ReceiveOutput(format!(
1312                "StatefulSet {namespace}/{sts_name} is being deleted"
1313            )));
1314        }
1315
1316        let spec = sts.spec.as_ref().ok_or_else(|| {
1317            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1318        })?;
1319
1320        let status = match sts.status {
1321            Some(ref s) => s,
1322            None => {
1323                if start.elapsed() >= timeout {
1324                    return Err(Error::ReceiveOutput(format!(
1325                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1326                    )));
1327                }
1328                sleep(poll_interval).await;
1329                continue;
1330            }
1331        };
1332
1333        let replicas = spec.replicas.unwrap_or(0);
1334
1335        if replicas == 0 {
1336            return Ok(());
1337        }
1338
1339        let updated_replicas = status.updated_replicas.unwrap_or(0);
1340        let current_revision = status.current_revision.as_deref().unwrap_or("");
1341        let update_revision = status.update_revision.as_deref().unwrap_or("");
1342        let ready_replicas = status.ready_replicas.unwrap_or(0);
1343
1344        if updated_replicas == replicas
1345            && current_revision == update_revision
1346            && ready_replicas == replicas
1347        {
1348            return Ok(());
1349        }
1350
1351        if start.elapsed() >= timeout {
1352            return Err(Error::ReceiveOutput(format!(
1353                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1354            )));
1355        }
1356
1357        sleep(poll_interval).await;
1358    }
1359}
1360
1361async fn show_replica_cert_with_retries(
1362    kanidm: &Kanidm,
1363    ctx: Arc<Context>,
1364    pod_name: &str,
1365    max_attempts: u32,
1366) -> std::result::Result<String, Error> {
1367    let mut last_error = None;
1368    for attempt in 0..max_attempts {
1369        if attempt > 0 {
1370            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1371        }
1372        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1373            Ok(cert) => return Ok(cert),
1374            Err(e) => {
1375                info!(
1376                    "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1377                    attempt + 1,
1378                    max_attempts
1379                );
1380                last_error = Some(e);
1381            }
1382        }
1383    }
1384    Err(last_error.unwrap_or_else(|| {
1385        Error::ReceiveOutput(format!(
1386            "failed to get certificate for {pod_name} after {max_attempts} attempts"
1387        ))
1388    }))
1389}
1390
1391impl Kanidm {
1392    // Convenience methods that handle context and operator name
1393    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1394    where
1395        K: Resource<Scope = NamespaceResourceScope>
1396            + Serialize
1397            + Clone
1398            + std::fmt::Debug
1399            + for<'de> Deserialize<'de>,
1400        <K as kube::Resource>::DynamicType: Default,
1401        <K as Resource>::Scope: std::marker::Sized,
1402    {
1403        self.kube_delete(
1404            ctx.kaniop_ctx.client.clone(),
1405            &ctx.kaniop_ctx.metrics,
1406            resource,
1407        )
1408        .await
1409    }
1410
1411    /// Server-side apply without any generic delete/recreate fallback.
1412    /// Resource-specific reconcilers use this when they own lifecycle decisions.
1413    pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1414    where
1415        K: Resource<Scope = NamespaceResourceScope>
1416            + Serialize
1417            + Clone
1418            + std::fmt::Debug
1419            + for<'de> Deserialize<'de>,
1420        <K as kube::Resource>::DynamicType: Default,
1421        <K as Resource>::Scope: std::marker::Sized,
1422    {
1423        self.kube_apply(
1424            ctx.kaniop_ctx.client.clone(),
1425            resource,
1426            KANIDM_OPERATOR_NAME,
1427        )
1428        .await
1429    }
1430
1431    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1432    where
1433        K: Resource<Scope = NamespaceResourceScope>
1434            + Serialize
1435            + Clone
1436            + std::fmt::Debug
1437            + for<'de> Deserialize<'de>,
1438        <K as kube::Resource>::DynamicType: Default,
1439        <K as Resource>::Scope: std::marker::Sized,
1440    {
1441        self.kube_patch(
1442            ctx.kaniop_ctx.client.clone(),
1443            &ctx.kaniop_ctx.metrics,
1444            resource,
1445            KANIDM_OPERATOR_NAME,
1446        )
1447        .await
1448    }
1449
1450    #[inline]
1451    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1452        LABELS
1453            .clone()
1454            .into_iter()
1455            .chain([
1456                (INSTANCE_LABEL.to_string(), self.name_any()),
1457                (CLUSTER_LABEL.to_string(), self.name_any()),
1458            ])
1459            .collect()
1460    }
1461
1462    #[inline]
1463    fn generate_labels(&self) -> BTreeMap<String, String> {
1464        self.generate_resource_labels()
1465    }
1466
1467    #[inline]
1468    fn get_tls_secret_name(&self) -> String {
1469        format!("{}-tls", self.name_any())
1470    }
1471
1472    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
1473    /// falling back to the ingress TLS secret and then to the default name.
1474    pub(crate) fn effective_tls_secret_name(&self) -> String {
1475        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1476            self.spec
1477                .ingress
1478                .as_ref()
1479                .and_then(|i| i.tls_secret_name.clone())
1480                .unwrap_or_else(|| self.get_tls_secret_name())
1481        })
1482    }
1483
1484    #[inline]
1485    fn get_namespace(&self) -> String {
1486        // safe unwrap: Kanidm is namespaced scoped
1487        self.namespace().unwrap()
1488    }
1489
1490    #[inline]
1491    fn is_replication_enabled(&self) -> bool {
1492        self.spec.replica_groups.len() > 1
1493            || self
1494                .spec
1495                .replica_groups
1496                .first()
1497                .is_some_and(|rg| rg.replicas > 1)
1498            || !self.spec.external_replication_nodes.is_empty()
1499    }
1500
1501    #[inline]
1502    fn is_updatable(&self, status: &KanidmStatus) -> bool {
1503        status
1504            .version
1505            .as_ref()
1506            .map(|v| {
1507                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1508                    return false;
1509                }
1510                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1511                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1512                match v.upgrade_check_result {
1513                    KanidmUpgradeCheckResult::Passed => {
1514                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1515                            minor <= v_minor + 1
1516                        } else {
1517                            true
1518                        }
1519                    }
1520                    KanidmUpgradeCheckResult::Failed => {
1521                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1522                            versions
1523                        {
1524                            major == v_major && minor == v_minor && patch >= v_patch
1525                        } else {
1526                            v.image_tag == current_tag
1527                        }
1528                    }
1529                }
1530            })
1531            .unwrap_or(true)
1532    }
1533
1534    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1535        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1536        let pod = pod_api.get(pod_name).await.map_err(|e| {
1537            Error::KubeError(
1538                format!("failed to get pod {namespace}/{pod_name}"),
1539                Box::new(e),
1540            )
1541        })?;
1542        Ok(pod
1543            .status
1544            .as_ref()
1545            .and_then(|s| s.conditions.as_ref())
1546            .is_some_and(|conditions| {
1547                conditions
1548                    .iter()
1549                    .any(|c| c.type_ == "Ready" && c.status == "True")
1550            }))
1551    }
1552
1553    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1554        let start = std::time::Instant::now();
1555        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1556        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1557
1558        loop {
1559            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1560                Ok(true) => return Ok(()),
1561                Ok(false) => {}
1562                Err(e) => {
1563                    if !e.is_retryable() {
1564                        return Err(e);
1565                    }
1566                    trace!(%e, "pod readiness check failed, retrying");
1567                }
1568            }
1569
1570            if start.elapsed() >= timeout {
1571                return Err(Error::ReceiveOutput(format!(
1572                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1573                )));
1574            }
1575
1576            sleep(poll_interval).await;
1577        }
1578    }
1579
1580    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1581    where
1582        I: IntoIterator<Item = T> + Debug,
1583        T: Into<String>,
1584    {
1585        let namespace = &self.get_namespace();
1586        trace!(
1587            resource.name = &pod_name,
1588            resource.namespace = &namespace,
1589            ?command,
1590            "pod exec"
1591        );
1592        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1593        let attached = pod
1594            .exec(
1595                pod_name,
1596                command,
1597                &AttachParams::default().container("kanidm"),
1598            )
1599            .await
1600            .map_err(|e| {
1601                Error::KubeError(
1602                    format!("failed to exec pod {namespace}/{pod_name}"),
1603                    Box::new(e),
1604                )
1605            })?;
1606        get_output(attached).await
1607    }
1608
1609    async fn exec_with_wait<I, T>(
1610        &self,
1611        ctx: Arc<Context>,
1612        pod_name: &str,
1613        command: I,
1614    ) -> Result<String>
1615    where
1616        I: IntoIterator<Item = T> + Debug,
1617        T: Into<String>,
1618    {
1619        let namespace = &self.get_namespace();
1620        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1621        self.exec(ctx, pod_name, command).await
1622    }
1623
1624    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1625    where
1626        I: IntoIterator<Item = T> + Debug,
1627        T: Into<String>,
1628    {
1629        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1630        let first_replica_group = self
1631            .spec
1632            .replica_groups
1633            .first()
1634            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1635        let sts_name = self.statefulset_name(&first_replica_group.name);
1636        let pod_name = format!("{sts_name}-0");
1637        self.exec(ctx, &pod_name, command).await
1638    }
1639
1640    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1641    where
1642        I: IntoIterator<Item = T> + Debug,
1643        T: Into<String>,
1644    {
1645        let first_replica_group = self
1646            .spec
1647            .replica_groups
1648            .first()
1649            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1650        let sts_name = self.statefulset_name(&first_replica_group.name);
1651        let pod_name = format!("{sts_name}-0");
1652        self.exec_with_wait(ctx, &pod_name, command).await
1653    }
1654}
1655
1656#[cfg(test)]
1657mod test {
1658    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1659    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1660
1661    use crate::controller::State;
1662    use crate::kanidm::controller::context::{Context, Stores};
1663    use crate::kanidm::crd::KanidmStatus;
1664
1665    use kaniop_k8s_util::error::Result;
1666    use kaniop_k8s_util::resources::hash_secret_data;
1667
1668    use std::collections::BTreeMap;
1669    use std::sync::Arc;
1670
1671    use http::{Request, Response};
1672    use k8s_openapi::ByteString;
1673    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1674    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1675    use k8s_openapi::api::networking::v1::Ingress;
1676    use kube::api::ObjectMeta;
1677    use kube::runtime::reflector::store::Writer;
1678    use kube::{Client, Resource, ResourceExt, client::Body};
1679    use opentelemetry::metrics::MeterProvider;
1680    use serde_json::json;
1681
1682    impl Kanidm {
1683        /// A normal test kanidm with a given status
1684        pub fn test() -> Self {
1685            let mut e = Kanidm::new(
1686                "test",
1687                serde_json::from_value(json!({
1688                    "domain": "idm.example.com",
1689                    "replicaGroups": [
1690                        {
1691                            "name": "default",
1692                            "replicas": 1
1693                        }
1694                    ]
1695                }))
1696                .unwrap(),
1697            );
1698            e.meta_mut().namespace = Some("default".into());
1699            // Pre-populate finalizer so reconcile callback runs immediately
1700            // (skips the "add finalizer + wait for next reconcile" step)
1701            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1702            e
1703        }
1704
1705        pub fn with_ingress(mut self) -> Self {
1706            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1707            self
1708        }
1709
1710        /// Modify kanidm replicas
1711        pub fn with_replicas(mut self, replicas: i32) -> Self {
1712            self.spec.replica_groups[0].replicas = replicas;
1713            self
1714        }
1715
1716        /// Modify kanidm to set a deletion timestamp
1717        pub fn needs_delete(mut self) -> Self {
1718            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1719            use k8s_openapi::jiff::Timestamp;
1720            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1721            self.meta_mut().deletion_timestamp = Some(Time(now));
1722            self
1723        }
1724
1725        /// Modify a kanidm to have an expected status
1726        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1727            self.status = Some(status);
1728            self
1729        }
1730    }
1731
1732    // We wrap tower_test::mock::Handle
1733    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1734    pub struct ApiServerVerifier(ApiServerHandle);
1735
1736    /// Scenarios we test for in ApiServerVerifier
1737    pub enum Scenario {
1738        Create(Kanidm),
1739        CreateWithTwoReplicas(Kanidm),
1740        CreateWithIngress(Kanidm),
1741        CreateWithIngressWithTwoReplicas(Kanidm),
1742        CreateWithTlsSecret(Kanidm, Box<Secret>),
1743    }
1744
1745    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1746        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1747            .await
1748            .expect("timeout on mock apiserver")
1749            .expect("scenario succeeded")
1750    }
1751
1752    impl ApiServerVerifier {
1753        /// Tests only get to run specific scenarios that has matching handlers
1754        ///
1755        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1756        ///
1757        /// NB: If the controller is making more calls than we are handling in the scenario,
1758        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1759        ///
1760        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1761        /// scenario runs to completion (i.e. all expected calls were responded to),
1762        /// using the timeout to catch missing api calls to Kubernetes.
1763        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1764            tokio::spawn(async move {
1765                // moving self => one scenario per test
1766                match scenario {
1767                    Scenario::Create(kanidm) => {
1768                        self.handle_kanidm_get(kanidm.clone())
1769                            .await
1770                            .unwrap()
1771                            .handle_kanidm_status_patch(kanidm.clone())
1772                            .await
1773                            .unwrap()
1774                            .handle_tls_secret_get_not_found(kanidm.clone())
1775                            .await
1776                            .unwrap()
1777                            .handle_backup_schedule_list()
1778                            .await
1779                            .unwrap()
1780                            .handle_backup_repository_list()
1781                            .await
1782                            .unwrap()
1783                            .handle_statefulset_patch(kanidm.clone(), None)
1784                            .await
1785                            .unwrap()
1786                            .handle_service_patch(kanidm.clone())
1787                            .await
1788                    }
1789                    Scenario::CreateWithTwoReplicas(kanidm) => {
1790                        self.handle_kanidm_get(kanidm.clone())
1791                            .await
1792                            .unwrap()
1793                            .handle_kanidm_status_patch(kanidm.clone())
1794                            .await
1795                            .unwrap()
1796                            .handle_tls_secret_get_not_found(kanidm.clone())
1797                            .await
1798                            .unwrap()
1799                            .handle_backup_schedule_list()
1800                            .await
1801                            .unwrap()
1802                            .handle_backup_repository_list()
1803                            .await
1804                            .unwrap()
1805                            .handle_statefulset_patch(kanidm.clone(), None)
1806                            .await
1807                            .unwrap()
1808                            .handle_service_patch(kanidm.clone())
1809                            .await
1810                    }
1811                    Scenario::CreateWithIngress(kanidm) => {
1812                        self.handle_kanidm_get(kanidm.clone())
1813                            .await
1814                            .unwrap()
1815                            .handle_kanidm_status_patch(kanidm.clone())
1816                            .await
1817                            .unwrap()
1818                            .handle_tls_secret_get_not_found(kanidm.clone())
1819                            .await
1820                            .unwrap()
1821                            .handle_backup_schedule_list()
1822                            .await
1823                            .unwrap()
1824                            .handle_backup_repository_list()
1825                            .await
1826                            .unwrap()
1827                            .handle_statefulset_patch(kanidm.clone(), None)
1828                            .await
1829                            .unwrap()
1830                            .handle_service_patch(kanidm.clone())
1831                            .await
1832                            .unwrap()
1833                            .handle_ingress_patch(kanidm.clone())
1834                            .await
1835                    }
1836                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1837                        self.handle_kanidm_get(kanidm.clone())
1838                            .await
1839                            .unwrap()
1840                            .handle_kanidm_status_patch(kanidm.clone())
1841                            .await
1842                            .unwrap()
1843                            .handle_tls_secret_get_not_found(kanidm.clone())
1844                            .await
1845                            .unwrap()
1846                            .handle_backup_schedule_list()
1847                            .await
1848                            .unwrap()
1849                            .handle_backup_repository_list()
1850                            .await
1851                            .unwrap()
1852                            .handle_statefulset_patch(kanidm.clone(), None)
1853                            .await
1854                            .unwrap()
1855                            .handle_service_patch(kanidm.clone())
1856                            .await
1857                            .unwrap()
1858                            .handle_ingress_patch(kanidm.clone())
1859                            .await
1860                    }
1861                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1862                        let expected_hash = hash_secret_data(&secret);
1863                        self.handle_kanidm_get(kanidm.clone())
1864                            .await
1865                            .unwrap()
1866                            .handle_kanidm_status_patch(kanidm.clone())
1867                            .await
1868                            .unwrap()
1869                            .handle_tls_secret_get(kanidm.clone(), *secret)
1870                            .await
1871                            .unwrap()
1872                            .handle_backup_schedule_list()
1873                            .await
1874                            .unwrap()
1875                            .handle_backup_repository_list()
1876                            .await
1877                            .unwrap()
1878                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1879                            .await
1880                            .unwrap()
1881                            .handle_service_patch(kanidm.clone())
1882                            .await
1883                    }
1884                }
1885                .expect("scenario completed without errors");
1886            })
1887        }
1888
1889        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1890            let (request, send) = self.0.next_request().await.expect("service not called");
1891            assert_eq!(request.method(), http::Method::GET);
1892            let expected_uri_prefix = format!(
1893                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1894                kanidm.name_any()
1895            );
1896            let uri = request.uri().to_string();
1897            assert!(
1898                uri.starts_with(&expected_uri_prefix),
1899                "expected uri to start with {}, got {}",
1900                expected_uri_prefix,
1901                uri
1902            );
1903            let response = serde_json::to_vec(&kanidm).unwrap();
1904            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1905            Ok(self)
1906        }
1907
1908        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1909            let (request, send) = self.0.next_request().await.expect("service not called");
1910            assert_eq!(request.method(), http::Method::PATCH);
1911            assert_eq!(
1912                request.uri().to_string(),
1913                format!(
1914                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1915                    kanidm.name_any()
1916                )
1917            );
1918
1919            let req_body = request.into_body().collect_bytes().await.unwrap();
1920            let json: serde_json::Value =
1921                serde_json::from_slice(&req_body).expect("patch object is json");
1922            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1923                .expect("valid kanidm status");
1924            let mut kanidm = Kanidm::test();
1925            kanidm.status = Some(status.clone());
1926            let response = serde_json::to_vec(&kanidm).unwrap();
1927            // pass through kanidm "patch accepted"
1928            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1929            Ok(self)
1930        }
1931
1932        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1933            let (request, send) = self.0.next_request().await.expect("service not called");
1934            assert_eq!(request.method(), http::Method::GET);
1935            let expected_uri_prefix = format!(
1936                "/api/v1/namespaces/default/secrets/{}",
1937                kanidm.effective_tls_secret_name()
1938            );
1939            let uri = request.uri().to_string();
1940            assert!(
1941                uri.starts_with(&expected_uri_prefix),
1942                "expected uri to start with {expected_uri_prefix}, got {uri}"
1943            );
1944            let status = serde_json::json!({
1945                "kind": "Status",
1946                "apiVersion": "v1",
1947                "metadata": {},
1948                "status": "Failure",
1949                "message": "secret not found",
1950                "reason": "NotFound",
1951                "code": 404
1952            });
1953            send.send_response(
1954                Response::builder()
1955                    .status(404)
1956                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1957                    .unwrap(),
1958            );
1959            Ok(self)
1960        }
1961
1962        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1963            let (request, send) = self.0.next_request().await.expect("service not called");
1964            assert_eq!(request.method(), http::Method::GET);
1965            let expected_uri_prefix = format!(
1966                "/api/v1/namespaces/default/secrets/{}",
1967                kanidm.effective_tls_secret_name()
1968            );
1969            let uri = request.uri().to_string();
1970            assert!(
1971                uri.starts_with(&expected_uri_prefix),
1972                "expected uri to start with {expected_uri_prefix}, got {uri}"
1973            );
1974            let response = serde_json::to_vec(&secret).unwrap();
1975            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1976            Ok(self)
1977        }
1978
1979        async fn handle_backup_schedule_list(mut self) -> Result<Self> {
1980            let (request, send) = self.0.next_request().await.expect("service not called");
1981            assert_eq!(request.method(), http::Method::GET);
1982            assert!(
1983                request
1984                    .uri()
1985                    .to_string()
1986                    .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
1987                "expected backup schedule list request, got {}",
1988                request.uri()
1989            );
1990            let list = json!({
1991                "apiVersion": "kaniop.rs/v1alpha1",
1992                "kind": "KanidmBackupScheduleList",
1993                "metadata": {},
1994                "items": []
1995            });
1996            let response = serde_json::to_vec(&list).unwrap();
1997            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1998            Ok(self)
1999        }
2000
2001        async fn handle_backup_repository_list(mut self) -> Result<Self> {
2002            let (request, send) = self.0.next_request().await.expect("service not called");
2003            assert_eq!(request.method(), http::Method::GET);
2004            assert!(
2005                request.uri().to_string().contains(
2006                    "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2007                ),
2008                "expected backup repository list request, got {}",
2009                request.uri()
2010            );
2011            let list = json!({
2012                "apiVersion": "kaniop.rs/v1alpha1",
2013                "kind": "KanidmBackupRepositoryList",
2014                "metadata": {},
2015                "items": []
2016            });
2017            let response = serde_json::to_vec(&list).unwrap();
2018            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2019            Ok(self)
2020        }
2021
2022        async fn handle_statefulset_patch(
2023            mut self,
2024            kanidm: Kanidm,
2025            expected_tls_hash: Option<String>,
2026        ) -> Result<Self> {
2027            for rg in kanidm.spec.replica_groups.iter() {
2028                let (request, send) = self.0.next_request().await.expect("service not called");
2029                assert_eq!(request.method(), http::Method::PATCH);
2030                assert_eq!(
2031                    request.uri().to_string(),
2032                    format!(
2033                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2034                        kanidm.statefulset_name(&rg.name)
2035                    )
2036                );
2037                let req_body = request.into_body().collect_bytes().await.unwrap();
2038                let json: serde_json::Value =
2039                    serde_json::from_slice(&req_body).expect("patch object is json");
2040                let statefulset: StatefulSet =
2041                    serde_json::from_value(json).expect("valid statefulset");
2042                assert_eq!(
2043                    statefulset.clone().spec.unwrap().replicas.unwrap(),
2044                    rg.replicas
2045                );
2046                let tls_hash_annotation = statefulset
2047                    .spec
2048                    .as_ref()
2049                    .unwrap()
2050                    .template
2051                    .metadata
2052                    .as_ref()
2053                    .unwrap()
2054                    .annotations
2055                    .as_ref()
2056                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2057                    .cloned();
2058                assert_eq!(tls_hash_annotation, expected_tls_hash);
2059                let response = serde_json::to_vec(&statefulset).unwrap();
2060                // pass through kanidm "patch accepted"
2061                send.send_response(Response::builder().body(Body::from(response)).unwrap());
2062            }
2063
2064            Ok(self)
2065        }
2066
2067        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2068            let (request, send) = self.0.next_request().await.expect("service not called");
2069            assert_eq!(request.method(), http::Method::PATCH);
2070            assert_eq!(
2071                request.uri().to_string(),
2072                format!(
2073                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2074                    kanidm.name_any()
2075                )
2076            );
2077
2078            let req_body = request.into_body().collect_bytes().await.unwrap();
2079            let json: serde_json::Value =
2080                serde_json::from_slice(&req_body).expect("patch object is json");
2081            let service: Service = serde_json::from_value(json).expect("valid service");
2082            let response = serde_json::to_vec(&service).unwrap();
2083            // pass through kanidm "patch accepted"
2084            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2085            Ok(self)
2086        }
2087
2088        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2089            let (request, send) = self.0.next_request().await.expect("service not called");
2090            assert_eq!(request.method(), http::Method::PATCH);
2091            assert_eq!(
2092                request.uri().to_string(),
2093                format!(
2094                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2095                    kanidm.name_any()
2096                )
2097            );
2098
2099            let req_body = request.into_body().collect_bytes().await.unwrap();
2100            let json: serde_json::Value =
2101                serde_json::from_slice(&req_body).expect("patch object is json");
2102            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2103            let response = serde_json::to_vec(&ingress).unwrap();
2104            // pass through kanidm "patch accepted"
2105            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2106            Ok(self)
2107        }
2108    }
2109
2110    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2111        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2112        let mock_client = Client::new(mock_service, "default");
2113        let stores = Stores {
2114            stateful_set_store: Writer::default().as_reader(),
2115            service_store: Writer::default().as_reader(),
2116            ingress_store: Writer::default().as_reader(),
2117            secret_store: Writer::default().as_reader(),
2118            http_route_store: Some(Writer::default().as_reader()),
2119            backend_tls_policy_store: Some(Writer::default().as_reader()),
2120            deployment_store: Writer::default().as_reader(),
2121            config_map_store: Writer::default().as_reader(),
2122        };
2123        let controller_id = "test";
2124
2125        // Create a test meter for metrics
2126        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2127        let meter = provider.meter("test");
2128        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2129
2130        let state = State::new(
2131            metrics,
2132            Writer::default().as_reader(),
2133            Writer::default().as_reader(),
2134            None,
2135        );
2136        let ctx = Arc::new(Context::new(
2137            state.to_context(mock_client, controller_id),
2138            stores,
2139        ));
2140        (ctx, ApiServerVerifier(handle))
2141    }
2142
2143    #[tokio::test]
2144    async fn kanidm_create() {
2145        let (testctx, fakeserver) = get_test_context();
2146        let kanidm = Kanidm::test();
2147        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2148        reconcile_kanidm(Arc::new(kanidm), testctx)
2149            .await
2150            .expect("reconciler");
2151        timeout_after_1s(mocksrv).await;
2152    }
2153
2154    #[tokio::test]
2155    async fn kanidm_create_with_two_replicas() {
2156        let (testctx, fakeserver) = get_test_context();
2157        let kanidm = Kanidm::test().with_replicas(2);
2158        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2159        reconcile_kanidm(Arc::new(kanidm), testctx)
2160            .await
2161            .expect("reconciler");
2162        timeout_after_1s(mocksrv).await;
2163    }
2164
2165    #[tokio::test]
2166    async fn kanidm_create_with_ingress() {
2167        let (testctx, fakeserver) = get_test_context();
2168        let kanidm = Kanidm::test().with_ingress();
2169        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2170        reconcile_kanidm(Arc::new(kanidm), testctx)
2171            .await
2172            .expect("reconciler");
2173        timeout_after_1s(mocksrv).await;
2174    }
2175
2176    #[tokio::test]
2177    async fn kanidm_create_with_ingress_with_two_replicas() {
2178        let (testctx, fakeserver) = get_test_context();
2179        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2180        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2181        reconcile_kanidm(Arc::new(kanidm), testctx)
2182            .await
2183            .expect("reconciler");
2184        timeout_after_1s(mocksrv).await;
2185    }
2186
2187    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2188        Arc::new(StatefulSet {
2189            metadata: ObjectMeta {
2190                name: Some(format!("{cluster}-default")),
2191                namespace: Some(namespace.to_string()),
2192                labels: Some(BTreeMap::from([(
2193                    CLUSTER_LABEL.to_string(),
2194                    cluster.to_string(),
2195                )])),
2196                ..Default::default()
2197            },
2198            spec: Some(StatefulSetSpec {
2199                template: PodTemplateSpec {
2200                    metadata: Some(ObjectMeta {
2201                        annotations: tls_hash.map(|h| {
2202                            BTreeMap::from([(
2203                                TLS_SECRET_HASH_ANNOTATION.to_string(),
2204                                h.to_string(),
2205                            )])
2206                        }),
2207                        ..Default::default()
2208                    }),
2209                    ..Default::default()
2210                },
2211                ..Default::default()
2212            }),
2213            ..Default::default()
2214        })
2215    }
2216
2217    #[test]
2218    fn test_previous_tls_secret_hash() {
2219        let kanidm = Kanidm::test();
2220
2221        // annotation carried over from an existing statefulset
2222        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2223        assert_eq!(
2224            previous_tls_secret_hash(&kanidm, stss),
2225            Some("abc123".to_string())
2226        );
2227
2228        // no annotation on the existing statefulset
2229        let stss = vec![statefulset_of("test", "default", None)];
2230        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2231
2232        // statefulsets from another cluster or namespace are ignored
2233        let stss = vec![
2234            statefulset_of("other", "default", Some("abc123")),
2235            statefulset_of("test", "other-ns", Some("abc123")),
2236        ];
2237        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2238    }
2239
2240    #[tokio::test]
2241    async fn kanidm_create_with_tls_secret() {
2242        let (testctx, fakeserver) = get_test_context();
2243        let kanidm = Kanidm::test();
2244        let secret = Secret {
2245            metadata: ObjectMeta {
2246                name: Some(kanidm.effective_tls_secret_name()),
2247                namespace: Some("default".to_string()),
2248                ..Default::default()
2249            },
2250            data: Some(BTreeMap::from([
2251                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2252                ("tls.key".to_string(), ByteString(b"key".to_vec())),
2253            ])),
2254            ..Default::default()
2255        };
2256        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2257            kanidm.clone(),
2258            Box::new(secret),
2259        ));
2260        reconcile_kanidm(Arc::new(kanidm), testctx)
2261            .await
2262            .expect("reconciler");
2263        timeout_after_1s(mocksrv).await;
2264    }
2265
2266    #[tokio::test]
2267    async fn replication_secret_gc_skips_non_replica_secrets() {
2268        use super::reconcile_replication_secrets;
2269        use super::secret::{SECRET_TYPE_LABEL, SecretType};
2270
2271        let kanidm = Kanidm::test();
2272        let status = kanidm.status.clone().unwrap_or_default();
2273
2274        let replica_cert_label =
2275            serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
2276
2277        let replica_cert_secret = Secret {
2278            metadata: ObjectMeta {
2279                name: Some(format!("{}-default-0-cert", kanidm.name_any())),
2280                namespace: Some("default".into()),
2281                labels: Some(BTreeMap::from([
2282                    (CLUSTER_LABEL.to_string(), kanidm.name_any()),
2283                    (SECRET_TYPE_LABEL.to_string(), replica_cert_label.clone()),
2284                ])),
2285                ..Default::default()
2286            },
2287            ..Default::default()
2288        };
2289
2290        let mail_sender_secret = Secret {
2291            metadata: ObjectMeta {
2292                name: Some(format!("{}-mail-sender-config", kanidm.name_any())),
2293                namespace: Some("default".into()),
2294                labels: Some(BTreeMap::from([(
2295                    CLUSTER_LABEL.to_string(),
2296                    kanidm.name_any(),
2297                )])),
2298                ..Default::default()
2299            },
2300            ..Default::default()
2301        };
2302
2303        let (mock_service, mut handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2304        let mock_client = Client::new(mock_service, "default");
2305
2306        let mut secret_writer = Writer::default();
2307        secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2308            replica_cert_secret.clone(),
2309        ));
2310        secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2311            mail_sender_secret.clone(),
2312        ));
2313
2314        let stores = Stores {
2315            stateful_set_store: Writer::default().as_reader(),
2316            service_store: Writer::default().as_reader(),
2317            ingress_store: Writer::default().as_reader(),
2318            secret_store: secret_writer.as_reader(),
2319            http_route_store: Some(Writer::default().as_reader()),
2320            backend_tls_policy_store: Some(Writer::default().as_reader()),
2321            deployment_store: Writer::default().as_reader(),
2322            config_map_store: Writer::default().as_reader(),
2323        };
2324
2325        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2326        let meter = provider.meter("test");
2327        let metrics = crate::metrics::Metrics::new(&meter, &["test"]);
2328        let state = State::new(
2329            metrics,
2330            Writer::default().as_reader(),
2331            Writer::default().as_reader(),
2332            None,
2333        );
2334        let ctx = Arc::new(Context::new(state.to_context(mock_client, "test"), stores));
2335
2336        let mock_srv = tokio::spawn(async move {
2337            let (request, send) = handle.next_request().await.expect("expected a request");
2338            assert_eq!(request.method(), http::Method::DELETE);
2339            let uri = request.uri().to_string();
2340            assert!(
2341                uri.contains("-default-0-cert"),
2342                "expected delete for replica cert secret, got {uri}"
2343            );
2344            let not_found = serde_json::json!({
2345                "kind": "Status",
2346                "apiVersion": "v1",
2347                "metadata": {},
2348                "status": "Failure",
2349                "message": "not found",
2350                "reason": "NotFound",
2351                "code": 404
2352            });
2353            send.send_response(
2354                Response::builder()
2355                    .status(404)
2356                    .body(Body::from(serde_json::to_vec(&not_found).unwrap()))
2357                    .unwrap(),
2358            );
2359
2360            if let Ok(Some((req, _))) =
2361                tokio::time::timeout(std::time::Duration::from_millis(200), handle.next_request())
2362                    .await
2363            {
2364                panic!("unexpected second request: {} {}", req.method(), req.uri());
2365            }
2366        });
2367
2368        let changed = reconcile_replication_secrets(Arc::new(kanidm), ctx, &status)
2369            .await
2370            .expect("reconcile succeeds despite 404 on delete");
2371        assert!(
2372            changed,
2373            "expected changed=true since a deprecated secret was found"
2374        );
2375
2376        tokio::time::timeout(std::time::Duration::from_secs(1), mock_srv)
2377            .await
2378            .expect("timeout waiting for mock server")
2379            .expect("mock server task succeeded");
2380    }
2381}