Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::{SECRET_TYPE_LABEL, SecretExt, SecretType};
20use self::service::ServiceExt;
21use self::statefulset::{
22    StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23    preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_backup_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::kanidm::KanidmUser;
31use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
32use crate::kanidm::crd::{
33    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
34};
35use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
36use crate::telemetry;
37
38use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
39use kaniop_k8s_util::client::get_output;
40use kaniop_k8s_util::error::{Error, Result};
41use kaniop_k8s_util::parse::parse_semver;
42use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
43
44use std::collections::{BTreeMap, BTreeSet};
45use std::fmt::Debug;
46use std::sync::{Arc, LazyLock};
47
48use futures::future::{TryJoinAll, join_all, try_join_all};
49use futures::stream::{self, StreamExt};
50use futures::try_join;
51use k8s_openapi::NamespaceResourceScope;
52use k8s_openapi::api::apps::v1::StatefulSet;
53use k8s_openapi::api::core::v1::{Pod, Secret, Service};
54use kube::Resource;
55use kube::ResourceExt;
56use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
57use kube::runtime::controller::Action;
58use kube::runtime::events::{Event, EventType};
59use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
60use serde::{Deserialize, Serialize};
61use tokio::time::{Duration, sleep};
62use tracing::{Span, debug, field, info, instrument, trace, warn};
63
64const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
65const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
66const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
67const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
68const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
69const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
70const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
71const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
72const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
73
74pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
75const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
76pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
77
78const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
79
80static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
81    BTreeMap::from([
82        (NAME_LABEL.to_string(), "kanidm".to_string()),
83        (
84            MANAGED_BY_LABEL.to_string(),
85            format!("kaniop-{CONTROLLER_ID}"),
86        ),
87    ])
88});
89
90async fn restart_statefulsets_after_replica_secret_update(
91    ctx: Arc<Context>,
92    sts_api: &Api<StatefulSet>,
93    namespace: &str,
94    statefulset_names: &[String],
95    sequential: bool,
96) -> Result<()> {
97    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
98    // Pods must restart after cert secret changes or replication keeps stale TLS material.
99    let restart_futures = statefulset_names
100        .iter()
101        .map(|sts_name| sts_api.restart(sts_name));
102
103    if sequential {
104        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
105        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
106            return Err(Error::kube_error(
107                "restart",
108                "StatefulSet",
109                namespace,
110                "replica certificate secret update statefulsets",
111                first_err,
112            ));
113        }
114    } else {
115        let results: Vec<_> = join_all(restart_futures).await;
116        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
117            return Err(Error::kube_error(
118                "restart",
119                "StatefulSet",
120                namespace,
121                "replica certificate secret update statefulsets",
122                first_err,
123            ));
124        }
125    }
126
127    let rollout_futures = statefulset_names
128        .iter()
129        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
130    try_join_all(rollout_futures).await?;
131
132    Ok(())
133}
134
135pub async fn reconcile_admins_secret(
136    kanidm: Arc<Kanidm>,
137    ctx: Arc<Context>,
138    status: &KanidmStatus,
139) -> Result<bool> {
140    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
141        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
142        kanidm.patch(&ctx, admins_secret).await?;
143        return Ok(true);
144    }
145    Ok(false)
146}
147
148pub async fn reconcile_replication_secrets(
149    kanidm: Arc<Kanidm>,
150    ctx: Arc<Context>,
151    status: &KanidmStatus,
152) -> Result<bool> {
153    let mut changed = false;
154    let expected_secret_names = if kanidm.is_replication_enabled() {
155        status
156            .replica_statuses
157            .iter()
158            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
159            .collect::<Vec<_>>()
160    } else {
161        Vec::new()
162    };
163    let replica_cert_label =
164        serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
165    let deprecated_secrets = ctx
166        .stores
167        .secret_store
168        .state()
169        .into_iter()
170        .filter(|secret| {
171            secret.namespace() == kanidm.namespace()
172                && kanidm.admins_secret_name() != secret.name_any()
173                && !expected_secret_names.contains(&secret.name_any())
174                && secret.metadata.labels.as_ref().is_some_and(|l| {
175                    l.get(SECRET_TYPE_LABEL) == Some(&replica_cert_label)
176                        && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
177                })
178        })
179        .collect::<Vec<_>>();
180    if !deprecated_secrets.is_empty() {
181        changed = true;
182    }
183    let secret_delete_future = deprecated_secrets
184        .iter()
185        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
186        .collect::<TryJoinAll<_>>();
187    try_join!(secret_delete_future)?;
188
189    if kanidm.is_replication_enabled() {
190        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
191        let statefulset_names = status
192            .replica_statuses
193            .iter()
194            .map(|rs| rs.statefulset_name.clone())
195            .collect::<BTreeSet<_>>()
196            .into_iter()
197            .collect::<Vec<_>>();
198
199        let has_pending = status
200            .replica_statuses
201            .iter()
202            .any(|rs| rs.state == KanidmReplicaState::Pending);
203
204        let sts_api =
205            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
206
207        if has_pending {
208            changed = true;
209            stream::iter(
210                status
211                    .replica_statuses
212                    .iter()
213                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
214            )
215            .then(|rs| async {
216                let secret = kanidm
217                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
218                    .await?;
219                kanidm.patch(&ctx, secret).await
220            })
221            .collect::<Vec<_>>()
222            .await
223            .into_iter()
224            .collect::<Result<Vec<_>>>()?;
225
226            let restart_futures = status
227                .replica_statuses
228                .iter()
229                .map(|rs| sts_api.restart(&rs.statefulset_name));
230
231            if has_single_replica {
232                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
233                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
234                    return Err(Error::kube_error(
235                        "restart",
236                        "StatefulSet",
237                        kanidm.get_namespace(),
238                        "pending replica statefulsets",
239                        first_err,
240                    ));
241                }
242            } else {
243                let results: Vec<_> = join_all(restart_futures).await;
244                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
245                    return Err(Error::kube_error(
246                        "restart",
247                        "StatefulSet",
248                        kanidm.get_namespace(),
249                        "pending replica statefulsets",
250                        first_err,
251                    ));
252                }
253            }
254        }
255
256        let has_certificate_host_invalid = status
257            .replica_statuses
258            .iter()
259            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
260
261        let has_certificate_expiring = status
262            .replica_statuses
263            .iter()
264            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
265
266        let has_non_ready_replicas = status
267            .replica_statuses
268            .iter()
269            .any(|rs| rs.state != KanidmReplicaState::Ready);
270
271        let host_invalid_replicas: Vec<_> = status
272            .replica_statuses
273            .iter()
274            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
275            .collect();
276
277        let cert_expiring_replicas: Vec<_> = status
278            .replica_statuses
279            .iter()
280            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
281            .collect();
282
283        if !host_invalid_replicas.is_empty() {
284            changed = true;
285            let restart_futures = host_invalid_replicas
286                .iter()
287                .map(|rs| sts_api.restart(&rs.statefulset_name));
288
289            if has_single_replica {
290                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
291                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
292                    return Err(Error::kube_error(
293                        "restart",
294                        "StatefulSet",
295                        kanidm.get_namespace(),
296                        "certificate host invalid statefulsets",
297                        first_err,
298                    ));
299                }
300            } else {
301                let results: Vec<_> = join_all(restart_futures).await;
302                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
303                    return Err(Error::kube_error(
304                        "restart",
305                        "StatefulSet",
306                        kanidm.get_namespace(),
307                        "certificate host invalid statefulsets",
308                        first_err,
309                    ));
310                }
311            }
312
313            let restarted_sts_names: Vec<_> = host_invalid_replicas
314                .iter()
315                .map(|rs| rs.statefulset_name.clone())
316                .collect::<BTreeSet<_>>()
317                .into_iter()
318                .collect();
319
320            let namespace = kanidm.get_namespace();
321            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
322                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
323            });
324            try_join_all(rollout_futures).await?;
325
326            let renew_futures = host_invalid_replicas
327                .iter()
328                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
329            let results: Vec<_> = join_all(renew_futures).await;
330            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
331                return Err(first_err);
332            }
333
334            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
335
336            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
337                let cert = show_replica_cert_with_retries(
338                    &kanidm,
339                    ctx.clone(),
340                    &rs.pod_name,
341                    CERT_SHOW_RETRY_ATTEMPTS,
342                )
343                .await?;
344                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
345                kanidm.patch(&ctx, secret.clone()).await
346            });
347            try_join_all(cert_futures).await?;
348
349            restart_statefulsets_after_replica_secret_update(
350                ctx.clone(),
351                &sts_api,
352                &kanidm.get_namespace(),
353                &statefulset_names,
354                has_single_replica,
355            )
356            .await?;
357
358            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
359        }
360
361        if !cert_expiring_replicas.is_empty() {
362            changed = true;
363            let renew_futures = cert_expiring_replicas
364                .iter()
365                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
366            let results: Vec<_> = join_all(renew_futures).await;
367            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
368                return Err(first_err);
369            }
370
371            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
372
373            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
374                let cert = show_replica_cert_with_retries(
375                    &kanidm,
376                    ctx.clone(),
377                    &rs.pod_name,
378                    CERT_SHOW_RETRY_ATTEMPTS,
379                )
380                .await?;
381                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
382                kanidm.patch(&ctx, secret.clone()).await
383            });
384            try_join_all(cert_futures).await?;
385
386            restart_statefulsets_after_replica_secret_update(
387                ctx.clone(),
388                &sts_api,
389                &kanidm.get_namespace(),
390                &statefulset_names,
391                has_single_replica,
392            )
393            .await?;
394
395            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
396        }
397
398        if has_non_ready_replicas
399            && !has_pending
400            && !has_certificate_host_invalid
401            && !has_certificate_expiring
402        {
403            changed = true;
404            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
405
406            let restart_futures = status
407                .replica_statuses
408                .iter()
409                .map(|rs| sts_api.restart(&rs.statefulset_name));
410
411            if has_single_replica {
412                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
413                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
414                    return Err(Error::kube_error(
415                        "restart",
416                        "StatefulSet",
417                        kanidm.get_namespace(),
418                        "replica statefulsets",
419                        first_err,
420                    ));
421                }
422            } else {
423                let results: Vec<_> = join_all(restart_futures).await;
424                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
425                    return Err(Error::kube_error(
426                        "restart",
427                        "StatefulSet",
428                        kanidm.get_namespace(),
429                        "replica statefulsets",
430                        first_err,
431                    ));
432                }
433            }
434        }
435    }
436
437    Ok(changed)
438}
439
440#[instrument(skip(ctx, kanidm))]
441pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
442    let trace_id = telemetry::get_trace_id();
443    Span::current().record("trace_id", field::display(&trace_id));
444    let _timer = ctx
445        .kaniop_ctx
446        .metrics
447        .reconcile_count_and_measure(&trace_id);
448    info!("reconciling Kanidm");
449
450    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
451        debug!(%e, "failed to reconcile status");
452        ctx.kaniop_ctx.metrics.status_update_errors_inc();
453        e
454    })?;
455    let kanidm_api: Api<Kanidm> =
456        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
457    let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
458    let outcome_clone = outcome.clone();
459    let kanidm_for_finalizer = kanidm.clone();
460    let ctx_for_finalizer = ctx.clone();
461    let action = finalizer(
462        &kanidm_api,
463        KANIDM_FINALIZER,
464        kanidm_for_finalizer,
465        move |event| {
466            let outcome = outcome_clone.clone();
467            let ctx = ctx_for_finalizer.clone();
468            let status = status.clone();
469            async move {
470                match event {
471                    Finalizer::Apply(kanidm) => {
472                        let (action, changed) = reconcile(kanidm, ctx, status).await?;
473                        outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
474                        Ok(action)
475                    }
476                    Finalizer::Cleanup(kanidm) => {
477                        let (action, changed) = cleanup(kanidm, ctx).await?;
478                        outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
479                        Ok(action)
480                    }
481                }
482            }
483        },
484    )
485    .await
486    .or_else(|e| match e {
487        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
488            debug!("resource already removed during finalizer cleanup");
489            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
490        }
491        _ => Err(Error::FinalizerError(
492            "failed on kanidm account finalizer".to_string(),
493            Box::new(e),
494        )),
495    });
496
497    // Update status after reconciliation completes, even if it failed.
498    // This ensures observers can detect that reconciliation was attempted.
499    if let Err(update_err) = kanidm.update_status(ctx.clone()).await {
500        debug!(%update_err, "failed to update status after reconciliation");
501        ctx.kaniop_ctx.metrics.status_update_errors_inc();
502    }
503
504    let action = action?;
505    let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
506    Ok((action, changed))
507}
508
509/// Hash of the TLS secret content, injected as a pod template annotation so
510/// certificate renewals roll the statefulsets. `None` if the secret does not
511/// exist yet.
512async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
513    let secret_name = kanidm.effective_tls_secret_name();
514    let secret_api =
515        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
516    let secret = secret_api
517        .get_opt(&secret_name)
518        .await
519        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
520    Ok(secret.as_ref().map(hash_secret_data))
521}
522
523/// Previous TLS secret hash annotation from the existing statefulsets.
524///
525/// Used when the TLS secret is missing: dropping the annotation would roll
526/// the pods onto a missing volume and leave them stuck in ContainerCreating.
527fn previous_tls_secret_hash(
528    kanidm: &Kanidm,
529    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
530) -> Option<String> {
531    stateful_sets
532        .into_iter()
533        .filter(|sts| {
534            sts.namespace() == kanidm.namespace()
535                && sts
536                    .metadata
537                    .labels
538                    .as_ref()
539                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
540        })
541        .find_map(|sts| {
542            sts.spec
543                .as_ref()?
544                .template
545                .metadata
546                .as_ref()?
547                .annotations
548                .as_ref()?
549                .get(TLS_SECRET_HASH_ANNOTATION)
550                .cloned()
551        })
552}
553
554fn is_headless_service(service: &Service) -> bool {
555    service
556        .spec
557        .as_ref()
558        .and_then(|spec| spec.cluster_ip.as_deref())
559        == Some("None")
560}
561
562fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
563    if is_headless_service(desired) != is_headless_service(current) {
564        return;
565    }
566    let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
567    else {
568        return;
569    };
570    if desired_spec.cluster_ip.is_none() {
571        desired_spec.cluster_ip = current_spec.cluster_ip.clone();
572    }
573    if desired_spec.cluster_ips.is_none() {
574        desired_spec.cluster_ips = current_spec.cluster_ips.clone();
575    }
576}
577
578async fn wait_for_statefulset_deletion(
579    client: kube::Client,
580    namespace: &str,
581    name: &str,
582) -> Result<()> {
583    let api = Api::<StatefulSet>::namespaced(client, namespace);
584    let start = std::time::Instant::now();
585    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
586    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
587
588    loop {
589        match api.get_opt(name).await {
590            Ok(None) => return Ok(()),
591            Ok(Some(_)) => {}
592            Err(e) => {
593                return Err(Error::kube_error(
594                    "wait for deletion of",
595                    "StatefulSet",
596                    namespace,
597                    name,
598                    e,
599                ));
600            }
601        }
602
603        if start.elapsed() >= timeout {
604            return Err(Error::ReceiveOutput(format!(
605                "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
606            )));
607        }
608        sleep(poll_interval).await;
609    }
610}
611
612async fn reconcile_service(
613    kanidm: &Kanidm,
614    ctx: &Context,
615    mut desired: Service,
616) -> Result<Service> {
617    let namespace = kanidm.get_namespace();
618    let name = desired.name_any();
619    let desired_headless = is_headless_service(&desired);
620
621    if let Some(cached) = ctx
622        .stores
623        .service_store
624        .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
625    {
626        if is_headless_service(cached.as_ref()) == desired_headless {
627            preserve_defaulted_service_fields(&mut desired, cached.as_ref());
628        } else {
629            let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
630            let current = service_api
631                .get_opt(&name)
632                .await
633                .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
634
635            if let Some(current) = current {
636                let current_headless = is_headless_service(&current);
637                if current_headless != desired_headless {
638                    info!(
639                        resource.name = &name,
640                        resource.namespace = &namespace,
641                        current_headless,
642                        desired_headless,
643                        "recreating Service to change headless mode"
644                    );
645                    kanidm.delete(ctx, &current).await?;
646                    ctx.kaniop_ctx
647                        .metrics
648                        .reconcile_deploy_delete_create_inc("Service", "headless_transition");
649                    wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
650                        .await?;
651                } else {
652                    preserve_defaulted_service_fields(&mut desired, &current);
653                }
654            }
655        }
656    }
657
658    kanidm.patch(ctx, desired).await
659}
660
661async fn wait_for_service_deletion(
662    client: kube::Client,
663    namespace: &str,
664    name: &str,
665) -> Result<()> {
666    let api = Api::<Service>::namespaced(client, namespace);
667    let start = std::time::Instant::now();
668    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
669    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
670
671    loop {
672        match api.get_opt(name).await {
673            Ok(None) => return Ok(()),
674            Ok(Some(_)) => {}
675            Err(e) => {
676                return Err(Error::kube_error(
677                    "wait for deletion of",
678                    "Service",
679                    namespace,
680                    name,
681                    e,
682                ));
683            }
684        }
685
686        if start.elapsed() >= timeout {
687            return Err(Error::ReceiveOutput(format!(
688                "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
689            )));
690        }
691        sleep(poll_interval).await;
692    }
693}
694
695async fn validate_statefulset_replacement(
696    api: &Api<StatefulSet>,
697    desired: &StatefulSet,
698    namespace: &str,
699    name: &str,
700) -> Result<()> {
701    let mut candidate = desired.clone();
702    candidate.metadata.name = None;
703    candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
704    candidate.metadata.namespace = None;
705    candidate.metadata.resource_version = None;
706    candidate.metadata.uid = None;
707    candidate.metadata.creation_timestamp = None;
708    candidate.metadata.generation = None;
709    candidate.metadata.managed_fields = None;
710    candidate.metadata.deletion_timestamp = None;
711    candidate.metadata.deletion_grace_period_seconds = None;
712    candidate.status = None;
713
714    api.create(
715        &PostParams {
716            dry_run: true,
717            ..Default::default()
718        },
719        &candidate,
720    )
721    .await
722    .map(|_| ())
723    .map_err(|e| {
724        Error::kube_error(
725            "validate replacement for",
726            "StatefulSet",
727            namespace,
728            name,
729            e,
730        )
731    })
732}
733
734async fn reconcile_statefulset(
735    kanidm: Arc<Kanidm>,
736    ctx: Arc<Context>,
737    desired: StatefulSet,
738) -> Result<StatefulSet> {
739    let namespace = kanidm.namespace().unwrap();
740    let name = desired.name_any();
741    let cached_current = ctx.stores.stateful_set_store.find(|current| {
742        current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
743    });
744
745    // On a cold reflector cache, keep the normal create/update path GET-free. If the optimistic
746    // apply fails with HTTP 422, the object may already exist with an immutable conflict that the
747    // cache has not observed yet. Only 422 errors enter the live classification/recreate recovery
748    // path; 429, 5xx, 403, and transport errors are returned immediately and non-destructively.
749    let cache_miss_422_error = if let Some(cached_current) = cached_current {
750        let mut cached_desired = desired.clone();
751        preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
752        if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
753            == StatefulSetApplyStrategy::Apply
754        {
755            return kanidm.apply(&ctx, cached_desired).await;
756        }
757        None
758    } else {
759        match kanidm.apply(&ctx, desired.clone()).await {
760            Ok(applied) => return Ok(applied),
761            Err(error) => {
762                // Only HTTP 422 can indicate an immutable-field conflict that is safe to
763                // recover from via delete/recreate. Arbitrary failures must not trigger
764                // destruction.
765                if is_kube_422(&error) {
766                    Some(error)
767                } else {
768                    return Err(error);
769                }
770            }
771        }
772    };
773
774    // Never make a destructive decision from reflector state or an SSA error alone. Re-read the
775    // live object and classify it immediately before deletion; another actor or an earlier
776    // reconcile may already have converged the immutable fields while the cache was catching up.
777    let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
778    let Some(live_current) = api
779        .get_opt(&name)
780        .await
781        .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
782    else {
783        return match cache_miss_422_error {
784            Some(error) => Err(error),
785            None => kanidm.apply(&ctx, desired).await,
786        };
787    };
788
789    // Start from the original desired object again. Defaults copied from a stale reflector entry
790    // must not survive into the live re-check and become the reason for a recreation.
791    let mut live_desired = desired.clone();
792    preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
793    match classify_statefulset_change(&live_current, &live_desired) {
794        StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
795            Some(error) => Err(error),
796            None => kanidm.apply(&ctx, live_desired).await,
797        },
798        StatefulSetApplyStrategy::Recreate { .. } => {
799            // An update 422 can combine an immutable conflict with independently-invalid desired
800            // state. Validate the replacement as a dry-run CREATE before deleting the live object,
801            // so create-time validation runs without the existing object's immutable-update check.
802            validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
803
804            // Re-read and reclassify immediately before destruction. Another actor or an earlier
805            // reconcile may have changed the live immutable fields or PVC-retention semantics.
806            let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
807                Error::kube_error(
808                    "re-read before deletion",
809                    "StatefulSet",
810                    &namespace,
811                    &name,
812                    e,
813                )
814            })?
815            else {
816                return kanidm.apply(&ctx, live_desired).await;
817            };
818            let mut latest_desired = live_desired.clone();
819            preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
820
821            match classify_statefulset_change(&latest_current, &latest_desired) {
822                StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
823                StatefulSetApplyStrategy::Recreate { immutable_fields } => {
824                    // Validate the replacement StatefulSet via a dry-run CREATE before deleting
825                    // the live object. This prevents deleting on a mixed 422 (immutable conflict
826                    // + independently-invalid desired state) where the replacement would still
827                    // be rejected by the API.
828                    validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
829                        .await?;
830
831                    info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
832                    kanidm.delete(&ctx, &latest_current).await?;
833                    ctx.kaniop_ctx
834                        .metrics
835                        .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
836                    wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
837                        .await?;
838                    kanidm.apply(&ctx, latest_desired).await
839                }
840            }
841        }
842    }
843}
844
845/// Return true only when the error structurally wraps a Kubernetes HTTP 422 response.
846fn is_kube_422(error: &Error) -> bool {
847    matches!(
848        error,
849        Error::KubeError(_, cause)
850            if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
851    )
852}
853
854#[cfg(test)]
855mod statefulset_recreate_recovery_tests {
856    use super::{Error, is_kube_422};
857
858    fn kube_api_error(code: u16) -> Error {
859        Error::KubeError(
860            "test error".to_string(),
861            Box::new(kube::Error::Api(Box::new(kube::error::Status {
862                code,
863                message: "test".to_string(),
864                reason: "test".to_string(),
865                ..Default::default()
866            }))),
867        )
868    }
869
870    #[test]
871    fn recreate_recovery_accepts_only_422() {
872        assert!(is_kube_422(&kube_api_error(422)));
873        assert!(!is_kube_422(&kube_api_error(403)));
874        assert!(!is_kube_422(&kube_api_error(429)));
875        assert!(!is_kube_422(&kube_api_error(500)));
876    }
877
878    #[test]
879    fn recreate_recovery_rejects_429_too_many_requests() {
880        assert!(!is_kube_422(&kube_api_error(429)));
881    }
882
883    #[test]
884    fn recreate_recovery_rejects_5xx_server_errors() {
885        assert!(!is_kube_422(&kube_api_error(500)));
886        assert!(!is_kube_422(&kube_api_error(502)));
887        assert!(!is_kube_422(&kube_api_error(503)));
888        assert!(!is_kube_422(&kube_api_error(504)));
889    }
890
891    #[test]
892    fn recreate_recovery_rejects_400_and_403() {
893        assert!(!is_kube_422(&kube_api_error(400)));
894        assert!(!is_kube_422(&kube_api_error(403)));
895    }
896}
897
898async fn reconcile(
899    kanidm: Arc<Kanidm>,
900    ctx: Arc<Context>,
901    status: KanidmStatus,
902) -> Result<(Action, bool)> {
903    if kanidm
904        .annotations()
905        .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
906    {
907        ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
908        return Ok((Action::requeue(Duration::from_secs(5)), false));
909    }
910
911    let mut changed = false;
912    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
913    let replication_secret_futures =
914        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
915
916    let sts_to_delete = {
917        let expected_sts_names = kanidm
918            .spec
919            .replica_groups
920            .iter()
921            .map(|rg| kanidm.statefulset_name(&rg.name))
922            .collect::<Vec<_>>();
923        ctx.stores
924            .stateful_set_store
925            .state()
926            .into_iter()
927            .filter(|sts| {
928                sts.namespace() == kanidm.namespace()
929                    && sts
930                        .metadata
931                        .labels
932                        .as_ref()
933                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
934                    && !expected_sts_names.contains(&sts.name_any())
935            })
936            .collect::<Vec<_>>()
937    };
938    if !sts_to_delete.is_empty() {
939        changed = true;
940    }
941    let sts_delete_futures = sts_to_delete
942        .iter()
943        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
944        .collect::<TryJoinAll<_>>();
945
946    let sts_futures = match kanidm.is_updatable(&status) {
947        true => {
948            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
949                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
950            });
951
952            let namespace = kanidm.get_namespace();
953            let schedule_api: Api<KanidmBackupSchedule> =
954                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
955            let schedules = schedule_api
956                .list(&Default::default())
957                .await
958                .map_err(|e| {
959                    Error::KubeError(
960                        "failed to list KanidmBackupSchedules for backup configuration resolution"
961                            .to_string(),
962                        Box::new(e),
963                    )
964                })?
965                .items;
966
967            let repo_api: Api<KanidmBackupRepository> =
968                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
969            let repositories = repo_api
970                .list(&Default::default())
971                .await
972                .map_err(|e| {
973                    Error::KubeError(
974                        "failed to list KanidmBackupRepositories for backup configuration resolution"
975                            .to_string(),
976                        Box::new(e),
977                    )
978                })?
979                .items;
980            let backup_config = resolve_backup_config(&kanidm, &schedules, &repositories);
981
982            kanidm
983                .spec
984                .replica_groups
985                .iter()
986                .map(|rg| {
987                    let sts = kanidm.create_statefulset(
988                        rg,
989                        tls_secret_hash.as_deref(),
990                        backup_config.as_ref(),
991                    )?;
992                    Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
993                })
994                .collect::<Result<TryJoinAll<_>, _>>()?
995        }
996        false => {
997            let note = match status.version.as_ref() {
998                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
999                    format!(
1000                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
1001                        v.image_tag,
1002                        crate::version::KANIDM_CLIENT_VERSION
1003                    )
1004                }
1005                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
1006            };
1007            let _ignore_error = ctx
1008                .kaniop_ctx
1009                .recorder
1010                .publish(
1011                    &Event {
1012                        type_: EventType::Warning,
1013                        reason: "UpgradeBlocked".to_string(),
1014                        note: Some(note),
1015                        action: "ReconcileStatefulSet".to_string(),
1016                        secondary: None,
1017                    },
1018                    &kanidm.object_ref(&()),
1019                )
1020                .await
1021                .map_err(|e| {
1022                    warn!(%e, "failed to publish KanidmError event");
1023                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
1024                });
1025            try_join_all([])
1026        }
1027    };
1028    let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1029
1030    let deprecated_rg_svcs = {
1031        let expected_rg_svcs_names = kanidm
1032            .spec
1033            .replica_groups
1034            .iter()
1035            .filter(|rg| rg.services.is_some())
1036            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1037            .collect::<Vec<_>>();
1038        ctx.stores
1039            .service_store
1040            .state()
1041            .into_iter()
1042            .filter(|svc| {
1043                let labels = svc.metadata.labels.as_ref();
1044                svc.namespace() == kanidm.namespace()
1045                    && !expected_rg_svcs_names.contains(&svc.name_any())
1046                    && labels.is_some_and(|l| {
1047                        l.contains_key(REPLICA_LABEL)
1048                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1049                    })
1050            })
1051            .collect::<Vec<_>>()
1052    };
1053    if !deprecated_rg_svcs.is_empty() {
1054        changed = true;
1055    }
1056    let rg_svcs_delete_futures = deprecated_rg_svcs
1057        .iter()
1058        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1059        .collect::<TryJoinAll<_>>();
1060
1061    let rg_services_futures = kanidm
1062        .spec
1063        .replica_groups
1064        .iter()
1065        .filter(|rg| rg.services.is_some())
1066        .flat_map(|rg| {
1067            (0..rg.replicas).map(|i| {
1068                let mut svc = kanidm.create_replica_group_service(rg, i);
1069                if let Some(current) = ctx.stores.service_store.find(|current| {
1070                    current.namespace() == kanidm.namespace()
1071                        && current.name_any() == svc.name_any()
1072                }) {
1073                    preserve_defaulted_service_fields(&mut svc, current.as_ref());
1074                }
1075                kanidm.patch(&ctx, svc)
1076            })
1077        })
1078        .collect::<TryJoinAll<_>>();
1079
1080    let deprecated_ingresses = {
1081        let names = [
1082            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1083            kanidm.generate_region_ingress_name(),
1084        ]
1085        .into_iter()
1086        .flatten()
1087        .collect::<Vec<_>>();
1088        ctx.stores
1089            .ingress_store
1090            .state()
1091            .into_iter()
1092            .filter(|ing| {
1093                ing.namespace() == kanidm.namespace()
1094                    && !names.contains(&ing.name_any())
1095                    && ing.metadata.labels == Some(kanidm.generate_labels())
1096            })
1097            .collect::<Vec<_>>()
1098    };
1099    if !deprecated_ingresses.is_empty() {
1100        changed = true;
1101    }
1102    let ingress_delete_futures = deprecated_ingresses
1103        .iter()
1104        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1105        .collect::<TryJoinAll<_>>();
1106
1107    let ingress_futures = kanidm
1108        .create_ingress()
1109        .into_iter()
1110        .chain(kanidm.create_region_ingress())
1111        .map(|ingress| kanidm.patch(&ctx, ingress))
1112        .collect::<TryJoinAll<_>>();
1113
1114    let deprecated_http_routes = match &ctx.stores.http_route_store {
1115        Some(store) => {
1116            let expected_names = kanidm
1117                .spec
1118                .gateway
1119                .as_ref()
1120                .map(|_| vec![kanidm.name_any()])
1121                .unwrap_or_default();
1122            let routes: Vec<_> = store
1123                .state()
1124                .into_iter()
1125                .filter(|route| {
1126                    route.namespace() == kanidm.namespace()
1127                        && !expected_names.contains(&route.name_any())
1128                        && route.metadata.labels == Some(kanidm.generate_labels())
1129                })
1130                .collect();
1131            if !routes.is_empty() {
1132                changed = true;
1133            }
1134            routes
1135        }
1136        None => Vec::new(),
1137    };
1138    let http_route_delete_futures = deprecated_http_routes
1139        .iter()
1140        .map(|route| kanidm.delete(&ctx, route.as_ref()))
1141        .collect::<TryJoinAll<_>>();
1142
1143    let http_route_futures = if ctx.stores.http_route_store.is_some() {
1144        kanidm
1145            .create_http_route()
1146            .into_iter()
1147            .map(|route| kanidm.patch(&ctx, route))
1148            .collect::<TryJoinAll<_>>()
1149    } else {
1150        try_join_all(Vec::new())
1151    };
1152
1153    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1154        Some(store) => {
1155            let expected_names = kanidm
1156                .spec
1157                .gateway
1158                .as_ref()
1159                .and_then(|g| g.backend_tls_policy.as_ref())
1160                .map(|_| vec![kanidm.name_any()])
1161                .unwrap_or_default();
1162            let policies: Vec<_> = store
1163                .state()
1164                .into_iter()
1165                .filter(|policy| {
1166                    policy.namespace() == kanidm.namespace()
1167                        && !expected_names.contains(&policy.name_any())
1168                        && policy.metadata.labels == Some(kanidm.generate_labels())
1169                })
1170                .collect();
1171            if !policies.is_empty() {
1172                changed = true;
1173            }
1174            policies
1175        }
1176        None => Vec::new(),
1177    };
1178    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1179        .iter()
1180        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1181        .collect::<TryJoinAll<_>>();
1182
1183    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1184        kanidm
1185            .create_backend_tls_policy()
1186            .into_iter()
1187            .map(|policy| kanidm.patch(&ctx, policy))
1188            .collect::<TryJoinAll<_>>()
1189    } else {
1190        try_join_all(Vec::new())
1191    };
1192
1193    let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1194        sts_delete_futures,
1195        admin_secret_future,
1196        replication_secret_futures,
1197        sts_futures,
1198        service_future,
1199        rg_svcs_delete_futures,
1200        rg_services_futures,
1201        ingress_delete_futures,
1202        ingress_futures,
1203        http_route_delete_futures,
1204        http_route_futures,
1205        backend_tls_policy_delete_futures,
1206        backend_tls_policy_futures
1207    )?;
1208
1209    if admin_secret_changed || replication_secrets_changed {
1210        changed = true;
1211    }
1212
1213    if admin_secret_changed {
1214        // New admin credentials invalidate any cached session of the previous ones.
1215        ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1216    }
1217
1218    if is_kanidm_available(status.clone()) {
1219        let namespace = kanidm.namespace().unwrap();
1220        let name = kanidm.name_any();
1221        match ctx
1222            .kaniop_ctx
1223            .get_kanidm_cluster_client(&kanidm, KanidmUser::Admin)
1224            .await
1225        {
1226            Ok(system_client) => {
1227                if let Err(e) =
1228                    reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1229                {
1230                    warn!(%e, "failed to reconcile domain appearance");
1231                }
1232            }
1233            Err(e) => {
1234                warn!(%e, "failed to create admin client for domain appearance reconciliation");
1235            }
1236        }
1237
1238        match ctx
1239            .kaniop_ctx
1240            .get_kanidm_cluster_client(&kanidm, KanidmUser::IdmAdmin)
1241            .await
1242        {
1243            Ok(kanidm_client) => {
1244                let (mail_sender_status, mail_sender_mutated) =
1245                    reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1246                        .await
1247                        .unwrap_or_else(|e| {
1248                            warn!(%e, "failed to reconcile mail sender");
1249                            (None, false)
1250                        });
1251                if mail_sender_mutated {
1252                    changed = true;
1253                }
1254                let current_mail_sender_status =
1255                    kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1256                if mail_sender_status != current_mail_sender_status {
1257                    let kanidm_api: Api<Kanidm> =
1258                        Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1259                    let status_patch = serde_json::json!({
1260                        "status": {
1261                            "mailSender": mail_sender_status
1262                        }
1263                    });
1264                    if let Err(e) = kanidm_api
1265                        .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1266                        .await
1267                    {
1268                        warn!(%e, "failed to patch Kanidm/status for mail sender");
1269                    }
1270                }
1271            }
1272            Err(e) => {
1273                warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1274            }
1275        }
1276    }
1277
1278    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1279}
1280
1281async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1282    debug!("cleanup");
1283
1284    let mut changed = false;
1285    let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1286    if mail_resources_cleaned {
1287        changed = true;
1288    }
1289
1290    let name = kanidm.name_any();
1291
1292    if let Ok(kanidm_client) = ctx
1293        .kaniop_ctx
1294        .get_kanidm_cluster_client(&kanidm, KanidmUser::IdmAdmin)
1295        .await
1296    {
1297        let kanidm_cleanup_done =
1298            cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1299        if kanidm_cleanup_done {
1300            changed = true;
1301        }
1302    }
1303
1304    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1305    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1306}
1307
1308async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1309    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1310    let start = std::time::Instant::now();
1311    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1312    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1313
1314    loop {
1315        let sts = sts_api.get(sts_name).await.map_err(|e| {
1316            Error::KubeError(
1317                format!("failed to get StatefulSet {namespace}/{sts_name}"),
1318                Box::new(e),
1319            )
1320        })?;
1321
1322        if sts.metadata.deletion_timestamp.is_some() {
1323            return Err(Error::ReceiveOutput(format!(
1324                "StatefulSet {namespace}/{sts_name} is being deleted"
1325            )));
1326        }
1327
1328        let spec = sts.spec.as_ref().ok_or_else(|| {
1329            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1330        })?;
1331
1332        let status = match sts.status {
1333            Some(ref s) => s,
1334            None => {
1335                if start.elapsed() >= timeout {
1336                    return Err(Error::ReceiveOutput(format!(
1337                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1338                    )));
1339                }
1340                sleep(poll_interval).await;
1341                continue;
1342            }
1343        };
1344
1345        let replicas = spec.replicas.unwrap_or(0);
1346
1347        if replicas == 0 {
1348            return Ok(());
1349        }
1350
1351        let updated_replicas = status.updated_replicas.unwrap_or(0);
1352        let current_revision = status.current_revision.as_deref().unwrap_or("");
1353        let update_revision = status.update_revision.as_deref().unwrap_or("");
1354        let ready_replicas = status.ready_replicas.unwrap_or(0);
1355
1356        if updated_replicas == replicas
1357            && current_revision == update_revision
1358            && ready_replicas == replicas
1359        {
1360            return Ok(());
1361        }
1362
1363        if start.elapsed() >= timeout {
1364            return Err(Error::ReceiveOutput(format!(
1365                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1366            )));
1367        }
1368
1369        sleep(poll_interval).await;
1370    }
1371}
1372
1373async fn show_replica_cert_with_retries(
1374    kanidm: &Kanidm,
1375    ctx: Arc<Context>,
1376    pod_name: &str,
1377    max_attempts: u32,
1378) -> std::result::Result<String, Error> {
1379    let mut last_error = None;
1380    for attempt in 0..max_attempts {
1381        if attempt > 0 {
1382            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1383        }
1384        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1385            Ok(cert) => return Ok(cert),
1386            Err(e) => {
1387                info!(
1388                    "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1389                    attempt + 1,
1390                    max_attempts
1391                );
1392                last_error = Some(e);
1393            }
1394        }
1395    }
1396    Err(last_error.unwrap_or_else(|| {
1397        Error::ReceiveOutput(format!(
1398            "failed to get certificate for {pod_name} after {max_attempts} attempts"
1399        ))
1400    }))
1401}
1402
1403impl Kanidm {
1404    // Convenience methods that handle context and operator name
1405    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1406    where
1407        K: Resource<Scope = NamespaceResourceScope>
1408            + Serialize
1409            + Clone
1410            + std::fmt::Debug
1411            + for<'de> Deserialize<'de>,
1412        <K as kube::Resource>::DynamicType: Default,
1413        <K as Resource>::Scope: std::marker::Sized,
1414    {
1415        self.kube_delete(
1416            ctx.kaniop_ctx.client.clone(),
1417            &ctx.kaniop_ctx.metrics,
1418            resource,
1419        )
1420        .await
1421    }
1422
1423    /// Server-side apply without any generic delete/recreate fallback.
1424    /// Resource-specific reconcilers use this when they own lifecycle decisions.
1425    pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1426    where
1427        K: Resource<Scope = NamespaceResourceScope>
1428            + Serialize
1429            + Clone
1430            + std::fmt::Debug
1431            + for<'de> Deserialize<'de>,
1432        <K as kube::Resource>::DynamicType: Default,
1433        <K as Resource>::Scope: std::marker::Sized,
1434    {
1435        self.kube_apply(
1436            ctx.kaniop_ctx.client.clone(),
1437            resource,
1438            KANIDM_OPERATOR_NAME,
1439        )
1440        .await
1441    }
1442
1443    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1444    where
1445        K: Resource<Scope = NamespaceResourceScope>
1446            + Serialize
1447            + Clone
1448            + std::fmt::Debug
1449            + for<'de> Deserialize<'de>,
1450        <K as kube::Resource>::DynamicType: Default,
1451        <K as Resource>::Scope: std::marker::Sized,
1452    {
1453        self.kube_patch(
1454            ctx.kaniop_ctx.client.clone(),
1455            &ctx.kaniop_ctx.metrics,
1456            resource,
1457            KANIDM_OPERATOR_NAME,
1458        )
1459        .await
1460    }
1461
1462    #[inline]
1463    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1464        LABELS
1465            .clone()
1466            .into_iter()
1467            .chain([
1468                (INSTANCE_LABEL.to_string(), self.name_any()),
1469                (CLUSTER_LABEL.to_string(), self.name_any()),
1470            ])
1471            .collect()
1472    }
1473
1474    #[inline]
1475    fn generate_labels(&self) -> BTreeMap<String, String> {
1476        self.generate_resource_labels()
1477    }
1478
1479    #[inline]
1480    fn get_tls_secret_name(&self) -> String {
1481        format!("{}-tls", self.name_any())
1482    }
1483
1484    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
1485    /// falling back to the ingress TLS secret and then to the default name.
1486    pub(crate) fn effective_tls_secret_name(&self) -> String {
1487        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1488            self.spec
1489                .ingress
1490                .as_ref()
1491                .and_then(|i| i.tls_secret_name.clone())
1492                .unwrap_or_else(|| self.get_tls_secret_name())
1493        })
1494    }
1495
1496    #[inline]
1497    fn get_namespace(&self) -> String {
1498        // safe unwrap: Kanidm is namespaced scoped
1499        self.namespace().unwrap()
1500    }
1501
1502    #[inline]
1503    fn is_replication_enabled(&self) -> bool {
1504        self.spec.replica_groups.len() > 1
1505            || self
1506                .spec
1507                .replica_groups
1508                .first()
1509                .is_some_and(|rg| rg.replicas > 1)
1510            || !self.spec.external_replication_nodes.is_empty()
1511    }
1512
1513    #[inline]
1514    fn is_updatable(&self, status: &KanidmStatus) -> bool {
1515        // The explicit upgrade-check bypass must remain effective even when
1516        // version metadata is retained for backup transport.
1517        if self.spec.disable_upgrade_checks {
1518            return true;
1519        }
1520        status
1521            .version
1522            .as_ref()
1523            .map(|v| {
1524                if matches!(
1525                    v.compatibility_result,
1526                    VersionCompatibilityResult::Incompatible
1527                        | VersionCompatibilityResult::NotChecked
1528                ) {
1529                    return false;
1530                }
1531                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1532                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1533                match v.upgrade_check_result {
1534                    KanidmUpgradeCheckResult::Passed => {
1535                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1536                            minor <= v_minor + 1
1537                        } else {
1538                            true
1539                        }
1540                    }
1541                    KanidmUpgradeCheckResult::Failed => {
1542                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1543                            versions
1544                        {
1545                            major == v_major && minor == v_minor && patch >= v_patch
1546                        } else {
1547                            v.image_tag == current_tag
1548                        }
1549                    }
1550                    // Skipped checks cannot count as successful validation.
1551                    KanidmUpgradeCheckResult::Skipped => false,
1552                }
1553            })
1554            .unwrap_or(true)
1555    }
1556
1557    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1558        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1559        let pod = pod_api.get(pod_name).await.map_err(|e| {
1560            Error::KubeError(
1561                format!("failed to get pod {namespace}/{pod_name}"),
1562                Box::new(e),
1563            )
1564        })?;
1565        Ok(pod
1566            .status
1567            .as_ref()
1568            .and_then(|s| s.conditions.as_ref())
1569            .is_some_and(|conditions| {
1570                conditions
1571                    .iter()
1572                    .any(|c| c.type_ == "Ready" && c.status == "True")
1573            }))
1574    }
1575
1576    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1577        let start = std::time::Instant::now();
1578        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1579        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1580
1581        loop {
1582            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1583                Ok(true) => return Ok(()),
1584                Ok(false) => {}
1585                Err(e) => {
1586                    if !e.is_retryable() {
1587                        return Err(e);
1588                    }
1589                    trace!(%e, "pod readiness check failed, retrying");
1590                }
1591            }
1592
1593            if start.elapsed() >= timeout {
1594                return Err(Error::ReceiveOutput(format!(
1595                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1596                )));
1597            }
1598
1599            sleep(poll_interval).await;
1600        }
1601    }
1602
1603    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1604    where
1605        I: IntoIterator<Item = T> + Debug,
1606        T: Into<String>,
1607    {
1608        let namespace = &self.get_namespace();
1609        trace!(
1610            resource.name = &pod_name,
1611            resource.namespace = &namespace,
1612            ?command,
1613            "pod exec"
1614        );
1615        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1616        let attached = pod
1617            .exec(
1618                pod_name,
1619                command,
1620                &AttachParams::default().container("kanidm"),
1621            )
1622            .await
1623            .map_err(|e| {
1624                Error::KubeError(
1625                    format!("failed to exec pod {namespace}/{pod_name}"),
1626                    Box::new(e),
1627                )
1628            })?;
1629        get_output(attached).await
1630    }
1631
1632    async fn exec_with_wait<I, T>(
1633        &self,
1634        ctx: Arc<Context>,
1635        pod_name: &str,
1636        command: I,
1637    ) -> Result<String>
1638    where
1639        I: IntoIterator<Item = T> + Debug,
1640        T: Into<String>,
1641    {
1642        let namespace = &self.get_namespace();
1643        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1644        self.exec(ctx, pod_name, command).await
1645    }
1646
1647    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1648    where
1649        I: IntoIterator<Item = T> + Debug,
1650        T: Into<String>,
1651    {
1652        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1653        let first_replica_group = self
1654            .spec
1655            .replica_groups
1656            .first()
1657            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1658        let sts_name = self.statefulset_name(&first_replica_group.name);
1659        let pod_name = format!("{sts_name}-0");
1660        self.exec(ctx, &pod_name, command).await
1661    }
1662
1663    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1664    where
1665        I: IntoIterator<Item = T> + Debug,
1666        T: Into<String>,
1667    {
1668        let first_replica_group = self
1669            .spec
1670            .replica_groups
1671            .first()
1672            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1673        let sts_name = self.statefulset_name(&first_replica_group.name);
1674        let pod_name = format!("{sts_name}-0");
1675        self.exec_with_wait(ctx, &pod_name, command).await
1676    }
1677}
1678
1679#[cfg(test)]
1680mod test {
1681    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1682    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1683
1684    use crate::controller::State;
1685    use crate::kanidm::controller::context::{Context, Stores};
1686    use crate::kanidm::crd::KanidmStatus;
1687
1688    use kaniop_k8s_util::error::Result;
1689    use kaniop_k8s_util::resources::hash_secret_data;
1690
1691    use std::collections::BTreeMap;
1692    use std::sync::Arc;
1693
1694    use http::{Request, Response};
1695    use k8s_openapi::ByteString;
1696    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1697    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1698    use k8s_openapi::api::networking::v1::Ingress;
1699    use kube::api::ObjectMeta;
1700    use kube::runtime::reflector::store::Writer;
1701    use kube::{Client, Resource, ResourceExt, client::Body};
1702    use opentelemetry::metrics::MeterProvider;
1703    use serde_json::json;
1704
1705    impl Kanidm {
1706        /// A normal test kanidm with a given status
1707        pub fn test() -> Self {
1708            let mut e = Kanidm::new(
1709                "test",
1710                serde_json::from_value(json!({
1711                    "domain": "idm.example.com",
1712                    "replicaGroups": [
1713                        {
1714                            "name": "default",
1715                            "replicas": 1
1716                        }
1717                    ]
1718                }))
1719                .unwrap(),
1720            );
1721            e.meta_mut().namespace = Some("default".into());
1722            // Pre-populate finalizer so reconcile callback runs immediately
1723            // (skips the "add finalizer + wait for next reconcile" step)
1724            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1725            e
1726        }
1727
1728        pub fn with_ingress(mut self) -> Self {
1729            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1730            self
1731        }
1732
1733        /// Modify kanidm replicas
1734        pub fn with_replicas(mut self, replicas: i32) -> Self {
1735            self.spec.replica_groups[0].replicas = replicas;
1736            self
1737        }
1738
1739        /// Modify kanidm to set a deletion timestamp
1740        pub fn needs_delete(mut self) -> Self {
1741            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1742            use k8s_openapi::jiff::Timestamp;
1743            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1744            self.meta_mut().deletion_timestamp = Some(Time(now));
1745            self
1746        }
1747
1748        /// Modify a kanidm to have an expected status
1749        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1750            self.status = Some(status);
1751            self
1752        }
1753    }
1754
1755    // We wrap tower_test::mock::Handle
1756    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1757    pub struct ApiServerVerifier(pub ApiServerHandle);
1758
1759    /// Scenarios we test for in ApiServerVerifier
1760    pub enum Scenario {
1761        Create(Kanidm),
1762        CreateWithTwoReplicas(Kanidm),
1763        CreateWithIngress(Kanidm),
1764        CreateWithIngressWithTwoReplicas(Kanidm),
1765        CreateWithTlsSecret(Kanidm, Box<Secret>),
1766    }
1767
1768    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1769        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1770            .await
1771            .expect("timeout on mock apiserver")
1772            .expect("scenario succeeded")
1773    }
1774
1775    impl ApiServerVerifier {
1776        /// Tests only get to run specific scenarios that has matching handlers
1777        ///
1778        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1779        ///
1780        /// NB: If the controller is making more calls than we are handling in the scenario,
1781        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1782        ///
1783        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1784        /// scenario runs to completion (i.e. all expected calls were responded to),
1785        /// using the timeout to catch missing api calls to Kubernetes.
1786        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1787            tokio::spawn(async move {
1788                // moving self => one scenario per test
1789                match scenario {
1790                    Scenario::Create(kanidm) => {
1791                        self.handle_kanidm_get(kanidm.clone())
1792                            .await
1793                            .unwrap()
1794                            .handle_kanidm_status_patch(kanidm.clone())
1795                            .await
1796                            .unwrap()
1797                            .handle_tls_secret_get_not_found(kanidm.clone())
1798                            .await
1799                            .unwrap()
1800                            .handle_backup_schedule_list()
1801                            .await
1802                            .unwrap()
1803                            .handle_backup_repository_list()
1804                            .await
1805                            .unwrap()
1806                            .handle_statefulset_patch(kanidm.clone(), None)
1807                            .await
1808                            .unwrap()
1809                            .handle_service_patch(kanidm.clone())
1810                            .await
1811                    }
1812                    Scenario::CreateWithTwoReplicas(kanidm) => {
1813                        self.handle_kanidm_get(kanidm.clone())
1814                            .await
1815                            .unwrap()
1816                            .handle_kanidm_status_patch(kanidm.clone())
1817                            .await
1818                            .unwrap()
1819                            .handle_tls_secret_get_not_found(kanidm.clone())
1820                            .await
1821                            .unwrap()
1822                            .handle_backup_schedule_list()
1823                            .await
1824                            .unwrap()
1825                            .handle_backup_repository_list()
1826                            .await
1827                            .unwrap()
1828                            .handle_statefulset_patch(kanidm.clone(), None)
1829                            .await
1830                            .unwrap()
1831                            .handle_service_patch(kanidm.clone())
1832                            .await
1833                    }
1834                    Scenario::CreateWithIngress(kanidm) => {
1835                        self.handle_kanidm_get(kanidm.clone())
1836                            .await
1837                            .unwrap()
1838                            .handle_kanidm_status_patch(kanidm.clone())
1839                            .await
1840                            .unwrap()
1841                            .handle_tls_secret_get_not_found(kanidm.clone())
1842                            .await
1843                            .unwrap()
1844                            .handle_backup_schedule_list()
1845                            .await
1846                            .unwrap()
1847                            .handle_backup_repository_list()
1848                            .await
1849                            .unwrap()
1850                            .handle_statefulset_patch(kanidm.clone(), None)
1851                            .await
1852                            .unwrap()
1853                            .handle_service_patch(kanidm.clone())
1854                            .await
1855                            .unwrap()
1856                            .handle_ingress_patch(kanidm.clone())
1857                            .await
1858                    }
1859                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1860                        self.handle_kanidm_get(kanidm.clone())
1861                            .await
1862                            .unwrap()
1863                            .handle_kanidm_status_patch(kanidm.clone())
1864                            .await
1865                            .unwrap()
1866                            .handle_tls_secret_get_not_found(kanidm.clone())
1867                            .await
1868                            .unwrap()
1869                            .handle_backup_schedule_list()
1870                            .await
1871                            .unwrap()
1872                            .handle_backup_repository_list()
1873                            .await
1874                            .unwrap()
1875                            .handle_statefulset_patch(kanidm.clone(), None)
1876                            .await
1877                            .unwrap()
1878                            .handle_service_patch(kanidm.clone())
1879                            .await
1880                            .unwrap()
1881                            .handle_ingress_patch(kanidm.clone())
1882                            .await
1883                    }
1884                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1885                        let expected_hash = hash_secret_data(&secret);
1886                        self.handle_kanidm_get(kanidm.clone())
1887                            .await
1888                            .unwrap()
1889                            .handle_kanidm_status_patch(kanidm.clone())
1890                            .await
1891                            .unwrap()
1892                            .handle_tls_secret_get(kanidm.clone(), *secret)
1893                            .await
1894                            .unwrap()
1895                            .handle_backup_schedule_list()
1896                            .await
1897                            .unwrap()
1898                            .handle_backup_repository_list()
1899                            .await
1900                            .unwrap()
1901                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1902                            .await
1903                            .unwrap()
1904                            .handle_service_patch(kanidm.clone())
1905                            .await
1906                    }
1907                }
1908                .expect("scenario completed without errors");
1909            })
1910        }
1911
1912        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1913            let (request, send) = self.0.next_request().await.expect("service not called");
1914            assert_eq!(request.method(), http::Method::GET);
1915            let expected_uri_prefix = format!(
1916                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1917                kanidm.name_any()
1918            );
1919            let uri = request.uri().to_string();
1920            assert!(
1921                uri.starts_with(&expected_uri_prefix),
1922                "expected uri to start with {}, got {}",
1923                expected_uri_prefix,
1924                uri
1925            );
1926            let response = serde_json::to_vec(&kanidm).unwrap();
1927            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1928            Ok(self)
1929        }
1930
1931        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1932            let (request, send) = self.0.next_request().await.expect("service not called");
1933            assert_eq!(request.method(), http::Method::PATCH);
1934            assert_eq!(
1935                request.uri().to_string(),
1936                format!(
1937                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1938                    kanidm.name_any()
1939                )
1940            );
1941
1942            let req_body = request.into_body().collect_bytes().await.unwrap();
1943            let json: serde_json::Value =
1944                serde_json::from_slice(&req_body).expect("patch object is json");
1945            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1946                .expect("valid kanidm status");
1947            let mut kanidm = Kanidm::test();
1948            kanidm.status = Some(status.clone());
1949            let response = serde_json::to_vec(&kanidm).unwrap();
1950            // pass through kanidm "patch accepted"
1951            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1952            Ok(self)
1953        }
1954
1955        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1956            let (request, send) = self.0.next_request().await.expect("service not called");
1957            assert_eq!(request.method(), http::Method::GET);
1958            let expected_uri_prefix = format!(
1959                "/api/v1/namespaces/default/secrets/{}",
1960                kanidm.effective_tls_secret_name()
1961            );
1962            let uri = request.uri().to_string();
1963            assert!(
1964                uri.starts_with(&expected_uri_prefix),
1965                "expected uri to start with {expected_uri_prefix}, got {uri}"
1966            );
1967            let status = serde_json::json!({
1968                "kind": "Status",
1969                "apiVersion": "v1",
1970                "metadata": {},
1971                "status": "Failure",
1972                "message": "secret not found",
1973                "reason": "NotFound",
1974                "code": 404
1975            });
1976            send.send_response(
1977                Response::builder()
1978                    .status(404)
1979                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1980                    .unwrap(),
1981            );
1982            Ok(self)
1983        }
1984
1985        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1986            let (request, send) = self.0.next_request().await.expect("service not called");
1987            assert_eq!(request.method(), http::Method::GET);
1988            let expected_uri_prefix = format!(
1989                "/api/v1/namespaces/default/secrets/{}",
1990                kanidm.effective_tls_secret_name()
1991            );
1992            let uri = request.uri().to_string();
1993            assert!(
1994                uri.starts_with(&expected_uri_prefix),
1995                "expected uri to start with {expected_uri_prefix}, got {uri}"
1996            );
1997            let response = serde_json::to_vec(&secret).unwrap();
1998            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1999            Ok(self)
2000        }
2001
2002        async fn handle_backup_schedule_list(mut self) -> Result<Self> {
2003            let (request, send) = self.0.next_request().await.expect("service not called");
2004            assert_eq!(request.method(), http::Method::GET);
2005            assert!(
2006                request
2007                    .uri()
2008                    .to_string()
2009                    .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
2010                "expected backup schedule list request, got {}",
2011                request.uri()
2012            );
2013            let list = json!({
2014                "apiVersion": "kaniop.rs/v1alpha1",
2015                "kind": "KanidmBackupScheduleList",
2016                "metadata": {},
2017                "items": []
2018            });
2019            let response = serde_json::to_vec(&list).unwrap();
2020            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2021            Ok(self)
2022        }
2023
2024        async fn handle_backup_repository_list(mut self) -> Result<Self> {
2025            let (request, send) = self.0.next_request().await.expect("service not called");
2026            assert_eq!(request.method(), http::Method::GET);
2027            assert!(
2028                request.uri().to_string().contains(
2029                    "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2030                ),
2031                "expected backup repository list request, got {}",
2032                request.uri()
2033            );
2034            let list = json!({
2035                "apiVersion": "kaniop.rs/v1alpha1",
2036                "kind": "KanidmBackupRepositoryList",
2037                "metadata": {},
2038                "items": []
2039            });
2040            let response = serde_json::to_vec(&list).unwrap();
2041            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2042            Ok(self)
2043        }
2044
2045        async fn handle_statefulset_patch(
2046            mut self,
2047            kanidm: Kanidm,
2048            expected_tls_hash: Option<String>,
2049        ) -> Result<Self> {
2050            for rg in kanidm.spec.replica_groups.iter() {
2051                let (request, send) = self.0.next_request().await.expect("service not called");
2052                assert_eq!(request.method(), http::Method::PATCH);
2053                assert_eq!(
2054                    request.uri().to_string(),
2055                    format!(
2056                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2057                        kanidm.statefulset_name(&rg.name)
2058                    )
2059                );
2060                let req_body = request.into_body().collect_bytes().await.unwrap();
2061                let json: serde_json::Value =
2062                    serde_json::from_slice(&req_body).expect("patch object is json");
2063                let statefulset: StatefulSet =
2064                    serde_json::from_value(json).expect("valid statefulset");
2065                assert_eq!(
2066                    statefulset.clone().spec.unwrap().replicas.unwrap(),
2067                    rg.replicas
2068                );
2069                let tls_hash_annotation = statefulset
2070                    .spec
2071                    .as_ref()
2072                    .unwrap()
2073                    .template
2074                    .metadata
2075                    .as_ref()
2076                    .unwrap()
2077                    .annotations
2078                    .as_ref()
2079                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2080                    .cloned();
2081                assert_eq!(tls_hash_annotation, expected_tls_hash);
2082                let response = serde_json::to_vec(&statefulset).unwrap();
2083                // pass through kanidm "patch accepted"
2084                send.send_response(Response::builder().body(Body::from(response)).unwrap());
2085            }
2086
2087            Ok(self)
2088        }
2089
2090        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2091            let (request, send) = self.0.next_request().await.expect("service not called");
2092            assert_eq!(request.method(), http::Method::PATCH);
2093            assert_eq!(
2094                request.uri().to_string(),
2095                format!(
2096                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2097                    kanidm.name_any()
2098                )
2099            );
2100
2101            let req_body = request.into_body().collect_bytes().await.unwrap();
2102            let json: serde_json::Value =
2103                serde_json::from_slice(&req_body).expect("patch object is json");
2104            let service: Service = serde_json::from_value(json).expect("valid service");
2105            let response = serde_json::to_vec(&service).unwrap();
2106            // pass through kanidm "patch accepted"
2107            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2108            Ok(self)
2109        }
2110
2111        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2112            let (request, send) = self.0.next_request().await.expect("service not called");
2113            assert_eq!(request.method(), http::Method::PATCH);
2114            assert_eq!(
2115                request.uri().to_string(),
2116                format!(
2117                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2118                    kanidm.name_any()
2119                )
2120            );
2121
2122            let req_body = request.into_body().collect_bytes().await.unwrap();
2123            let json: serde_json::Value =
2124                serde_json::from_slice(&req_body).expect("patch object is json");
2125            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2126            let response = serde_json::to_vec(&ingress).unwrap();
2127            // pass through kanidm "patch accepted"
2128            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2129            Ok(self)
2130        }
2131    }
2132
2133    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2134        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2135        let mock_client = Client::new(mock_service, "default");
2136        let stores = Stores {
2137            stateful_set_store: Writer::default().as_reader(),
2138            service_store: Writer::default().as_reader(),
2139            ingress_store: Writer::default().as_reader(),
2140            secret_store: Writer::default().as_reader(),
2141            http_route_store: Some(Writer::default().as_reader()),
2142            backend_tls_policy_store: Some(Writer::default().as_reader()),
2143            deployment_store: Writer::default().as_reader(),
2144            config_map_store: Writer::default().as_reader(),
2145        };
2146        let controller_id = "test";
2147
2148        // Create a test meter for metrics
2149        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2150        let meter = provider.meter("test");
2151        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2152
2153        let state = State::new(
2154            metrics,
2155            Writer::default().as_reader(),
2156            Writer::default().as_reader(),
2157            None,
2158        );
2159        let ctx = Arc::new(Context::new(
2160            state.to_context(mock_client, controller_id),
2161            stores,
2162        ));
2163        (ctx, ApiServerVerifier(handle))
2164    }
2165
2166    #[tokio::test]
2167    async fn kanidm_create() {
2168        let (testctx, fakeserver) = get_test_context();
2169        let kanidm = Kanidm::test();
2170        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2171        reconcile_kanidm(Arc::new(kanidm), testctx)
2172            .await
2173            .expect("reconciler");
2174        timeout_after_1s(mocksrv).await;
2175    }
2176
2177    #[tokio::test]
2178    async fn kanidm_create_with_two_replicas() {
2179        let (testctx, fakeserver) = get_test_context();
2180        let kanidm = Kanidm::test().with_replicas(2);
2181        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2182        reconcile_kanidm(Arc::new(kanidm), testctx)
2183            .await
2184            .expect("reconciler");
2185        timeout_after_1s(mocksrv).await;
2186    }
2187
2188    #[tokio::test]
2189    async fn kanidm_create_with_ingress() {
2190        let (testctx, fakeserver) = get_test_context();
2191        let kanidm = Kanidm::test().with_ingress();
2192        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2193        reconcile_kanidm(Arc::new(kanidm), testctx)
2194            .await
2195            .expect("reconciler");
2196        timeout_after_1s(mocksrv).await;
2197    }
2198
2199    #[tokio::test]
2200    async fn kanidm_create_with_ingress_with_two_replicas() {
2201        let (testctx, fakeserver) = get_test_context();
2202        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2203        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2204        reconcile_kanidm(Arc::new(kanidm), testctx)
2205            .await
2206            .expect("reconciler");
2207        timeout_after_1s(mocksrv).await;
2208    }
2209
2210    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2211        Arc::new(StatefulSet {
2212            metadata: ObjectMeta {
2213                name: Some(format!("{cluster}-default")),
2214                namespace: Some(namespace.to_string()),
2215                labels: Some(BTreeMap::from([(
2216                    CLUSTER_LABEL.to_string(),
2217                    cluster.to_string(),
2218                )])),
2219                ..Default::default()
2220            },
2221            spec: Some(StatefulSetSpec {
2222                template: PodTemplateSpec {
2223                    metadata: Some(ObjectMeta {
2224                        annotations: tls_hash.map(|h| {
2225                            BTreeMap::from([(
2226                                TLS_SECRET_HASH_ANNOTATION.to_string(),
2227                                h.to_string(),
2228                            )])
2229                        }),
2230                        ..Default::default()
2231                    }),
2232                    ..Default::default()
2233                },
2234                ..Default::default()
2235            }),
2236            ..Default::default()
2237        })
2238    }
2239
2240    #[test]
2241    fn test_previous_tls_secret_hash() {
2242        let kanidm = Kanidm::test();
2243
2244        // annotation carried over from an existing statefulset
2245        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2246        assert_eq!(
2247            previous_tls_secret_hash(&kanidm, stss),
2248            Some("abc123".to_string())
2249        );
2250
2251        // no annotation on the existing statefulset
2252        let stss = vec![statefulset_of("test", "default", None)];
2253        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2254
2255        // statefulsets from another cluster or namespace are ignored
2256        let stss = vec![
2257            statefulset_of("other", "default", Some("abc123")),
2258            statefulset_of("test", "other-ns", Some("abc123")),
2259        ];
2260        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2261    }
2262
2263    #[tokio::test]
2264    async fn kanidm_create_with_tls_secret() {
2265        let (testctx, fakeserver) = get_test_context();
2266        let kanidm = Kanidm::test();
2267        let secret = Secret {
2268            metadata: ObjectMeta {
2269                name: Some(kanidm.effective_tls_secret_name()),
2270                namespace: Some("default".to_string()),
2271                ..Default::default()
2272            },
2273            data: Some(BTreeMap::from([
2274                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2275                ("tls.key".to_string(), ByteString(b"key".to_vec())),
2276            ])),
2277            ..Default::default()
2278        };
2279        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2280            kanidm.clone(),
2281            Box::new(secret),
2282        ));
2283        reconcile_kanidm(Arc::new(kanidm), testctx)
2284            .await
2285            .expect("reconciler");
2286        timeout_after_1s(mocksrv).await;
2287    }
2288
2289    #[tokio::test]
2290    async fn replication_secret_gc_skips_non_replica_secrets() {
2291        use super::reconcile_replication_secrets;
2292        use super::secret::{SECRET_TYPE_LABEL, SecretType};
2293
2294        let kanidm = Kanidm::test();
2295        let status = kanidm.status.clone().unwrap_or_default();
2296
2297        let replica_cert_label =
2298            serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
2299
2300        let replica_cert_secret = Secret {
2301            metadata: ObjectMeta {
2302                name: Some(format!("{}-default-0-cert", kanidm.name_any())),
2303                namespace: Some("default".into()),
2304                labels: Some(BTreeMap::from([
2305                    (CLUSTER_LABEL.to_string(), kanidm.name_any()),
2306                    (SECRET_TYPE_LABEL.to_string(), replica_cert_label.clone()),
2307                ])),
2308                ..Default::default()
2309            },
2310            ..Default::default()
2311        };
2312
2313        let mail_sender_secret = Secret {
2314            metadata: ObjectMeta {
2315                name: Some(format!("{}-mail-sender-config", kanidm.name_any())),
2316                namespace: Some("default".into()),
2317                labels: Some(BTreeMap::from([(
2318                    CLUSTER_LABEL.to_string(),
2319                    kanidm.name_any(),
2320                )])),
2321                ..Default::default()
2322            },
2323            ..Default::default()
2324        };
2325
2326        let (mock_service, mut handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2327        let mock_client = Client::new(mock_service, "default");
2328
2329        let mut secret_writer = Writer::default();
2330        secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2331            replica_cert_secret.clone(),
2332        ));
2333        secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2334            mail_sender_secret.clone(),
2335        ));
2336
2337        let stores = Stores {
2338            stateful_set_store: Writer::default().as_reader(),
2339            service_store: Writer::default().as_reader(),
2340            ingress_store: Writer::default().as_reader(),
2341            secret_store: secret_writer.as_reader(),
2342            http_route_store: Some(Writer::default().as_reader()),
2343            backend_tls_policy_store: Some(Writer::default().as_reader()),
2344            deployment_store: Writer::default().as_reader(),
2345            config_map_store: Writer::default().as_reader(),
2346        };
2347
2348        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2349        let meter = provider.meter("test");
2350        let metrics = crate::metrics::Metrics::new(&meter, &["test"]);
2351        let state = State::new(
2352            metrics,
2353            Writer::default().as_reader(),
2354            Writer::default().as_reader(),
2355            None,
2356        );
2357        let ctx = Arc::new(Context::new(state.to_context(mock_client, "test"), stores));
2358
2359        let mock_srv = tokio::spawn(async move {
2360            let (request, send) = handle.next_request().await.expect("expected a request");
2361            assert_eq!(request.method(), http::Method::DELETE);
2362            let uri = request.uri().to_string();
2363            assert!(
2364                uri.contains("-default-0-cert"),
2365                "expected delete for replica cert secret, got {uri}"
2366            );
2367            let not_found = serde_json::json!({
2368                "kind": "Status",
2369                "apiVersion": "v1",
2370                "metadata": {},
2371                "status": "Failure",
2372                "message": "not found",
2373                "reason": "NotFound",
2374                "code": 404
2375            });
2376            send.send_response(
2377                Response::builder()
2378                    .status(404)
2379                    .body(Body::from(serde_json::to_vec(&not_found).unwrap()))
2380                    .unwrap(),
2381            );
2382
2383            if let Ok(Some((req, _))) =
2384                tokio::time::timeout(std::time::Duration::from_millis(200), handle.next_request())
2385                    .await
2386            {
2387                panic!("unexpected second request: {} {}", req.method(), req.uri());
2388            }
2389        });
2390
2391        let changed = reconcile_replication_secrets(Arc::new(kanidm), ctx, &status)
2392            .await
2393            .expect("reconcile succeeds despite 404 on delete");
2394        assert!(
2395            changed,
2396            "expected changed=true since a deprecated secret was found"
2397        );
2398
2399        tokio::time::timeout(std::time::Duration::from_secs(1), mock_srv)
2400            .await
2401            .expect("timeout waiting for mock server")
2402            .expect("mock server task succeeded");
2403    }
2404
2405    #[test]
2406    fn disable_upgrade_checks_allows_unchecked_version_status() {
2407        let mut kanidm = Kanidm::test();
2408        kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2409        kanidm.spec.disable_upgrade_checks = true;
2410        let status = KanidmStatus {
2411            version: Some(crate::kanidm::crd::KanidmVersionStatus {
2412                image_tag: "1.11.2".to_string(),
2413                upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Skipped,
2414                compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::NotChecked,
2415            }),
2416            ..Default::default()
2417        };
2418        assert!(kanidm.is_updatable(&status));
2419    }
2420
2421    #[test]
2422    fn disable_upgrade_checks_allows_previous_incompatible_status() {
2423        let mut kanidm = Kanidm::test();
2424        kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2425        kanidm.spec.disable_upgrade_checks = true;
2426        let status = KanidmStatus {
2427            version: Some(crate::kanidm::crd::KanidmVersionStatus {
2428                image_tag: "1.11.2".to_string(),
2429                upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Passed,
2430                compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::Incompatible,
2431            }),
2432            ..Default::default()
2433        };
2434        assert!(kanidm.is_updatable(&status));
2435    }
2436
2437    #[test]
2438    fn upgrade_checks_still_block_incompatible_versions_when_enabled() {
2439        let mut kanidm = Kanidm::test();
2440        kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2441        kanidm.spec.disable_upgrade_checks = false;
2442        let status = KanidmStatus {
2443            version: Some(crate::kanidm::crd::KanidmVersionStatus {
2444                image_tag: "1.11.2".to_string(),
2445                upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Passed,
2446                compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::Incompatible,
2447            }),
2448            ..Default::default()
2449        };
2450        assert!(!kanidm.is_updatable(&status));
2451    }
2452
2453    #[test]
2454    fn upgrade_checks_do_not_treat_skipped_checks_as_passed() {
2455        let mut kanidm = Kanidm::test();
2456        kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2457        kanidm.spec.disable_upgrade_checks = false;
2458        let status = KanidmStatus {
2459            version: Some(crate::kanidm::crd::KanidmVersionStatus {
2460                image_tag: "1.11.2".to_string(),
2461                upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Skipped,
2462                compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::NotChecked,
2463            }),
2464            ..Default::default()
2465        };
2466        assert!(!kanidm.is_updatable(&status));
2467    }
2468}