Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::SecretExt;
20use self::service::ServiceExt;
21use self::statefulset::{
22    StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23    preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_transport_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
31use crate::kanidm::crd::{
32    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
33};
34use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
35use crate::telemetry;
36
37use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
38use kaniop_k8s_util::client::get_output;
39use kaniop_k8s_util::error::{Error, Result};
40use kaniop_k8s_util::parse::parse_semver;
41use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::fmt::Debug;
45use std::sync::{Arc, LazyLock};
46
47use futures::future::{TryJoinAll, join_all, try_join_all};
48use futures::stream::{self, StreamExt};
49use futures::try_join;
50use k8s_openapi::NamespaceResourceScope;
51use k8s_openapi::api::apps::v1::StatefulSet;
52use k8s_openapi::api::core::v1::{Pod, Secret, Service};
53use kube::Resource;
54use kube::ResourceExt;
55use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
56use kube::runtime::controller::Action;
57use kube::runtime::events::{Event, EventType};
58use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
59use serde::{Deserialize, Serialize};
60use tokio::time::{Duration, sleep};
61use tracing::{Span, debug, field, info, instrument, trace, warn};
62
63const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
64const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
65const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
66const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
67const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
68const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
69const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
70const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
71const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
72
73pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
74const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
75pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
76
77const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
78
79static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
80    BTreeMap::from([
81        (NAME_LABEL.to_string(), "kanidm".to_string()),
82        (
83            MANAGED_BY_LABEL.to_string(),
84            format!("kaniop-{CONTROLLER_ID}"),
85        ),
86    ])
87});
88
89async fn restart_statefulsets_after_replica_secret_update(
90    ctx: Arc<Context>,
91    sts_api: &Api<StatefulSet>,
92    namespace: &str,
93    statefulset_names: &[String],
94    sequential: bool,
95) -> Result<()> {
96    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
97    // Pods must restart after cert secret changes or replication keeps stale TLS material.
98    let restart_futures = statefulset_names
99        .iter()
100        .map(|sts_name| sts_api.restart(sts_name));
101
102    if sequential {
103        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
104        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
105            return Err(Error::kube_error(
106                "restart",
107                "StatefulSet",
108                namespace,
109                "replica certificate secret update statefulsets",
110                first_err,
111            ));
112        }
113    } else {
114        let results: Vec<_> = join_all(restart_futures).await;
115        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
116            return Err(Error::kube_error(
117                "restart",
118                "StatefulSet",
119                namespace,
120                "replica certificate secret update statefulsets",
121                first_err,
122            ));
123        }
124    }
125
126    let rollout_futures = statefulset_names
127        .iter()
128        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
129    try_join_all(rollout_futures).await?;
130
131    Ok(())
132}
133
134pub async fn reconcile_admins_secret(
135    kanidm: Arc<Kanidm>,
136    ctx: Arc<Context>,
137    status: &KanidmStatus,
138) -> Result<bool> {
139    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
140        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
141        kanidm.patch(&ctx, admins_secret).await?;
142        return Ok(true);
143    }
144    Ok(false)
145}
146
147pub async fn reconcile_replication_secrets(
148    kanidm: Arc<Kanidm>,
149    ctx: Arc<Context>,
150    status: &KanidmStatus,
151) -> Result<bool> {
152    let mut changed = false;
153    let expected_secret_names = if kanidm.is_replication_enabled() {
154        status
155            .replica_statuses
156            .iter()
157            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
158            .collect::<Vec<_>>()
159    } else {
160        Vec::new()
161    };
162    let deprecated_secrets = ctx
163        .stores
164        .secret_store
165        .state()
166        .into_iter()
167        .filter(|secret| {
168            secret.namespace() == kanidm.namespace()
169                && kanidm.admins_secret_name() != secret.name_any()
170                && !expected_secret_names.contains(&secret.name_any())
171                && secret
172                    .metadata
173                    .labels
174                    .as_ref()
175                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
176        })
177        .collect::<Vec<_>>();
178    if !deprecated_secrets.is_empty() {
179        changed = true;
180    }
181    let secret_delete_future = deprecated_secrets
182        .iter()
183        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
184        .collect::<TryJoinAll<_>>();
185    try_join!(secret_delete_future)?;
186
187    if kanidm.is_replication_enabled() {
188        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
189        let statefulset_names = status
190            .replica_statuses
191            .iter()
192            .map(|rs| rs.statefulset_name.clone())
193            .collect::<BTreeSet<_>>()
194            .into_iter()
195            .collect::<Vec<_>>();
196
197        let has_pending = status
198            .replica_statuses
199            .iter()
200            .any(|rs| rs.state == KanidmReplicaState::Pending);
201
202        let sts_api =
203            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
204
205        if has_pending {
206            changed = true;
207            stream::iter(
208                status
209                    .replica_statuses
210                    .iter()
211                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
212            )
213            .then(|rs| async {
214                let secret = kanidm
215                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
216                    .await?;
217                kanidm.patch(&ctx, secret).await
218            })
219            .collect::<Vec<_>>()
220            .await
221            .into_iter()
222            .collect::<Result<Vec<_>>>()?;
223
224            let restart_futures = status
225                .replica_statuses
226                .iter()
227                .map(|rs| sts_api.restart(&rs.statefulset_name));
228
229            if has_single_replica {
230                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
231                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
232                    return Err(Error::kube_error(
233                        "restart",
234                        "StatefulSet",
235                        kanidm.get_namespace(),
236                        "pending replica statefulsets",
237                        first_err,
238                    ));
239                }
240            } else {
241                let results: Vec<_> = join_all(restart_futures).await;
242                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
243                    return Err(Error::kube_error(
244                        "restart",
245                        "StatefulSet",
246                        kanidm.get_namespace(),
247                        "pending replica statefulsets",
248                        first_err,
249                    ));
250                }
251            }
252        }
253
254        let has_certificate_host_invalid = status
255            .replica_statuses
256            .iter()
257            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
258
259        let has_certificate_expiring = status
260            .replica_statuses
261            .iter()
262            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
263
264        let has_non_ready_replicas = status
265            .replica_statuses
266            .iter()
267            .any(|rs| rs.state != KanidmReplicaState::Ready);
268
269        let host_invalid_replicas: Vec<_> = status
270            .replica_statuses
271            .iter()
272            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
273            .collect();
274
275        let cert_expiring_replicas: Vec<_> = status
276            .replica_statuses
277            .iter()
278            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
279            .collect();
280
281        if !host_invalid_replicas.is_empty() {
282            changed = true;
283            let restart_futures = host_invalid_replicas
284                .iter()
285                .map(|rs| sts_api.restart(&rs.statefulset_name));
286
287            if has_single_replica {
288                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
289                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
290                    return Err(Error::kube_error(
291                        "restart",
292                        "StatefulSet",
293                        kanidm.get_namespace(),
294                        "certificate host invalid statefulsets",
295                        first_err,
296                    ));
297                }
298            } else {
299                let results: Vec<_> = join_all(restart_futures).await;
300                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
301                    return Err(Error::kube_error(
302                        "restart",
303                        "StatefulSet",
304                        kanidm.get_namespace(),
305                        "certificate host invalid statefulsets",
306                        first_err,
307                    ));
308                }
309            }
310
311            let restarted_sts_names: Vec<_> = host_invalid_replicas
312                .iter()
313                .map(|rs| rs.statefulset_name.clone())
314                .collect::<BTreeSet<_>>()
315                .into_iter()
316                .collect();
317
318            let namespace = kanidm.get_namespace();
319            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
320                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
321            });
322            try_join_all(rollout_futures).await?;
323
324            let renew_futures = host_invalid_replicas
325                .iter()
326                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
327            let results: Vec<_> = join_all(renew_futures).await;
328            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
329                return Err(first_err);
330            }
331
332            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
333
334            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
335                let cert = show_replica_cert_with_retries(
336                    &kanidm,
337                    ctx.clone(),
338                    &rs.pod_name,
339                    CERT_SHOW_RETRY_ATTEMPTS,
340                )
341                .await?;
342                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
343                kanidm.patch(&ctx, secret.clone()).await
344            });
345            try_join_all(cert_futures).await?;
346
347            restart_statefulsets_after_replica_secret_update(
348                ctx.clone(),
349                &sts_api,
350                &kanidm.get_namespace(),
351                &statefulset_names,
352                has_single_replica,
353            )
354            .await?;
355
356            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
357        }
358
359        if !cert_expiring_replicas.is_empty() {
360            changed = true;
361            let renew_futures = cert_expiring_replicas
362                .iter()
363                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
364            let results: Vec<_> = join_all(renew_futures).await;
365            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
366                return Err(first_err);
367            }
368
369            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
370
371            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
372                let cert = show_replica_cert_with_retries(
373                    &kanidm,
374                    ctx.clone(),
375                    &rs.pod_name,
376                    CERT_SHOW_RETRY_ATTEMPTS,
377                )
378                .await?;
379                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
380                kanidm.patch(&ctx, secret.clone()).await
381            });
382            try_join_all(cert_futures).await?;
383
384            restart_statefulsets_after_replica_secret_update(
385                ctx.clone(),
386                &sts_api,
387                &kanidm.get_namespace(),
388                &statefulset_names,
389                has_single_replica,
390            )
391            .await?;
392
393            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
394        }
395
396        if has_non_ready_replicas
397            && !has_pending
398            && !has_certificate_host_invalid
399            && !has_certificate_expiring
400        {
401            changed = true;
402            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
403
404            let restart_futures = status
405                .replica_statuses
406                .iter()
407                .map(|rs| sts_api.restart(&rs.statefulset_name));
408
409            if has_single_replica {
410                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
411                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
412                    return Err(Error::kube_error(
413                        "restart",
414                        "StatefulSet",
415                        kanidm.get_namespace(),
416                        "replica statefulsets",
417                        first_err,
418                    ));
419                }
420            } else {
421                let results: Vec<_> = join_all(restart_futures).await;
422                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
423                    return Err(Error::kube_error(
424                        "restart",
425                        "StatefulSet",
426                        kanidm.get_namespace(),
427                        "replica statefulsets",
428                        first_err,
429                    ));
430                }
431            }
432        }
433    }
434
435    Ok(changed)
436}
437
438#[instrument(skip(ctx, kanidm))]
439pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
440    let trace_id = telemetry::get_trace_id();
441    Span::current().record("trace_id", field::display(&trace_id));
442    let _timer = ctx
443        .kaniop_ctx
444        .metrics
445        .reconcile_count_and_measure(&trace_id);
446    info!("reconciling Kanidm");
447
448    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
449        debug!(%e, "failed to reconcile status");
450        ctx.kaniop_ctx.metrics.status_update_errors_inc();
451        e
452    })?;
453    let kanidm_api: Api<Kanidm> =
454        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
455    let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
456    let outcome_clone = outcome.clone();
457    let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
458        let outcome = outcome_clone.clone();
459        let ctx = ctx.clone();
460        let status = status.clone();
461        async move {
462            match event {
463                Finalizer::Apply(kanidm) => {
464                    let (action, changed) = reconcile(kanidm, ctx, status).await?;
465                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
466                    Ok(action)
467                }
468                Finalizer::Cleanup(kanidm) => {
469                    let (action, changed) = cleanup(kanidm, ctx).await?;
470                    outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
471                    Ok(action)
472                }
473            }
474        }
475    })
476    .await
477    .or_else(|e| match e {
478        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
479            debug!("resource already removed during finalizer cleanup");
480            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
481        }
482        _ => Err(Error::FinalizerError(
483            "failed on kanidm account finalizer".to_string(),
484            Box::new(e),
485        )),
486    })?;
487    let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
488    Ok((action, changed))
489}
490
491/// Hash of the TLS secret content, injected as a pod template annotation so
492/// certificate renewals roll the statefulsets. `None` if the secret does not
493/// exist yet.
494async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
495    let secret_name = kanidm.effective_tls_secret_name();
496    let secret_api =
497        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
498    let secret = secret_api
499        .get_opt(&secret_name)
500        .await
501        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
502    Ok(secret.as_ref().map(hash_secret_data))
503}
504
505/// Previous TLS secret hash annotation from the existing statefulsets.
506///
507/// Used when the TLS secret is missing: dropping the annotation would roll
508/// the pods onto a missing volume and leave them stuck in ContainerCreating.
509fn previous_tls_secret_hash(
510    kanidm: &Kanidm,
511    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
512) -> Option<String> {
513    stateful_sets
514        .into_iter()
515        .filter(|sts| {
516            sts.namespace() == kanidm.namespace()
517                && sts
518                    .metadata
519                    .labels
520                    .as_ref()
521                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
522        })
523        .find_map(|sts| {
524            sts.spec
525                .as_ref()?
526                .template
527                .metadata
528                .as_ref()?
529                .annotations
530                .as_ref()?
531                .get(TLS_SECRET_HASH_ANNOTATION)
532                .cloned()
533        })
534}
535
536fn is_headless_service(service: &Service) -> bool {
537    service
538        .spec
539        .as_ref()
540        .and_then(|spec| spec.cluster_ip.as_deref())
541        == Some("None")
542}
543
544fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
545    if is_headless_service(desired) != is_headless_service(current) {
546        return;
547    }
548    let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
549    else {
550        return;
551    };
552    if desired_spec.cluster_ip.is_none() {
553        desired_spec.cluster_ip = current_spec.cluster_ip.clone();
554    }
555    if desired_spec.cluster_ips.is_none() {
556        desired_spec.cluster_ips = current_spec.cluster_ips.clone();
557    }
558}
559
560async fn wait_for_statefulset_deletion(
561    client: kube::Client,
562    namespace: &str,
563    name: &str,
564) -> Result<()> {
565    let api = Api::<StatefulSet>::namespaced(client, namespace);
566    let start = std::time::Instant::now();
567    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
568    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
569
570    loop {
571        match api.get_opt(name).await {
572            Ok(None) => return Ok(()),
573            Ok(Some(_)) => {}
574            Err(e) => {
575                return Err(Error::kube_error(
576                    "wait for deletion of",
577                    "StatefulSet",
578                    namespace,
579                    name,
580                    e,
581                ));
582            }
583        }
584
585        if start.elapsed() >= timeout {
586            return Err(Error::ReceiveOutput(format!(
587                "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
588            )));
589        }
590        sleep(poll_interval).await;
591    }
592}
593
594async fn reconcile_service(
595    kanidm: &Kanidm,
596    ctx: &Context,
597    mut desired: Service,
598) -> Result<Service> {
599    let namespace = kanidm.get_namespace();
600    let name = desired.name_any();
601    let desired_headless = is_headless_service(&desired);
602
603    if let Some(cached) = ctx
604        .stores
605        .service_store
606        .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
607    {
608        if is_headless_service(cached.as_ref()) == desired_headless {
609            preserve_defaulted_service_fields(&mut desired, cached.as_ref());
610        } else {
611            let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
612            let current = service_api
613                .get_opt(&name)
614                .await
615                .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
616
617            if let Some(current) = current {
618                let current_headless = is_headless_service(&current);
619                if current_headless != desired_headless {
620                    info!(
621                        resource.name = &name,
622                        resource.namespace = &namespace,
623                        current_headless,
624                        desired_headless,
625                        "recreating Service to change headless mode"
626                    );
627                    kanidm.delete(ctx, &current).await?;
628                    ctx.kaniop_ctx
629                        .metrics
630                        .reconcile_deploy_delete_create_inc("Service", "headless_transition");
631                    wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
632                        .await?;
633                } else {
634                    preserve_defaulted_service_fields(&mut desired, &current);
635                }
636            }
637        }
638    }
639
640    kanidm.patch(ctx, desired).await
641}
642
643async fn wait_for_service_deletion(
644    client: kube::Client,
645    namespace: &str,
646    name: &str,
647) -> Result<()> {
648    let api = Api::<Service>::namespaced(client, namespace);
649    let start = std::time::Instant::now();
650    let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
651    let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
652
653    loop {
654        match api.get_opt(name).await {
655            Ok(None) => return Ok(()),
656            Ok(Some(_)) => {}
657            Err(e) => {
658                return Err(Error::kube_error(
659                    "wait for deletion of",
660                    "Service",
661                    namespace,
662                    name,
663                    e,
664                ));
665            }
666        }
667
668        if start.elapsed() >= timeout {
669            return Err(Error::ReceiveOutput(format!(
670                "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
671            )));
672        }
673        sleep(poll_interval).await;
674    }
675}
676
677async fn validate_statefulset_replacement(
678    api: &Api<StatefulSet>,
679    desired: &StatefulSet,
680    namespace: &str,
681    name: &str,
682) -> Result<()> {
683    let mut candidate = desired.clone();
684    candidate.metadata.name = None;
685    candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
686    candidate.metadata.namespace = None;
687    candidate.metadata.resource_version = None;
688    candidate.metadata.uid = None;
689    candidate.metadata.creation_timestamp = None;
690    candidate.metadata.generation = None;
691    candidate.metadata.managed_fields = None;
692    candidate.metadata.deletion_timestamp = None;
693    candidate.metadata.deletion_grace_period_seconds = None;
694    candidate.status = None;
695
696    api.create(
697        &PostParams {
698            dry_run: true,
699            ..Default::default()
700        },
701        &candidate,
702    )
703    .await
704    .map(|_| ())
705    .map_err(|e| {
706        Error::kube_error(
707            "validate replacement for",
708            "StatefulSet",
709            namespace,
710            name,
711            e,
712        )
713    })
714}
715
716async fn reconcile_statefulset(
717    kanidm: Arc<Kanidm>,
718    ctx: Arc<Context>,
719    desired: StatefulSet,
720) -> Result<StatefulSet> {
721    let namespace = kanidm.namespace().unwrap();
722    let name = desired.name_any();
723    let cached_current = ctx.stores.stateful_set_store.find(|current| {
724        current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
725    });
726
727    // On a cold reflector cache, keep the normal create/update path GET-free. If the optimistic
728    // apply fails with HTTP 422, the object may already exist with an immutable conflict that the
729    // cache has not observed yet. Only 422 errors enter the live classification/recreate recovery
730    // path; 429, 5xx, 403, and transport errors are returned immediately and non-destructively.
731    let cache_miss_422_error = if let Some(cached_current) = cached_current {
732        let mut cached_desired = desired.clone();
733        preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
734        if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
735            == StatefulSetApplyStrategy::Apply
736        {
737            return kanidm.apply(&ctx, cached_desired).await;
738        }
739        None
740    } else {
741        match kanidm.apply(&ctx, desired.clone()).await {
742            Ok(applied) => return Ok(applied),
743            Err(error) => {
744                // Only HTTP 422 can indicate an immutable-field conflict that is safe to
745                // recover from via delete/recreate. Arbitrary failures must not trigger
746                // destruction.
747                if is_kube_422(&error) {
748                    Some(error)
749                } else {
750                    return Err(error);
751                }
752            }
753        }
754    };
755
756    // Never make a destructive decision from reflector state or an SSA error alone. Re-read the
757    // live object and classify it immediately before deletion; another actor or an earlier
758    // reconcile may already have converged the immutable fields while the cache was catching up.
759    let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
760    let Some(live_current) = api
761        .get_opt(&name)
762        .await
763        .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
764    else {
765        return match cache_miss_422_error {
766            Some(error) => Err(error),
767            None => kanidm.apply(&ctx, desired).await,
768        };
769    };
770
771    // Start from the original desired object again. Defaults copied from a stale reflector entry
772    // must not survive into the live re-check and become the reason for a recreation.
773    let mut live_desired = desired.clone();
774    preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
775    match classify_statefulset_change(&live_current, &live_desired) {
776        StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
777            Some(error) => Err(error),
778            None => kanidm.apply(&ctx, live_desired).await,
779        },
780        StatefulSetApplyStrategy::Recreate { .. } => {
781            // An update 422 can combine an immutable conflict with independently-invalid desired
782            // state. Validate the replacement as a dry-run CREATE before deleting the live object,
783            // so create-time validation runs without the existing object's immutable-update check.
784            validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
785
786            // Re-read and reclassify immediately before destruction. Another actor or an earlier
787            // reconcile may have changed the live immutable fields or PVC-retention semantics.
788            let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
789                Error::kube_error(
790                    "re-read before deletion",
791                    "StatefulSet",
792                    &namespace,
793                    &name,
794                    e,
795                )
796            })?
797            else {
798                return kanidm.apply(&ctx, live_desired).await;
799            };
800            let mut latest_desired = live_desired.clone();
801            preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
802
803            match classify_statefulset_change(&latest_current, &latest_desired) {
804                StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
805                StatefulSetApplyStrategy::Recreate { immutable_fields } => {
806                    // Validate the replacement StatefulSet via a dry-run CREATE before deleting
807                    // the live object. This prevents deleting on a mixed 422 (immutable conflict
808                    // + independently-invalid desired state) where the replacement would still
809                    // be rejected by the API.
810                    validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
811                        .await?;
812
813                    info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
814                    kanidm.delete(&ctx, &latest_current).await?;
815                    ctx.kaniop_ctx
816                        .metrics
817                        .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
818                    wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
819                        .await?;
820                    kanidm.apply(&ctx, latest_desired).await
821                }
822            }
823        }
824    }
825}
826
827/// Return true only when the error structurally wraps a Kubernetes HTTP 422 response.
828fn is_kube_422(error: &Error) -> bool {
829    matches!(
830        error,
831        Error::KubeError(_, cause)
832            if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
833    )
834}
835
836#[cfg(test)]
837mod statefulset_recreate_recovery_tests {
838    use super::{Error, is_kube_422};
839
840    fn kube_api_error(code: u16) -> Error {
841        Error::KubeError(
842            "test error".to_string(),
843            Box::new(kube::Error::Api(Box::new(kube::error::Status {
844                code,
845                message: "test".to_string(),
846                reason: "test".to_string(),
847                ..Default::default()
848            }))),
849        )
850    }
851
852    #[test]
853    fn recreate_recovery_accepts_only_422() {
854        assert!(is_kube_422(&kube_api_error(422)));
855        assert!(!is_kube_422(&kube_api_error(403)));
856        assert!(!is_kube_422(&kube_api_error(429)));
857        assert!(!is_kube_422(&kube_api_error(500)));
858    }
859
860    #[test]
861    fn recreate_recovery_rejects_429_too_many_requests() {
862        assert!(!is_kube_422(&kube_api_error(429)));
863    }
864
865    #[test]
866    fn recreate_recovery_rejects_5xx_server_errors() {
867        assert!(!is_kube_422(&kube_api_error(500)));
868        assert!(!is_kube_422(&kube_api_error(502)));
869        assert!(!is_kube_422(&kube_api_error(503)));
870        assert!(!is_kube_422(&kube_api_error(504)));
871    }
872
873    #[test]
874    fn recreate_recovery_rejects_400_and_403() {
875        assert!(!is_kube_422(&kube_api_error(400)));
876        assert!(!is_kube_422(&kube_api_error(403)));
877    }
878}
879
880async fn reconcile(
881    kanidm: Arc<Kanidm>,
882    ctx: Arc<Context>,
883    status: KanidmStatus,
884) -> Result<(Action, bool)> {
885    if kanidm
886        .annotations()
887        .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
888    {
889        ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
890        return Ok((Action::requeue(Duration::from_secs(5)), false));
891    }
892
893    let mut changed = false;
894    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
895    let replication_secret_futures =
896        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
897
898    let sts_to_delete = {
899        let expected_sts_names = kanidm
900            .spec
901            .replica_groups
902            .iter()
903            .map(|rg| kanidm.statefulset_name(&rg.name))
904            .collect::<Vec<_>>();
905        ctx.stores
906            .stateful_set_store
907            .state()
908            .into_iter()
909            .filter(|sts| {
910                sts.namespace() == kanidm.namespace()
911                    && sts
912                        .metadata
913                        .labels
914                        .as_ref()
915                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
916                    && !expected_sts_names.contains(&sts.name_any())
917            })
918            .collect::<Vec<_>>()
919    };
920    if !sts_to_delete.is_empty() {
921        changed = true;
922    }
923    let sts_delete_futures = sts_to_delete
924        .iter()
925        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
926        .collect::<TryJoinAll<_>>();
927
928    let sts_futures = match kanidm.is_updatable(&status) {
929        true => {
930            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
931                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
932            });
933
934            let namespace = kanidm.get_namespace();
935            let schedule_api: Api<KanidmBackupSchedule> =
936                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
937            let schedules = schedule_api
938                .list(&Default::default())
939                .await
940                .map_err(|e| {
941                    Error::KubeError(
942                        "failed to list KanidmBackupSchedules for transport sidecar resolution"
943                            .to_string(),
944                        Box::new(e),
945                    )
946                })?
947                .items;
948
949            let repo_api: Api<KanidmBackupRepository> =
950                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
951            let repositories = repo_api
952                .list(&Default::default())
953                .await
954                .map_err(|e| {
955                    Error::KubeError(
956                        "failed to list KanidmBackupRepositories for transport sidecar resolution"
957                            .to_string(),
958                        Box::new(e),
959                    )
960                })?
961                .items;
962
963            let transport_config = resolve_transport_config(&kanidm, &schedules, &repositories);
964
965            kanidm
966                .spec
967                .replica_groups
968                .iter()
969                .map(|rg| {
970                    let sts = kanidm.create_statefulset(
971                        rg,
972                        tls_secret_hash.as_deref(),
973                        transport_config.as_ref(),
974                    )?;
975                    Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
976                })
977                .collect::<Result<TryJoinAll<_>, _>>()?
978        }
979        false => {
980            let note = match status.version.as_ref() {
981                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
982                    format!(
983                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
984                        v.image_tag,
985                        crate::version::KANIDM_CLIENT_VERSION
986                    )
987                }
988                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
989            };
990            let _ignore_error = ctx
991                .kaniop_ctx
992                .recorder
993                .publish(
994                    &Event {
995                        type_: EventType::Warning,
996                        reason: "UpgradeBlocked".to_string(),
997                        note: Some(note),
998                        action: "ReconcileStatefulSet".to_string(),
999                        secondary: None,
1000                    },
1001                    &kanidm.object_ref(&()),
1002                )
1003                .await
1004                .map_err(|e| {
1005                    warn!(%e, "failed to publish KanidmError event");
1006                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
1007                });
1008            try_join_all([])
1009        }
1010    };
1011    let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1012
1013    let deprecated_rg_svcs = {
1014        let expected_rg_svcs_names = kanidm
1015            .spec
1016            .replica_groups
1017            .iter()
1018            .filter(|rg| rg.services.is_some())
1019            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1020            .collect::<Vec<_>>();
1021        ctx.stores
1022            .service_store
1023            .state()
1024            .into_iter()
1025            .filter(|svc| {
1026                let labels = svc.metadata.labels.as_ref();
1027                svc.namespace() == kanidm.namespace()
1028                    && !expected_rg_svcs_names.contains(&svc.name_any())
1029                    && labels.is_some_and(|l| {
1030                        l.contains_key(REPLICA_LABEL)
1031                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1032                    })
1033            })
1034            .collect::<Vec<_>>()
1035    };
1036    if !deprecated_rg_svcs.is_empty() {
1037        changed = true;
1038    }
1039    let rg_svcs_delete_futures = deprecated_rg_svcs
1040        .iter()
1041        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1042        .collect::<TryJoinAll<_>>();
1043
1044    let rg_services_futures = kanidm
1045        .spec
1046        .replica_groups
1047        .iter()
1048        .filter(|rg| rg.services.is_some())
1049        .flat_map(|rg| {
1050            (0..rg.replicas).map(|i| {
1051                let mut svc = kanidm.create_replica_group_service(rg, i);
1052                if let Some(current) = ctx.stores.service_store.find(|current| {
1053                    current.namespace() == kanidm.namespace()
1054                        && current.name_any() == svc.name_any()
1055                }) {
1056                    preserve_defaulted_service_fields(&mut svc, current.as_ref());
1057                }
1058                kanidm.patch(&ctx, svc)
1059            })
1060        })
1061        .collect::<TryJoinAll<_>>();
1062
1063    let deprecated_ingresses = {
1064        let names = [
1065            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1066            kanidm.generate_region_ingress_name(),
1067        ]
1068        .into_iter()
1069        .flatten()
1070        .collect::<Vec<_>>();
1071        ctx.stores
1072            .ingress_store
1073            .state()
1074            .into_iter()
1075            .filter(|ing| {
1076                ing.namespace() == kanidm.namespace()
1077                    && !names.contains(&ing.name_any())
1078                    && ing.metadata.labels == Some(kanidm.generate_labels())
1079            })
1080            .collect::<Vec<_>>()
1081    };
1082    if !deprecated_ingresses.is_empty() {
1083        changed = true;
1084    }
1085    let ingress_delete_futures = deprecated_ingresses
1086        .iter()
1087        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1088        .collect::<TryJoinAll<_>>();
1089
1090    let ingress_futures = kanidm
1091        .create_ingress()
1092        .into_iter()
1093        .chain(kanidm.create_region_ingress())
1094        .map(|ingress| kanidm.patch(&ctx, ingress))
1095        .collect::<TryJoinAll<_>>();
1096
1097    let deprecated_http_routes = match &ctx.stores.http_route_store {
1098        Some(store) => {
1099            let expected_names = kanidm
1100                .spec
1101                .gateway
1102                .as_ref()
1103                .map(|_| vec![kanidm.name_any()])
1104                .unwrap_or_default();
1105            let routes: Vec<_> = store
1106                .state()
1107                .into_iter()
1108                .filter(|route| {
1109                    route.namespace() == kanidm.namespace()
1110                        && !expected_names.contains(&route.name_any())
1111                        && route.metadata.labels == Some(kanidm.generate_labels())
1112                })
1113                .collect();
1114            if !routes.is_empty() {
1115                changed = true;
1116            }
1117            routes
1118        }
1119        None => Vec::new(),
1120    };
1121    let http_route_delete_futures = deprecated_http_routes
1122        .iter()
1123        .map(|route| kanidm.delete(&ctx, route.as_ref()))
1124        .collect::<TryJoinAll<_>>();
1125
1126    let http_route_futures = if ctx.stores.http_route_store.is_some() {
1127        kanidm
1128            .create_http_route()
1129            .into_iter()
1130            .map(|route| kanidm.patch(&ctx, route))
1131            .collect::<TryJoinAll<_>>()
1132    } else {
1133        try_join_all(Vec::new())
1134    };
1135
1136    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1137        Some(store) => {
1138            let expected_names = kanidm
1139                .spec
1140                .gateway
1141                .as_ref()
1142                .and_then(|g| g.backend_tls_policy.as_ref())
1143                .map(|_| vec![kanidm.name_any()])
1144                .unwrap_or_default();
1145            let policies: Vec<_> = store
1146                .state()
1147                .into_iter()
1148                .filter(|policy| {
1149                    policy.namespace() == kanidm.namespace()
1150                        && !expected_names.contains(&policy.name_any())
1151                        && policy.metadata.labels == Some(kanidm.generate_labels())
1152                })
1153                .collect();
1154            if !policies.is_empty() {
1155                changed = true;
1156            }
1157            policies
1158        }
1159        None => Vec::new(),
1160    };
1161    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1162        .iter()
1163        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1164        .collect::<TryJoinAll<_>>();
1165
1166    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1167        kanidm
1168            .create_backend_tls_policy()
1169            .into_iter()
1170            .map(|policy| kanidm.patch(&ctx, policy))
1171            .collect::<TryJoinAll<_>>()
1172    } else {
1173        try_join_all(Vec::new())
1174    };
1175
1176    let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1177        sts_delete_futures,
1178        admin_secret_future,
1179        replication_secret_futures,
1180        sts_futures,
1181        service_future,
1182        rg_svcs_delete_futures,
1183        rg_services_futures,
1184        ingress_delete_futures,
1185        ingress_futures,
1186        http_route_delete_futures,
1187        http_route_futures,
1188        backend_tls_policy_delete_futures,
1189        backend_tls_policy_futures
1190    )?;
1191
1192    if admin_secret_changed || replication_secrets_changed {
1193        changed = true;
1194    }
1195
1196    if is_kanidm_available(status.clone()) {
1197        let namespace = kanidm.namespace().unwrap();
1198        let name = kanidm.name_any();
1199        match crate::controller::kanidm::KanidmClients::create_client(
1200            &namespace,
1201            &name,
1202            crate::controller::kanidm::KanidmUser::Admin,
1203            ctx.kaniop_ctx.client.clone(),
1204        )
1205        .await
1206        {
1207            Ok(system_client) => {
1208                if let Err(e) =
1209                    reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1210                {
1211                    warn!(%e, "failed to reconcile domain appearance");
1212                }
1213            }
1214            Err(e) => {
1215                warn!(%e, "failed to create admin client for domain appearance reconciliation");
1216            }
1217        }
1218
1219        match crate::controller::kanidm::KanidmClients::create_client(
1220            &namespace,
1221            &name,
1222            crate::controller::kanidm::KanidmUser::IdmAdmin,
1223            ctx.kaniop_ctx.client.clone(),
1224        )
1225        .await
1226        {
1227            Ok(kanidm_client) => {
1228                let (mail_sender_status, mail_sender_mutated) =
1229                    reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1230                        .await
1231                        .unwrap_or_else(|e| {
1232                            warn!(%e, "failed to reconcile mail sender");
1233                            (None, false)
1234                        });
1235                if mail_sender_mutated {
1236                    changed = true;
1237                }
1238                let current_mail_sender_status =
1239                    kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1240                if mail_sender_status != current_mail_sender_status {
1241                    let kanidm_api: Api<Kanidm> =
1242                        Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1243                    let status_patch = serde_json::json!({
1244                        "status": {
1245                            "mailSender": mail_sender_status
1246                        }
1247                    });
1248                    if let Err(e) = kanidm_api
1249                        .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1250                        .await
1251                    {
1252                        warn!(%e, "failed to patch Kanidm/status for mail sender");
1253                    }
1254                }
1255            }
1256            Err(e) => {
1257                warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1258            }
1259        }
1260    }
1261
1262    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1263}
1264
1265async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1266    debug!("cleanup");
1267
1268    let mut changed = false;
1269    let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1270    if mail_resources_cleaned {
1271        changed = true;
1272    }
1273
1274    let namespace = kanidm.namespace().unwrap();
1275    let name = kanidm.name_any();
1276
1277    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1278        &namespace,
1279        &name,
1280        crate::controller::kanidm::KanidmUser::IdmAdmin,
1281        ctx.kaniop_ctx.client.clone(),
1282    )
1283    .await
1284    {
1285        let kanidm_cleanup_done =
1286            cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1287        if kanidm_cleanup_done {
1288            changed = true;
1289        }
1290    }
1291
1292    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1293    Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1294}
1295
1296async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1297    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1298    let start = std::time::Instant::now();
1299    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1300    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1301
1302    loop {
1303        let sts = sts_api.get(sts_name).await.map_err(|e| {
1304            Error::KubeError(
1305                format!("failed to get StatefulSet {namespace}/{sts_name}"),
1306                Box::new(e),
1307            )
1308        })?;
1309
1310        if sts.metadata.deletion_timestamp.is_some() {
1311            return Err(Error::ReceiveOutput(format!(
1312                "StatefulSet {namespace}/{sts_name} is being deleted"
1313            )));
1314        }
1315
1316        let spec = sts.spec.as_ref().ok_or_else(|| {
1317            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1318        })?;
1319
1320        let status = match sts.status {
1321            Some(ref s) => s,
1322            None => {
1323                if start.elapsed() >= timeout {
1324                    return Err(Error::ReceiveOutput(format!(
1325                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1326                    )));
1327                }
1328                sleep(poll_interval).await;
1329                continue;
1330            }
1331        };
1332
1333        let replicas = spec.replicas.unwrap_or(0);
1334
1335        if replicas == 0 {
1336            return Ok(());
1337        }
1338
1339        let updated_replicas = status.updated_replicas.unwrap_or(0);
1340        let current_revision = status.current_revision.as_deref().unwrap_or("");
1341        let update_revision = status.update_revision.as_deref().unwrap_or("");
1342        let ready_replicas = status.ready_replicas.unwrap_or(0);
1343
1344        if updated_replicas == replicas
1345            && current_revision == update_revision
1346            && ready_replicas == replicas
1347        {
1348            return Ok(());
1349        }
1350
1351        if start.elapsed() >= timeout {
1352            return Err(Error::ReceiveOutput(format!(
1353                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1354            )));
1355        }
1356
1357        sleep(poll_interval).await;
1358    }
1359}
1360
1361async fn show_replica_cert_with_retries(
1362    kanidm: &Kanidm,
1363    ctx: Arc<Context>,
1364    pod_name: &str,
1365    max_attempts: u32,
1366) -> std::result::Result<String, Error> {
1367    let mut last_error = None;
1368    for attempt in 0..max_attempts {
1369        if attempt > 0 {
1370            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1371        }
1372        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1373            Ok(cert) => return Ok(cert),
1374            Err(e) => {
1375                info!(
1376                    "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1377                    attempt + 1,
1378                    max_attempts
1379                );
1380                last_error = Some(e);
1381            }
1382        }
1383    }
1384    Err(last_error.unwrap_or_else(|| {
1385        Error::ReceiveOutput(format!(
1386            "failed to get certificate for {pod_name} after {max_attempts} attempts"
1387        ))
1388    }))
1389}
1390
1391impl Kanidm {
1392    // Convenience methods that handle context and operator name
1393    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1394    where
1395        K: Resource<Scope = NamespaceResourceScope>
1396            + Serialize
1397            + Clone
1398            + std::fmt::Debug
1399            + for<'de> Deserialize<'de>,
1400        <K as kube::Resource>::DynamicType: Default,
1401        <K as Resource>::Scope: std::marker::Sized,
1402    {
1403        self.kube_delete(
1404            ctx.kaniop_ctx.client.clone(),
1405            &ctx.kaniop_ctx.metrics,
1406            resource,
1407        )
1408        .await
1409    }
1410
1411    /// Server-side apply without any generic delete/recreate fallback.
1412    /// Resource-specific reconcilers use this when they own lifecycle decisions.
1413    pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1414    where
1415        K: Resource<Scope = NamespaceResourceScope>
1416            + Serialize
1417            + Clone
1418            + std::fmt::Debug
1419            + for<'de> Deserialize<'de>,
1420        <K as kube::Resource>::DynamicType: Default,
1421        <K as Resource>::Scope: std::marker::Sized,
1422    {
1423        self.kube_apply(
1424            ctx.kaniop_ctx.client.clone(),
1425            resource,
1426            KANIDM_OPERATOR_NAME,
1427        )
1428        .await
1429    }
1430
1431    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1432    where
1433        K: Resource<Scope = NamespaceResourceScope>
1434            + Serialize
1435            + Clone
1436            + std::fmt::Debug
1437            + for<'de> Deserialize<'de>,
1438        <K as kube::Resource>::DynamicType: Default,
1439        <K as Resource>::Scope: std::marker::Sized,
1440    {
1441        self.kube_patch(
1442            ctx.kaniop_ctx.client.clone(),
1443            &ctx.kaniop_ctx.metrics,
1444            resource,
1445            KANIDM_OPERATOR_NAME,
1446        )
1447        .await
1448    }
1449
1450    #[inline]
1451    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1452        LABELS
1453            .clone()
1454            .into_iter()
1455            .chain([
1456                (INSTANCE_LABEL.to_string(), self.name_any()),
1457                (CLUSTER_LABEL.to_string(), self.name_any()),
1458            ])
1459            .collect()
1460    }
1461
1462    #[inline]
1463    fn generate_labels(&self) -> BTreeMap<String, String> {
1464        self.generate_resource_labels()
1465    }
1466
1467    #[inline]
1468    fn get_tls_secret_name(&self) -> String {
1469        format!("{}-tls", self.name_any())
1470    }
1471
1472    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
1473    /// falling back to the ingress TLS secret and then to the default name.
1474    pub(crate) fn effective_tls_secret_name(&self) -> String {
1475        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1476            self.spec
1477                .ingress
1478                .as_ref()
1479                .and_then(|i| i.tls_secret_name.clone())
1480                .unwrap_or_else(|| self.get_tls_secret_name())
1481        })
1482    }
1483
1484    #[inline]
1485    fn get_namespace(&self) -> String {
1486        // safe unwrap: Kanidm is namespaced scoped
1487        self.namespace().unwrap()
1488    }
1489
1490    #[inline]
1491    fn is_replication_enabled(&self) -> bool {
1492        self.spec.replica_groups.len() > 1
1493            || self
1494                .spec
1495                .replica_groups
1496                .first()
1497                .is_some_and(|rg| rg.replicas > 1)
1498            || !self.spec.external_replication_nodes.is_empty()
1499    }
1500
1501    #[inline]
1502    fn is_updatable(&self, status: &KanidmStatus) -> bool {
1503        status
1504            .version
1505            .as_ref()
1506            .map(|v| {
1507                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1508                    return false;
1509                }
1510                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1511                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1512                match v.upgrade_check_result {
1513                    KanidmUpgradeCheckResult::Passed => {
1514                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1515                            minor <= v_minor + 1
1516                        } else {
1517                            true
1518                        }
1519                    }
1520                    KanidmUpgradeCheckResult::Failed => {
1521                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1522                            versions
1523                        {
1524                            major == v_major && minor == v_minor && patch >= v_patch
1525                        } else {
1526                            v.image_tag == current_tag
1527                        }
1528                    }
1529                }
1530            })
1531            .unwrap_or(true)
1532    }
1533
1534    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1535        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1536        let pod = pod_api.get(pod_name).await.map_err(|e| {
1537            Error::KubeError(
1538                format!("failed to get pod {namespace}/{pod_name}"),
1539                Box::new(e),
1540            )
1541        })?;
1542        Ok(pod
1543            .status
1544            .as_ref()
1545            .and_then(|s| s.conditions.as_ref())
1546            .is_some_and(|conditions| {
1547                conditions
1548                    .iter()
1549                    .any(|c| c.type_ == "Ready" && c.status == "True")
1550            }))
1551    }
1552
1553    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1554        let start = std::time::Instant::now();
1555        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1556        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1557
1558        loop {
1559            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1560                Ok(true) => return Ok(()),
1561                Ok(false) => {}
1562                Err(e) => {
1563                    if !e.is_retryable() {
1564                        return Err(e);
1565                    }
1566                    trace!(%e, "pod readiness check failed, retrying");
1567                }
1568            }
1569
1570            if start.elapsed() >= timeout {
1571                return Err(Error::ReceiveOutput(format!(
1572                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1573                )));
1574            }
1575
1576            sleep(poll_interval).await;
1577        }
1578    }
1579
1580    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1581    where
1582        I: IntoIterator<Item = T> + Debug,
1583        T: Into<String>,
1584    {
1585        let namespace = &self.get_namespace();
1586        trace!(
1587            resource.name = &pod_name,
1588            resource.namespace = &namespace,
1589            ?command,
1590            "pod exec"
1591        );
1592        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1593        let attached = pod
1594            .exec(
1595                pod_name,
1596                command,
1597                &AttachParams::default().container("kanidm"),
1598            )
1599            .await
1600            .map_err(|e| {
1601                Error::KubeError(
1602                    format!("failed to exec pod {namespace}/{pod_name}"),
1603                    Box::new(e),
1604                )
1605            })?;
1606        get_output(attached).await
1607    }
1608
1609    async fn exec_with_wait<I, T>(
1610        &self,
1611        ctx: Arc<Context>,
1612        pod_name: &str,
1613        command: I,
1614    ) -> Result<String>
1615    where
1616        I: IntoIterator<Item = T> + Debug,
1617        T: Into<String>,
1618    {
1619        let namespace = &self.get_namespace();
1620        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1621        self.exec(ctx, pod_name, command).await
1622    }
1623
1624    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1625    where
1626        I: IntoIterator<Item = T> + Debug,
1627        T: Into<String>,
1628    {
1629        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1630        let first_replica_group = self
1631            .spec
1632            .replica_groups
1633            .first()
1634            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1635        let sts_name = self.statefulset_name(&first_replica_group.name);
1636        let pod_name = format!("{sts_name}-0");
1637        self.exec(ctx, &pod_name, command).await
1638    }
1639
1640    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1641    where
1642        I: IntoIterator<Item = T> + Debug,
1643        T: Into<String>,
1644    {
1645        let first_replica_group = self
1646            .spec
1647            .replica_groups
1648            .first()
1649            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1650        let sts_name = self.statefulset_name(&first_replica_group.name);
1651        let pod_name = format!("{sts_name}-0");
1652        self.exec_with_wait(ctx, &pod_name, command).await
1653    }
1654}
1655
1656#[cfg(test)]
1657mod test {
1658    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1659    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1660
1661    use crate::controller::State;
1662    use crate::kanidm::controller::context::{Context, Stores};
1663    use crate::kanidm::crd::KanidmStatus;
1664
1665    use kaniop_k8s_util::error::Result;
1666    use kaniop_k8s_util::resources::hash_secret_data;
1667
1668    use std::collections::BTreeMap;
1669    use std::sync::Arc;
1670
1671    use http::{Request, Response};
1672    use k8s_openapi::ByteString;
1673    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1674    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1675    use k8s_openapi::api::networking::v1::Ingress;
1676    use kube::api::ObjectMeta;
1677    use kube::runtime::reflector::store::Writer;
1678    use kube::{Client, Resource, ResourceExt, client::Body};
1679    use opentelemetry::metrics::MeterProvider;
1680    use serde_json::json;
1681
1682    impl Kanidm {
1683        /// A normal test kanidm with a given status
1684        pub fn test() -> Self {
1685            let mut e = Kanidm::new(
1686                "test",
1687                serde_json::from_value(json!({
1688                    "domain": "idm.example.com",
1689                    "replicaGroups": [
1690                        {
1691                            "name": "default",
1692                            "replicas": 1
1693                        }
1694                    ]
1695                }))
1696                .unwrap(),
1697            );
1698            e.meta_mut().namespace = Some("default".into());
1699            // Pre-populate finalizer so reconcile callback runs immediately
1700            // (skips the "add finalizer + wait for next reconcile" step)
1701            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1702            e
1703        }
1704
1705        pub fn with_ingress(mut self) -> Self {
1706            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1707            self
1708        }
1709
1710        /// Modify kanidm replicas
1711        pub fn with_replicas(mut self, replicas: i32) -> Self {
1712            self.spec.replica_groups[0].replicas = replicas;
1713            self
1714        }
1715
1716        /// Modify kanidm to set a deletion timestamp
1717        pub fn needs_delete(mut self) -> Self {
1718            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1719            use k8s_openapi::jiff::Timestamp;
1720            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1721            self.meta_mut().deletion_timestamp = Some(Time(now));
1722            self
1723        }
1724
1725        /// Modify a kanidm to have an expected status
1726        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1727            self.status = Some(status);
1728            self
1729        }
1730    }
1731
1732    // We wrap tower_test::mock::Handle
1733    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1734    pub struct ApiServerVerifier(ApiServerHandle);
1735
1736    /// Scenarios we test for in ApiServerVerifier
1737    pub enum Scenario {
1738        Create(Kanidm),
1739        CreateWithTwoReplicas(Kanidm),
1740        CreateWithIngress(Kanidm),
1741        CreateWithIngressWithTwoReplicas(Kanidm),
1742        CreateWithTlsSecret(Kanidm, Box<Secret>),
1743    }
1744
1745    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1746        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1747            .await
1748            .expect("timeout on mock apiserver")
1749            .expect("scenario succeeded")
1750    }
1751
1752    impl ApiServerVerifier {
1753        /// Tests only get to run specific scenarios that has matching handlers
1754        ///
1755        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1756        ///
1757        /// NB: If the controller is making more calls than we are handling in the scenario,
1758        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1759        ///
1760        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1761        /// scenario runs to completion (i.e. all expected calls were responded to),
1762        /// using the timeout to catch missing api calls to Kubernetes.
1763        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1764            tokio::spawn(async move {
1765                // moving self => one scenario per test
1766                match scenario {
1767                    Scenario::Create(kanidm) => {
1768                        self.handle_kanidm_get(kanidm.clone())
1769                            .await
1770                            .unwrap()
1771                            .handle_kanidm_status_patch(kanidm.clone())
1772                            .await
1773                            .unwrap()
1774                            .handle_tls_secret_get_not_found(kanidm.clone())
1775                            .await
1776                            .unwrap()
1777                            .handle_backup_schedule_list()
1778                            .await
1779                            .unwrap()
1780                            .handle_backup_repository_list()
1781                            .await
1782                            .unwrap()
1783                            .handle_statefulset_patch(kanidm.clone(), None)
1784                            .await
1785                            .unwrap()
1786                            .handle_service_patch(kanidm.clone())
1787                            .await
1788                    }
1789                    Scenario::CreateWithTwoReplicas(kanidm) => {
1790                        self.handle_kanidm_get(kanidm.clone())
1791                            .await
1792                            .unwrap()
1793                            .handle_kanidm_status_patch(kanidm.clone())
1794                            .await
1795                            .unwrap()
1796                            .handle_tls_secret_get_not_found(kanidm.clone())
1797                            .await
1798                            .unwrap()
1799                            .handle_backup_schedule_list()
1800                            .await
1801                            .unwrap()
1802                            .handle_backup_repository_list()
1803                            .await
1804                            .unwrap()
1805                            .handle_statefulset_patch(kanidm.clone(), None)
1806                            .await
1807                            .unwrap()
1808                            .handle_service_patch(kanidm.clone())
1809                            .await
1810                    }
1811                    Scenario::CreateWithIngress(kanidm) => {
1812                        self.handle_kanidm_get(kanidm.clone())
1813                            .await
1814                            .unwrap()
1815                            .handle_kanidm_status_patch(kanidm.clone())
1816                            .await
1817                            .unwrap()
1818                            .handle_tls_secret_get_not_found(kanidm.clone())
1819                            .await
1820                            .unwrap()
1821                            .handle_backup_schedule_list()
1822                            .await
1823                            .unwrap()
1824                            .handle_backup_repository_list()
1825                            .await
1826                            .unwrap()
1827                            .handle_statefulset_patch(kanidm.clone(), None)
1828                            .await
1829                            .unwrap()
1830                            .handle_service_patch(kanidm.clone())
1831                            .await
1832                            .unwrap()
1833                            .handle_ingress_patch(kanidm.clone())
1834                            .await
1835                    }
1836                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1837                        self.handle_kanidm_get(kanidm.clone())
1838                            .await
1839                            .unwrap()
1840                            .handle_kanidm_status_patch(kanidm.clone())
1841                            .await
1842                            .unwrap()
1843                            .handle_tls_secret_get_not_found(kanidm.clone())
1844                            .await
1845                            .unwrap()
1846                            .handle_backup_schedule_list()
1847                            .await
1848                            .unwrap()
1849                            .handle_backup_repository_list()
1850                            .await
1851                            .unwrap()
1852                            .handle_statefulset_patch(kanidm.clone(), None)
1853                            .await
1854                            .unwrap()
1855                            .handle_service_patch(kanidm.clone())
1856                            .await
1857                            .unwrap()
1858                            .handle_ingress_patch(kanidm.clone())
1859                            .await
1860                    }
1861                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1862                        let expected_hash = hash_secret_data(&secret);
1863                        self.handle_kanidm_get(kanidm.clone())
1864                            .await
1865                            .unwrap()
1866                            .handle_kanidm_status_patch(kanidm.clone())
1867                            .await
1868                            .unwrap()
1869                            .handle_tls_secret_get(kanidm.clone(), *secret)
1870                            .await
1871                            .unwrap()
1872                            .handle_backup_schedule_list()
1873                            .await
1874                            .unwrap()
1875                            .handle_backup_repository_list()
1876                            .await
1877                            .unwrap()
1878                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1879                            .await
1880                            .unwrap()
1881                            .handle_service_patch(kanidm.clone())
1882                            .await
1883                    }
1884                }
1885                .expect("scenario completed without errors");
1886            })
1887        }
1888
1889        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1890            let (request, send) = self.0.next_request().await.expect("service not called");
1891            assert_eq!(request.method(), http::Method::GET);
1892            let expected_uri_prefix = format!(
1893                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1894                kanidm.name_any()
1895            );
1896            let uri = request.uri().to_string();
1897            assert!(
1898                uri.starts_with(&expected_uri_prefix),
1899                "expected uri to start with {}, got {}",
1900                expected_uri_prefix,
1901                uri
1902            );
1903            let response = serde_json::to_vec(&kanidm).unwrap();
1904            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1905            Ok(self)
1906        }
1907
1908        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1909            let (request, send) = self.0.next_request().await.expect("service not called");
1910            assert_eq!(request.method(), http::Method::PATCH);
1911            assert_eq!(
1912                request.uri().to_string(),
1913                format!(
1914                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1915                    kanidm.name_any()
1916                )
1917            );
1918
1919            let req_body = request.into_body().collect_bytes().await.unwrap();
1920            let json: serde_json::Value =
1921                serde_json::from_slice(&req_body).expect("patch object is json");
1922            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1923                .expect("valid kanidm status");
1924            let mut kanidm = Kanidm::test();
1925            kanidm.status = Some(status.clone());
1926            let response = serde_json::to_vec(&kanidm).unwrap();
1927            // pass through kanidm "patch accepted"
1928            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1929            Ok(self)
1930        }
1931
1932        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1933            let (request, send) = self.0.next_request().await.expect("service not called");
1934            assert_eq!(request.method(), http::Method::GET);
1935            let expected_uri_prefix = format!(
1936                "/api/v1/namespaces/default/secrets/{}",
1937                kanidm.effective_tls_secret_name()
1938            );
1939            let uri = request.uri().to_string();
1940            assert!(
1941                uri.starts_with(&expected_uri_prefix),
1942                "expected uri to start with {expected_uri_prefix}, got {uri}"
1943            );
1944            let status = serde_json::json!({
1945                "kind": "Status",
1946                "apiVersion": "v1",
1947                "metadata": {},
1948                "status": "Failure",
1949                "message": "secret not found",
1950                "reason": "NotFound",
1951                "code": 404
1952            });
1953            send.send_response(
1954                Response::builder()
1955                    .status(404)
1956                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1957                    .unwrap(),
1958            );
1959            Ok(self)
1960        }
1961
1962        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1963            let (request, send) = self.0.next_request().await.expect("service not called");
1964            assert_eq!(request.method(), http::Method::GET);
1965            let expected_uri_prefix = format!(
1966                "/api/v1/namespaces/default/secrets/{}",
1967                kanidm.effective_tls_secret_name()
1968            );
1969            let uri = request.uri().to_string();
1970            assert!(
1971                uri.starts_with(&expected_uri_prefix),
1972                "expected uri to start with {expected_uri_prefix}, got {uri}"
1973            );
1974            let response = serde_json::to_vec(&secret).unwrap();
1975            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1976            Ok(self)
1977        }
1978
1979        async fn handle_backup_schedule_list(mut self) -> Result<Self> {
1980            let (request, send) = self.0.next_request().await.expect("service not called");
1981            assert_eq!(request.method(), http::Method::GET);
1982            assert!(
1983                request
1984                    .uri()
1985                    .to_string()
1986                    .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
1987                "expected backup schedule list request, got {}",
1988                request.uri()
1989            );
1990            let list = json!({
1991                "apiVersion": "kaniop.rs/v1alpha1",
1992                "kind": "KanidmBackupScheduleList",
1993                "metadata": {},
1994                "items": []
1995            });
1996            let response = serde_json::to_vec(&list).unwrap();
1997            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1998            Ok(self)
1999        }
2000
2001        async fn handle_backup_repository_list(mut self) -> Result<Self> {
2002            let (request, send) = self.0.next_request().await.expect("service not called");
2003            assert_eq!(request.method(), http::Method::GET);
2004            assert!(
2005                request.uri().to_string().contains(
2006                    "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2007                ),
2008                "expected backup repository list request, got {}",
2009                request.uri()
2010            );
2011            let list = json!({
2012                "apiVersion": "kaniop.rs/v1alpha1",
2013                "kind": "KanidmBackupRepositoryList",
2014                "metadata": {},
2015                "items": []
2016            });
2017            let response = serde_json::to_vec(&list).unwrap();
2018            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2019            Ok(self)
2020        }
2021
2022        async fn handle_statefulset_patch(
2023            mut self,
2024            kanidm: Kanidm,
2025            expected_tls_hash: Option<String>,
2026        ) -> Result<Self> {
2027            for rg in kanidm.spec.replica_groups.iter() {
2028                let (request, send) = self.0.next_request().await.expect("service not called");
2029                assert_eq!(request.method(), http::Method::PATCH);
2030                assert_eq!(
2031                    request.uri().to_string(),
2032                    format!(
2033                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2034                        kanidm.statefulset_name(&rg.name)
2035                    )
2036                );
2037                let req_body = request.into_body().collect_bytes().await.unwrap();
2038                let json: serde_json::Value =
2039                    serde_json::from_slice(&req_body).expect("patch object is json");
2040                let statefulset: StatefulSet =
2041                    serde_json::from_value(json).expect("valid statefulset");
2042                assert_eq!(
2043                    statefulset.clone().spec.unwrap().replicas.unwrap(),
2044                    rg.replicas
2045                );
2046                let tls_hash_annotation = statefulset
2047                    .spec
2048                    .as_ref()
2049                    .unwrap()
2050                    .template
2051                    .metadata
2052                    .as_ref()
2053                    .unwrap()
2054                    .annotations
2055                    .as_ref()
2056                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2057                    .cloned();
2058                assert_eq!(tls_hash_annotation, expected_tls_hash);
2059                let response = serde_json::to_vec(&statefulset).unwrap();
2060                // pass through kanidm "patch accepted"
2061                send.send_response(Response::builder().body(Body::from(response)).unwrap());
2062            }
2063
2064            Ok(self)
2065        }
2066
2067        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2068            let (request, send) = self.0.next_request().await.expect("service not called");
2069            assert_eq!(request.method(), http::Method::PATCH);
2070            assert_eq!(
2071                request.uri().to_string(),
2072                format!(
2073                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2074                    kanidm.name_any()
2075                )
2076            );
2077
2078            let req_body = request.into_body().collect_bytes().await.unwrap();
2079            let json: serde_json::Value =
2080                serde_json::from_slice(&req_body).expect("patch object is json");
2081            let service: Service = serde_json::from_value(json).expect("valid service");
2082            let response = serde_json::to_vec(&service).unwrap();
2083            // pass through kanidm "patch accepted"
2084            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2085            Ok(self)
2086        }
2087
2088        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2089            let (request, send) = self.0.next_request().await.expect("service not called");
2090            assert_eq!(request.method(), http::Method::PATCH);
2091            assert_eq!(
2092                request.uri().to_string(),
2093                format!(
2094                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2095                    kanidm.name_any()
2096                )
2097            );
2098
2099            let req_body = request.into_body().collect_bytes().await.unwrap();
2100            let json: serde_json::Value =
2101                serde_json::from_slice(&req_body).expect("patch object is json");
2102            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2103            let response = serde_json::to_vec(&ingress).unwrap();
2104            // pass through kanidm "patch accepted"
2105            send.send_response(Response::builder().body(Body::from(response)).unwrap());
2106            Ok(self)
2107        }
2108    }
2109
2110    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2111        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2112        let mock_client = Client::new(mock_service, "default");
2113        let stores = Stores {
2114            stateful_set_store: Writer::default().as_reader(),
2115            service_store: Writer::default().as_reader(),
2116            ingress_store: Writer::default().as_reader(),
2117            secret_store: Writer::default().as_reader(),
2118            http_route_store: Some(Writer::default().as_reader()),
2119            backend_tls_policy_store: Some(Writer::default().as_reader()),
2120            deployment_store: Writer::default().as_reader(),
2121            config_map_store: Writer::default().as_reader(),
2122        };
2123        let controller_id = "test";
2124
2125        // Create a test meter for metrics
2126        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2127        let meter = provider.meter("test");
2128        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2129
2130        let state = State::new(
2131            metrics,
2132            Writer::default().as_reader(),
2133            Writer::default().as_reader(),
2134            None,
2135        );
2136        let ctx = Arc::new(Context::new(
2137            state.to_context(mock_client, controller_id),
2138            stores,
2139        ));
2140        (ctx, ApiServerVerifier(handle))
2141    }
2142
2143    #[tokio::test]
2144    async fn kanidm_create() {
2145        let (testctx, fakeserver) = get_test_context();
2146        let kanidm = Kanidm::test();
2147        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2148        reconcile_kanidm(Arc::new(kanidm), testctx)
2149            .await
2150            .expect("reconciler");
2151        timeout_after_1s(mocksrv).await;
2152    }
2153
2154    #[tokio::test]
2155    async fn kanidm_create_with_two_replicas() {
2156        let (testctx, fakeserver) = get_test_context();
2157        let kanidm = Kanidm::test().with_replicas(2);
2158        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2159        reconcile_kanidm(Arc::new(kanidm), testctx)
2160            .await
2161            .expect("reconciler");
2162        timeout_after_1s(mocksrv).await;
2163    }
2164
2165    #[tokio::test]
2166    async fn kanidm_create_with_ingress() {
2167        let (testctx, fakeserver) = get_test_context();
2168        let kanidm = Kanidm::test().with_ingress();
2169        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2170        reconcile_kanidm(Arc::new(kanidm), testctx)
2171            .await
2172            .expect("reconciler");
2173        timeout_after_1s(mocksrv).await;
2174    }
2175
2176    #[tokio::test]
2177    async fn kanidm_create_with_ingress_with_two_replicas() {
2178        let (testctx, fakeserver) = get_test_context();
2179        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2180        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2181        reconcile_kanidm(Arc::new(kanidm), testctx)
2182            .await
2183            .expect("reconciler");
2184        timeout_after_1s(mocksrv).await;
2185    }
2186
2187    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2188        Arc::new(StatefulSet {
2189            metadata: ObjectMeta {
2190                name: Some(format!("{cluster}-default")),
2191                namespace: Some(namespace.to_string()),
2192                labels: Some(BTreeMap::from([(
2193                    CLUSTER_LABEL.to_string(),
2194                    cluster.to_string(),
2195                )])),
2196                ..Default::default()
2197            },
2198            spec: Some(StatefulSetSpec {
2199                template: PodTemplateSpec {
2200                    metadata: Some(ObjectMeta {
2201                        annotations: tls_hash.map(|h| {
2202                            BTreeMap::from([(
2203                                TLS_SECRET_HASH_ANNOTATION.to_string(),
2204                                h.to_string(),
2205                            )])
2206                        }),
2207                        ..Default::default()
2208                    }),
2209                    ..Default::default()
2210                },
2211                ..Default::default()
2212            }),
2213            ..Default::default()
2214        })
2215    }
2216
2217    #[test]
2218    fn test_previous_tls_secret_hash() {
2219        let kanidm = Kanidm::test();
2220
2221        // annotation carried over from an existing statefulset
2222        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2223        assert_eq!(
2224            previous_tls_secret_hash(&kanidm, stss),
2225            Some("abc123".to_string())
2226        );
2227
2228        // no annotation on the existing statefulset
2229        let stss = vec![statefulset_of("test", "default", None)];
2230        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2231
2232        // statefulsets from another cluster or namespace are ignored
2233        let stss = vec![
2234            statefulset_of("other", "default", Some("abc123")),
2235            statefulset_of("test", "other-ns", Some("abc123")),
2236        ];
2237        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2238    }
2239
2240    #[tokio::test]
2241    async fn kanidm_create_with_tls_secret() {
2242        let (testctx, fakeserver) = get_test_context();
2243        let kanidm = Kanidm::test();
2244        let secret = Secret {
2245            metadata: ObjectMeta {
2246                name: Some(kanidm.effective_tls_secret_name()),
2247                namespace: Some("default".to_string()),
2248                ..Default::default()
2249            },
2250            data: Some(BTreeMap::from([
2251                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2252                ("tls.key".to_string(), ByteString(b"key".to_vec())),
2253            ])),
2254            ..Default::default()
2255        };
2256        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2257            kanidm.clone(),
2258            Box::new(secret),
2259        ));
2260        reconcile_kanidm(Arc::new(kanidm), testctx)
2261            .await
2262            .expect("reconciler");
2263        timeout_after_1s(mocksrv).await;
2264    }
2265}