Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9
10use self::domain_appearance::reconcile_domain_appearance;
11use self::mail_sender::{
12    cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
13};
14use super::controller::{CONTROLLER_ID, context::Context};
15
16use self::gateway::GatewayExt;
17use self::ingress::IngressExt;
18use self::secret::SecretExt;
19use self::service::ServiceExt;
20use self::statefulset::StatefulSetExt;
21use self::status::StatusExt;
22use self::status::{is_kanidm_available, is_kanidm_initialized};
23
24use crate::controller::context::KubeOperations;
25use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
26use crate::kanidm::crd::{
27    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
28};
29use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
30use crate::telemetry;
31
32use kaniop_k8s_util::client::get_output;
33use kaniop_k8s_util::error::{Error, Result};
34use kaniop_k8s_util::parse::parse_semver;
35use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
36
37use std::collections::{BTreeMap, BTreeSet};
38use std::fmt::Debug;
39use std::sync::{Arc, LazyLock};
40
41use futures::future::{TryJoinAll, join_all, try_join_all};
42use futures::stream::{self, StreamExt};
43use futures::try_join;
44use k8s_openapi::NamespaceResourceScope;
45use k8s_openapi::api::apps::v1::StatefulSet;
46use k8s_openapi::api::core::v1::{Pod, Secret};
47use kube::Resource;
48use kube::ResourceExt;
49use kube::api::{Api, AttachParams, Patch, PatchParams};
50use kube::runtime::controller::Action;
51use kube::runtime::events::{Event, EventType};
52use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
53use serde::{Deserialize, Serialize};
54use tokio::time::{Duration, sleep};
55use tracing::{Span, debug, field, info, instrument, trace, warn};
56
57const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
58const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
59const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
60const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
61const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
62const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
63const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
64
65pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
66const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
67pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
68
69const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
70
71static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
72    BTreeMap::from([
73        (NAME_LABEL.to_string(), "kanidm".to_string()),
74        (
75            MANAGED_BY_LABEL.to_string(),
76            format!("kaniop-{CONTROLLER_ID}"),
77        ),
78    ])
79});
80
81async fn restart_statefulsets_after_replica_secret_update(
82    ctx: Arc<Context>,
83    sts_api: &Api<StatefulSet>,
84    namespace: &str,
85    statefulset_names: &[String],
86    sequential: bool,
87) -> Result<()> {
88    // Replica cert secrets are consumed as pod env vars and rendered into config at startup.
89    // Pods must restart after cert secret changes or replication keeps stale TLS material.
90    let restart_futures = statefulset_names
91        .iter()
92        .map(|sts_name| sts_api.restart(sts_name));
93
94    if sequential {
95        let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
96        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
97            return Err(Error::kube_error(
98                "restart",
99                "StatefulSet",
100                namespace,
101                "replica certificate secret update statefulsets",
102                first_err,
103            ));
104        }
105    } else {
106        let results: Vec<_> = join_all(restart_futures).await;
107        if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
108            return Err(Error::kube_error(
109                "restart",
110                "StatefulSet",
111                namespace,
112                "replica certificate secret update statefulsets",
113                first_err,
114            ));
115        }
116    }
117
118    let rollout_futures = statefulset_names
119        .iter()
120        .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
121    try_join_all(rollout_futures).await?;
122
123    Ok(())
124}
125
126pub async fn reconcile_admins_secret(
127    kanidm: Arc<Kanidm>,
128    ctx: Arc<Context>,
129    status: &KanidmStatus,
130) -> Result<()> {
131    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
132        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
133        kanidm.patch(&ctx, admins_secret).await?;
134    }
135    Ok(())
136}
137
138pub async fn reconcile_replication_secrets(
139    kanidm: Arc<Kanidm>,
140    ctx: Arc<Context>,
141    status: &KanidmStatus,
142) -> Result<()> {
143    let expected_secret_names = if kanidm.is_replication_enabled() {
144        status
145            .replica_statuses
146            .iter()
147            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
148            .collect::<Vec<_>>()
149    } else {
150        Vec::new()
151    };
152    let deprecated_secrets = ctx
153        .stores
154        .secret_store
155        .state()
156        .into_iter()
157        .filter(|secret| {
158            secret.namespace() == kanidm.namespace()
159                && kanidm.admins_secret_name() != secret.name_any()
160                && !expected_secret_names.contains(&secret.name_any())
161                && secret
162                    .metadata
163                    .labels
164                    .as_ref()
165                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
166        })
167        .collect::<Vec<_>>();
168    let secret_delete_future = deprecated_secrets
169        .iter()
170        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
171        .collect::<TryJoinAll<_>>();
172    try_join!(secret_delete_future)?;
173
174    if kanidm.is_replication_enabled() {
175        let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
176        let statefulset_names = status
177            .replica_statuses
178            .iter()
179            .map(|rs| rs.statefulset_name.clone())
180            .collect::<BTreeSet<_>>()
181            .into_iter()
182            .collect::<Vec<_>>();
183
184        let has_pending = status
185            .replica_statuses
186            .iter()
187            .any(|rs| rs.state == KanidmReplicaState::Pending);
188
189        let sts_api =
190            Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
191
192        if has_pending {
193            stream::iter(
194                status
195                    .replica_statuses
196                    .iter()
197                    .filter(|rs| rs.state == KanidmReplicaState::Pending),
198            )
199            .then(|rs| async {
200                let secret = kanidm
201                    .generate_replica_secret(ctx.clone(), &rs.pod_name)
202                    .await?;
203                kanidm.patch(&ctx, secret).await
204            })
205            .collect::<Vec<_>>()
206            .await
207            .into_iter()
208            .collect::<Result<Vec<_>>>()?;
209
210            let restart_futures = status
211                .replica_statuses
212                .iter()
213                .map(|rs| sts_api.restart(&rs.statefulset_name));
214
215            if has_single_replica {
216                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
217                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
218                    return Err(Error::kube_error(
219                        "restart",
220                        "StatefulSet",
221                        kanidm.get_namespace(),
222                        "pending replica statefulsets",
223                        first_err,
224                    ));
225                }
226            } else {
227                let results: Vec<_> = join_all(restart_futures).await;
228                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
229                    return Err(Error::kube_error(
230                        "restart",
231                        "StatefulSet",
232                        kanidm.get_namespace(),
233                        "pending replica statefulsets",
234                        first_err,
235                    ));
236                }
237            }
238        }
239
240        let has_certificate_host_invalid = status
241            .replica_statuses
242            .iter()
243            .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
244
245        let has_certificate_expiring = status
246            .replica_statuses
247            .iter()
248            .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
249
250        let has_non_ready_replicas = status
251            .replica_statuses
252            .iter()
253            .any(|rs| rs.state != KanidmReplicaState::Ready);
254
255        let host_invalid_replicas: Vec<_> = status
256            .replica_statuses
257            .iter()
258            .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
259            .collect();
260
261        let cert_expiring_replicas: Vec<_> = status
262            .replica_statuses
263            .iter()
264            .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
265            .collect();
266
267        if !host_invalid_replicas.is_empty() {
268            let restart_futures = host_invalid_replicas
269                .iter()
270                .map(|rs| sts_api.restart(&rs.statefulset_name));
271
272            if has_single_replica {
273                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
274                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
275                    return Err(Error::kube_error(
276                        "restart",
277                        "StatefulSet",
278                        kanidm.get_namespace(),
279                        "certificate host invalid statefulsets",
280                        first_err,
281                    ));
282                }
283            } else {
284                let results: Vec<_> = join_all(restart_futures).await;
285                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
286                    return Err(Error::kube_error(
287                        "restart",
288                        "StatefulSet",
289                        kanidm.get_namespace(),
290                        "certificate host invalid statefulsets",
291                        first_err,
292                    ));
293                }
294            }
295
296            let restarted_sts_names: Vec<_> = host_invalid_replicas
297                .iter()
298                .map(|rs| rs.statefulset_name.clone())
299                .collect::<BTreeSet<_>>()
300                .into_iter()
301                .collect();
302
303            let namespace = kanidm.get_namespace();
304            let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
305                wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
306            });
307            try_join_all(rollout_futures).await?;
308
309            let renew_futures = host_invalid_replicas
310                .iter()
311                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
312            let results: Vec<_> = join_all(renew_futures).await;
313            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
314                return Err(first_err);
315            }
316
317            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
318
319            let cert_futures = host_invalid_replicas.iter().map(|rs| async {
320                let cert = show_replica_cert_with_retries(
321                    &kanidm,
322                    ctx.clone(),
323                    &rs.pod_name,
324                    CERT_SHOW_RETRY_ATTEMPTS,
325                )
326                .await?;
327                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
328                kanidm.patch(&ctx, secret.clone()).await
329            });
330            try_join_all(cert_futures).await?;
331
332            restart_statefulsets_after_replica_secret_update(
333                ctx.clone(),
334                &sts_api,
335                &kanidm.get_namespace(),
336                &statefulset_names,
337                has_single_replica,
338            )
339            .await?;
340
341            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
342        }
343
344        if !cert_expiring_replicas.is_empty() {
345            let renew_futures = cert_expiring_replicas
346                .iter()
347                .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
348            let results: Vec<_> = join_all(renew_futures).await;
349            if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
350                return Err(first_err);
351            }
352
353            sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
354
355            let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
356                let cert = show_replica_cert_with_retries(
357                    &kanidm,
358                    ctx.clone(),
359                    &rs.pod_name,
360                    CERT_SHOW_RETRY_ATTEMPTS,
361                )
362                .await?;
363                let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
364                kanidm.patch(&ctx, secret.clone()).await
365            });
366            try_join_all(cert_futures).await?;
367
368            restart_statefulsets_after_replica_secret_update(
369                ctx.clone(),
370                &sts_api,
371                &kanidm.get_namespace(),
372                &statefulset_names,
373                has_single_replica,
374            )
375            .await?;
376
377            ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
378        }
379
380        if has_non_ready_replicas
381            && !has_pending
382            && !has_certificate_host_invalid
383            && !has_certificate_expiring
384        {
385            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
386
387            let restart_futures = status
388                .replica_statuses
389                .iter()
390                .map(|rs| sts_api.restart(&rs.statefulset_name));
391
392            if has_single_replica {
393                let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
394                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
395                    return Err(Error::kube_error(
396                        "restart",
397                        "StatefulSet",
398                        kanidm.get_namespace(),
399                        "replica statefulsets",
400                        first_err,
401                    ));
402                }
403            } else {
404                let results: Vec<_> = join_all(restart_futures).await;
405                if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
406                    return Err(Error::kube_error(
407                        "restart",
408                        "StatefulSet",
409                        kanidm.get_namespace(),
410                        "replica statefulsets",
411                        first_err,
412                    ));
413                }
414            }
415        }
416    }
417
418    Ok(())
419}
420
421#[instrument(skip(ctx, kanidm))]
422pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
423    let trace_id = telemetry::get_trace_id();
424    Span::current().record("trace_id", field::display(&trace_id));
425    let _timer = ctx
426        .kaniop_ctx
427        .metrics
428        .reconcile_count_and_measure(&trace_id);
429    info!(msg = "reconciling Kanidm");
430
431    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
432        debug!(msg = "failed to reconcile status", %e);
433        ctx.kaniop_ctx.metrics.status_update_errors_inc();
434        e
435    })?;
436    let kanidm_api: Api<Kanidm> =
437        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
438    finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
439        match event {
440            Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
441            Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
442        }
443    })
444    .await
445    .or_else(|e| match e {
446        FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
447            debug!(msg = "resource already removed during finalizer cleanup");
448            Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
449        }
450        _ => Err(Error::FinalizerError(
451            "failed on kanidm account finalizer".to_string(),
452            Box::new(e),
453        )),
454    })
455}
456
457/// Hash of the TLS secret content, injected as a pod template annotation so
458/// certificate renewals roll the statefulsets. `None` if the secret does not
459/// exist yet.
460async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
461    let secret_name = kanidm.effective_tls_secret_name();
462    let secret_api =
463        Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
464    let secret = secret_api
465        .get_opt(&secret_name)
466        .await
467        .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
468    Ok(secret.as_ref().map(hash_secret_data))
469}
470
471/// Previous TLS secret hash annotation from the existing statefulsets.
472///
473/// Used when the TLS secret is missing: dropping the annotation would roll
474/// the pods onto a missing volume and leave them stuck in ContainerCreating.
475fn previous_tls_secret_hash(
476    kanidm: &Kanidm,
477    stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
478) -> Option<String> {
479    stateful_sets
480        .into_iter()
481        .filter(|sts| {
482            sts.namespace() == kanidm.namespace()
483                && sts
484                    .metadata
485                    .labels
486                    .as_ref()
487                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
488        })
489        .find_map(|sts| {
490            sts.spec
491                .as_ref()?
492                .template
493                .metadata
494                .as_ref()?
495                .annotations
496                .as_ref()?
497                .get(TLS_SECRET_HASH_ANNOTATION)
498                .cloned()
499        })
500}
501
502async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
503    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
504    let replication_secret_futures =
505        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
506
507    let sts_to_delete = {
508        let expected_sts_names = kanidm
509            .spec
510            .replica_groups
511            .iter()
512            .map(|rg| kanidm.statefulset_name(&rg.name))
513            .collect::<Vec<_>>();
514        ctx.stores
515            .stateful_set_store
516            .state()
517            .into_iter()
518            .filter(|sts| {
519                sts.namespace() == kanidm.namespace()
520                    && sts
521                        .metadata
522                        .labels
523                        .as_ref()
524                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
525                    && !expected_sts_names.contains(&sts.name_any())
526            })
527            .collect::<Vec<_>>()
528    };
529    let sts_delete_futures = sts_to_delete
530        .iter()
531        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
532        .collect::<TryJoinAll<_>>();
533
534    let sts_futures = match kanidm.is_updatable(&status) {
535        true => {
536            let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
537                previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
538            });
539            kanidm
540                .spec
541                .replica_groups
542                .iter()
543                .map(|rg| {
544                    let sts = kanidm.create_statefulset(rg, tls_secret_hash.as_deref())?;
545                    Ok(kanidm.patch(&ctx, sts))
546                })
547                .collect::<Result<TryJoinAll<_>, _>>()?
548        }
549        false => {
550            let note = match status.version.as_ref() {
551                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
552                    format!(
553                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
554                        v.image_tag,
555                        crate::version::KANIDM_CLIENT_VERSION
556                    )
557                }
558                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
559            };
560            let _ignore_error = ctx
561                .kaniop_ctx
562                .recorder
563                .publish(
564                    &Event {
565                        type_: EventType::Warning,
566                        reason: "UpgradeBlocked".to_string(),
567                        note: Some(note),
568                        action: "ReconcileStatefulSet".to_string(),
569                        secondary: None,
570                    },
571                    &kanidm.object_ref(&()),
572                )
573                .await
574                .map_err(|e| {
575                    warn!(msg = "failed to publish KanidmError event", %e);
576                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
577                });
578            try_join_all([])
579        }
580    };
581    let service_future = kanidm.patch(&ctx, kanidm.create_service());
582
583    let deprecated_rg_svcs = {
584        let expected_rg_svcs_names = kanidm
585            .spec
586            .replica_groups
587            .iter()
588            .filter(|rg| rg.services.is_some())
589            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
590            .collect::<Vec<_>>();
591        ctx.stores
592            .service_store
593            .state()
594            .into_iter()
595            .filter(|svc| {
596                let labels = svc.metadata.labels.as_ref();
597                svc.namespace() == kanidm.namespace()
598                    && !expected_rg_svcs_names.contains(&svc.name_any())
599                    && labels.is_some_and(|l| {
600                        l.contains_key(REPLICA_LABEL)
601                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
602                    })
603            })
604            .collect::<Vec<_>>()
605    };
606    let rg_svcs_delete_futures = deprecated_rg_svcs
607        .iter()
608        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
609        .collect::<TryJoinAll<_>>();
610
611    let rg_services_futures = kanidm
612        .spec
613        .replica_groups
614        .iter()
615        .filter(|rg| rg.services.is_some())
616        .flat_map(|rg| {
617            (0..rg.replicas).map(|i| kanidm.patch(&ctx, kanidm.create_replica_group_service(rg, i)))
618        })
619        .collect::<TryJoinAll<_>>();
620
621    let deprecated_ingresses = {
622        let names = [
623            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
624            kanidm.generate_region_ingress_name(),
625        ]
626        .into_iter()
627        .flatten()
628        .collect::<Vec<_>>();
629        ctx.stores
630            .ingress_store
631            .state()
632            .into_iter()
633            .filter(|ing| {
634                ing.namespace() == kanidm.namespace()
635                    && !names.contains(&ing.name_any())
636                    && ing.metadata.labels == Some(kanidm.generate_labels())
637            })
638            .collect::<Vec<_>>()
639    };
640    let ingress_delete_futures = deprecated_ingresses
641        .iter()
642        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
643        .collect::<TryJoinAll<_>>();
644
645    let ingress_futures = kanidm
646        .create_ingress()
647        .into_iter()
648        .chain(kanidm.create_region_ingress())
649        .map(|ingress| kanidm.patch(&ctx, ingress))
650        .collect::<TryJoinAll<_>>();
651
652    let deprecated_http_routes = match &ctx.stores.http_route_store {
653        Some(store) => {
654            let expected_names = kanidm
655                .spec
656                .gateway
657                .as_ref()
658                .map(|_| vec![kanidm.name_any()])
659                .unwrap_or_default();
660            store
661                .state()
662                .into_iter()
663                .filter(|route| {
664                    route.namespace() == kanidm.namespace()
665                        && !expected_names.contains(&route.name_any())
666                        && route.metadata.labels == Some(kanidm.generate_labels())
667                })
668                .collect::<Vec<_>>()
669        }
670        None => Vec::new(),
671    };
672    let http_route_delete_futures = deprecated_http_routes
673        .iter()
674        .map(|route| kanidm.delete(&ctx, route.as_ref()))
675        .collect::<TryJoinAll<_>>();
676
677    let http_route_futures = if ctx.stores.http_route_store.is_some() {
678        kanidm
679            .create_http_route()
680            .into_iter()
681            .map(|route| kanidm.patch(&ctx, route))
682            .collect::<TryJoinAll<_>>()
683    } else {
684        try_join_all(Vec::new())
685    };
686
687    let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
688        Some(store) => {
689            let expected_names = kanidm
690                .spec
691                .gateway
692                .as_ref()
693                .and_then(|g| g.backend_tls_policy.as_ref())
694                .map(|_| vec![kanidm.name_any()])
695                .unwrap_or_default();
696            store
697                .state()
698                .into_iter()
699                .filter(|policy| {
700                    policy.namespace() == kanidm.namespace()
701                        && !expected_names.contains(&policy.name_any())
702                        && policy.metadata.labels == Some(kanidm.generate_labels())
703                })
704                .collect::<Vec<_>>()
705        }
706        None => Vec::new(),
707    };
708    let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
709        .iter()
710        .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
711        .collect::<TryJoinAll<_>>();
712
713    let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
714        kanidm
715            .create_backend_tls_policy()
716            .into_iter()
717            .map(|policy| kanidm.patch(&ctx, policy))
718            .collect::<TryJoinAll<_>>()
719    } else {
720        try_join_all(Vec::new())
721    };
722
723    try_join!(
724        sts_delete_futures,
725        admin_secret_future,
726        replication_secret_futures,
727        sts_futures,
728        service_future,
729        rg_svcs_delete_futures,
730        rg_services_futures,
731        ingress_delete_futures,
732        ingress_futures,
733        http_route_delete_futures,
734        http_route_futures,
735        backend_tls_policy_delete_futures,
736        backend_tls_policy_futures
737    )?;
738
739    if is_kanidm_available(status.clone()) {
740        let namespace = kanidm.namespace().unwrap();
741        let name = kanidm.name_any();
742        let system_client = crate::controller::kanidm::KanidmClients::create_client(
743            &namespace,
744            &name,
745            crate::controller::kanidm::KanidmUser::Admin,
746            ctx.kaniop_ctx.client.clone(),
747        )
748        .await?;
749        reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await?;
750
751        let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
752            &namespace,
753            &name,
754            crate::controller::kanidm::KanidmUser::IdmAdmin,
755            ctx.kaniop_ctx.client.clone(),
756        )
757        .await?;
758        let mail_sender_status =
759            reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone()).await?;
760        let current_mail_sender_status = kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
761        if mail_sender_status != current_mail_sender_status {
762            let kanidm_api: Api<Kanidm> =
763                Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
764            let status_patch = serde_json::json!({
765                "status": {
766                    "mailSender": mail_sender_status
767                }
768            });
769            kanidm_api
770                .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
771                .await
772                .map_err(|e| {
773                    Error::KubeError(
774                        format!("failed to patch Kanidm/status {namespace}/{name}"),
775                        Box::new(e),
776                    )
777                })?;
778        }
779    }
780
781    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
782}
783
784async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
785    debug!(msg = "cleanup");
786
787    cleanup_mail_sender_resources(&kanidm, &ctx).await?;
788
789    let namespace = kanidm.namespace().unwrap();
790    let name = kanidm.name_any();
791
792    if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
793        &namespace,
794        &name,
795        crate::controller::kanidm::KanidmUser::IdmAdmin,
796        ctx.kaniop_ctx.client.clone(),
797    )
798    .await
799    {
800        cleanup_mail_sender_in_kanidm(&kanidm_client, &name).await?;
801    }
802
803    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
804    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
805}
806
807async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
808    let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
809    let start = std::time::Instant::now();
810    let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
811    let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
812
813    loop {
814        let sts = sts_api.get(sts_name).await.map_err(|e| {
815            Error::KubeError(
816                format!("failed to get StatefulSet {namespace}/{sts_name}"),
817                Box::new(e),
818            )
819        })?;
820
821        if sts.metadata.deletion_timestamp.is_some() {
822            return Err(Error::ReceiveOutput(format!(
823                "StatefulSet {namespace}/{sts_name} is being deleted"
824            )));
825        }
826
827        let spec = sts.spec.as_ref().ok_or_else(|| {
828            Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
829        })?;
830
831        let status = match sts.status {
832            Some(ref s) => s,
833            None => {
834                if start.elapsed() >= timeout {
835                    return Err(Error::ReceiveOutput(format!(
836                        "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
837                    )));
838                }
839                sleep(poll_interval).await;
840                continue;
841            }
842        };
843
844        let replicas = spec.replicas.unwrap_or(0);
845
846        if replicas == 0 {
847            return Ok(());
848        }
849
850        let updated_replicas = status.updated_replicas.unwrap_or(0);
851        let current_revision = status.current_revision.as_deref().unwrap_or("");
852        let update_revision = status.update_revision.as_deref().unwrap_or("");
853        let ready_replicas = status.ready_replicas.unwrap_or(0);
854
855        if updated_replicas == replicas
856            && current_revision == update_revision
857            && ready_replicas == replicas
858        {
859            return Ok(());
860        }
861
862        if start.elapsed() >= timeout {
863            return Err(Error::ReceiveOutput(format!(
864                "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
865            )));
866        }
867
868        sleep(poll_interval).await;
869    }
870}
871
872async fn show_replica_cert_with_retries(
873    kanidm: &Kanidm,
874    ctx: Arc<Context>,
875    pod_name: &str,
876    max_attempts: u32,
877) -> std::result::Result<String, Error> {
878    let mut last_error = None;
879    for attempt in 0..max_attempts {
880        if attempt > 0 {
881            sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
882        }
883        match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
884            Ok(cert) => return Ok(cert),
885            Err(e) => {
886                info!(
887                    msg = format!(
888                        "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
889                        attempt + 1,
890                        max_attempts
891                    )
892                );
893                last_error = Some(e);
894            }
895        }
896    }
897    Err(last_error.unwrap_or_else(|| {
898        Error::ReceiveOutput(format!(
899            "failed to get certificate for {pod_name} after {max_attempts} attempts"
900        ))
901    }))
902}
903
904impl Kanidm {
905    // Convenience methods that handle context and operator name
906    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
907    where
908        K: Resource<Scope = NamespaceResourceScope>
909            + Serialize
910            + Clone
911            + std::fmt::Debug
912            + for<'de> Deserialize<'de>,
913        <K as kube::Resource>::DynamicType: Default,
914        <K as Resource>::Scope: std::marker::Sized,
915    {
916        self.kube_delete(
917            ctx.kaniop_ctx.client.clone(),
918            &ctx.kaniop_ctx.metrics,
919            resource,
920        )
921        .await
922    }
923
924    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
925    where
926        K: Resource<Scope = NamespaceResourceScope>
927            + Serialize
928            + Clone
929            + std::fmt::Debug
930            + for<'de> Deserialize<'de>,
931        <K as kube::Resource>::DynamicType: Default,
932        <K as Resource>::Scope: std::marker::Sized,
933    {
934        self.kube_patch(
935            ctx.kaniop_ctx.client.clone(),
936            &ctx.kaniop_ctx.metrics,
937            resource,
938            KANIDM_OPERATOR_NAME,
939        )
940        .await
941    }
942
943    #[inline]
944    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
945        LABELS
946            .clone()
947            .into_iter()
948            .chain([
949                (INSTANCE_LABEL.to_string(), self.name_any()),
950                (CLUSTER_LABEL.to_string(), self.name_any()),
951            ])
952            .collect()
953    }
954
955    #[inline]
956    fn generate_labels(&self) -> BTreeMap<String, String> {
957        self.generate_resource_labels()
958    }
959
960    #[inline]
961    fn get_tls_secret_name(&self) -> String {
962        format!("{}-tls", self.name_any())
963    }
964
965    /// TLS secret name mounted by the statefulsets: explicit `tlsSecretName`,
966    /// falling back to the ingress TLS secret and then to the default name.
967    pub(crate) fn effective_tls_secret_name(&self) -> String {
968        self.spec.tls_secret_name.clone().unwrap_or_else(|| {
969            self.spec
970                .ingress
971                .as_ref()
972                .and_then(|i| i.tls_secret_name.clone())
973                .unwrap_or_else(|| self.get_tls_secret_name())
974        })
975    }
976
977    #[inline]
978    fn get_namespace(&self) -> String {
979        // safe unwrap: Kanidm is namespaced scoped
980        self.namespace().unwrap()
981    }
982
983    #[inline]
984    fn is_replication_enabled(&self) -> bool {
985        self.spec.replica_groups.len() > 1
986            || self
987                .spec
988                .replica_groups
989                .first()
990                .is_some_and(|rg| rg.replicas > 1)
991            || !self.spec.external_replication_nodes.is_empty()
992    }
993
994    #[inline]
995    fn is_updatable(&self, status: &KanidmStatus) -> bool {
996        status
997            .version
998            .as_ref()
999            .map(|v| {
1000                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1001                    return false;
1002                }
1003                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1004                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
1005                match v.upgrade_check_result {
1006                    KanidmUpgradeCheckResult::Passed => {
1007                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1008                            minor <= v_minor + 1
1009                        } else {
1010                            true
1011                        }
1012                    }
1013                    KanidmUpgradeCheckResult::Failed => {
1014                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1015                            versions
1016                        {
1017                            major == v_major && minor == v_minor && patch >= v_patch
1018                        } else {
1019                            v.image_tag == current_tag
1020                        }
1021                    }
1022                }
1023            })
1024            .unwrap_or(true)
1025    }
1026
1027    async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1028        let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1029        let pod = pod_api.get(pod_name).await.map_err(|e| {
1030            Error::KubeError(
1031                format!("failed to get pod {namespace}/{pod_name}"),
1032                Box::new(e),
1033            )
1034        })?;
1035        Ok(pod
1036            .status
1037            .as_ref()
1038            .and_then(|s| s.conditions.as_ref())
1039            .is_some_and(|conditions| {
1040                conditions
1041                    .iter()
1042                    .any(|c| c.type_ == "Ready" && c.status == "True")
1043            }))
1044    }
1045
1046    async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1047        let start = std::time::Instant::now();
1048        let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1049        let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1050
1051        loop {
1052            match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1053                Ok(true) => return Ok(()),
1054                Ok(false) => {}
1055                Err(e) => {
1056                    if !e.is_retryable() {
1057                        return Err(e);
1058                    }
1059                    trace!(msg = "pod readiness check failed, retrying", %e);
1060                }
1061            }
1062
1063            if start.elapsed() >= timeout {
1064                return Err(Error::ReceiveOutput(format!(
1065                    "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1066                )));
1067            }
1068
1069            sleep(poll_interval).await;
1070        }
1071    }
1072
1073    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1074    where
1075        I: IntoIterator<Item = T> + Debug,
1076        T: Into<String>,
1077    {
1078        let namespace = &self.get_namespace();
1079        trace!(
1080            msg = "pod exec",
1081            resource.name = &pod_name,
1082            resource.namespace = &namespace,
1083            ?command
1084        );
1085        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1086        let attached = pod
1087            .exec(
1088                pod_name,
1089                command,
1090                &AttachParams::default().container("kanidm"),
1091            )
1092            .await
1093            .map_err(|e| {
1094                Error::KubeError(
1095                    format!("failed to exec pod {namespace}/{pod_name}"),
1096                    Box::new(e),
1097                )
1098            })?;
1099        get_output(attached).await
1100    }
1101
1102    async fn exec_with_wait<I, T>(
1103        &self,
1104        ctx: Arc<Context>,
1105        pod_name: &str,
1106        command: I,
1107    ) -> Result<String>
1108    where
1109        I: IntoIterator<Item = T> + Debug,
1110        T: Into<String>,
1111    {
1112        let namespace = &self.get_namespace();
1113        Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1114        self.exec(ctx, pod_name, command).await
1115    }
1116
1117    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1118    where
1119        I: IntoIterator<Item = T> + Debug,
1120        T: Into<String>,
1121    {
1122        // TODO: if replicas > 1 and replicas initialized, exec on pod available
1123        let first_replica_group = self
1124            .spec
1125            .replica_groups
1126            .first()
1127            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1128        let sts_name = self.statefulset_name(&first_replica_group.name);
1129        let pod_name = format!("{sts_name}-0");
1130        self.exec(ctx, &pod_name, command).await
1131    }
1132
1133    async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1134    where
1135        I: IntoIterator<Item = T> + Debug,
1136        T: Into<String>,
1137    {
1138        let first_replica_group = self
1139            .spec
1140            .replica_groups
1141            .first()
1142            .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1143        let sts_name = self.statefulset_name(&first_replica_group.name);
1144        let pod_name = format!("{sts_name}-0");
1145        self.exec_with_wait(ctx, &pod_name, command).await
1146    }
1147}
1148
1149#[cfg(test)]
1150mod test {
1151    use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1152    use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1153
1154    use crate::controller::State;
1155    use crate::kanidm::controller::context::{Context, Stores};
1156    use crate::kanidm::crd::KanidmStatus;
1157
1158    use kaniop_k8s_util::error::Result;
1159    use kaniop_k8s_util::resources::hash_secret_data;
1160
1161    use std::collections::BTreeMap;
1162    use std::sync::Arc;
1163
1164    use http::{Request, Response};
1165    use k8s_openapi::ByteString;
1166    use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1167    use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1168    use k8s_openapi::api::networking::v1::Ingress;
1169    use kube::api::ObjectMeta;
1170    use kube::runtime::reflector::store::Writer;
1171    use kube::{Client, Resource, ResourceExt, client::Body};
1172    use opentelemetry::metrics::MeterProvider;
1173    use serde_json::json;
1174
1175    impl Kanidm {
1176        /// A normal test kanidm with a given status
1177        pub fn test() -> Self {
1178            let mut e = Kanidm::new(
1179                "test",
1180                serde_json::from_value(json!({
1181                    "domain": "idm.example.com",
1182                    "replicaGroups": [
1183                        {
1184                            "name": "default",
1185                            "replicas": 1
1186                        }
1187                    ]
1188                }))
1189                .unwrap(),
1190            );
1191            e.meta_mut().namespace = Some("default".into());
1192            // Pre-populate finalizer so reconcile callback runs immediately
1193            // (skips the "add finalizer + wait for next reconcile" step)
1194            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1195            e
1196        }
1197
1198        pub fn with_ingress(mut self) -> Self {
1199            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1200            self
1201        }
1202
1203        /// Modify kanidm replicas
1204        pub fn with_replicas(mut self, replicas: i32) -> Self {
1205            self.spec.replica_groups[0].replicas = replicas;
1206            self
1207        }
1208
1209        /// Modify kanidm to set a deletion timestamp
1210        pub fn needs_delete(mut self) -> Self {
1211            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1212            use k8s_openapi::jiff::Timestamp;
1213            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
1214            self.meta_mut().deletion_timestamp = Some(Time(now));
1215            self
1216        }
1217
1218        /// Modify a kanidm to have an expected status
1219        pub fn with_status(mut self, status: KanidmStatus) -> Self {
1220            self.status = Some(status);
1221            self
1222        }
1223    }
1224
1225    // We wrap tower_test::mock::Handle
1226    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1227    pub struct ApiServerVerifier(ApiServerHandle);
1228
1229    /// Scenarios we test for in ApiServerVerifier
1230    pub enum Scenario {
1231        Create(Kanidm),
1232        CreateWithTwoReplicas(Kanidm),
1233        CreateWithIngress(Kanidm),
1234        CreateWithIngressWithTwoReplicas(Kanidm),
1235        CreateWithTlsSecret(Kanidm, Box<Secret>),
1236    }
1237
1238    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1239        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1240            .await
1241            .expect("timeout on mock apiserver")
1242            .expect("scenario succeeded")
1243    }
1244
1245    impl ApiServerVerifier {
1246        /// Tests only get to run specific scenarios that has matching handlers
1247        ///
1248        /// This setup makes it easy to handle multiple requests by chaining handlers together.
1249        ///
1250        /// NB: If the controller is making more calls than we are handling in the scenario,
1251        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
1252        ///
1253        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
1254        /// scenario runs to completion (i.e. all expected calls were responded to),
1255        /// using the timeout to catch missing api calls to Kubernetes.
1256        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1257            tokio::spawn(async move {
1258                // moving self => one scenario per test
1259                match scenario {
1260                    Scenario::Create(kanidm) => {
1261                        self.handle_kanidm_get(kanidm.clone())
1262                            .await
1263                            .unwrap()
1264                            .handle_kanidm_status_patch(kanidm.clone())
1265                            .await
1266                            .unwrap()
1267                            .handle_tls_secret_get_not_found(kanidm.clone())
1268                            .await
1269                            .unwrap()
1270                            .handle_statefulset_patch(kanidm.clone(), None)
1271                            .await
1272                            .unwrap()
1273                            .handle_service_patch(kanidm.clone())
1274                            .await
1275                    }
1276                    Scenario::CreateWithTwoReplicas(kanidm) => {
1277                        self.handle_kanidm_get(kanidm.clone())
1278                            .await
1279                            .unwrap()
1280                            .handle_kanidm_status_patch(kanidm.clone())
1281                            .await
1282                            .unwrap()
1283                            .handle_tls_secret_get_not_found(kanidm.clone())
1284                            .await
1285                            .unwrap()
1286                            .handle_statefulset_patch(kanidm.clone(), None)
1287                            .await
1288                            .unwrap()
1289                            .handle_service_patch(kanidm.clone())
1290                            .await
1291                    }
1292                    Scenario::CreateWithIngress(kanidm) => {
1293                        self.handle_kanidm_get(kanidm.clone())
1294                            .await
1295                            .unwrap()
1296                            .handle_kanidm_status_patch(kanidm.clone())
1297                            .await
1298                            .unwrap()
1299                            .handle_tls_secret_get_not_found(kanidm.clone())
1300                            .await
1301                            .unwrap()
1302                            .handle_statefulset_patch(kanidm.clone(), None)
1303                            .await
1304                            .unwrap()
1305                            .handle_service_patch(kanidm.clone())
1306                            .await
1307                            .unwrap()
1308                            .handle_ingress_patch(kanidm.clone())
1309                            .await
1310                    }
1311                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1312                        self.handle_kanidm_get(kanidm.clone())
1313                            .await
1314                            .unwrap()
1315                            .handle_kanidm_status_patch(kanidm.clone())
1316                            .await
1317                            .unwrap()
1318                            .handle_tls_secret_get_not_found(kanidm.clone())
1319                            .await
1320                            .unwrap()
1321                            .handle_statefulset_patch(kanidm.clone(), None)
1322                            .await
1323                            .unwrap()
1324                            .handle_service_patch(kanidm.clone())
1325                            .await
1326                            .unwrap()
1327                            .handle_ingress_patch(kanidm.clone())
1328                            .await
1329                    }
1330                    Scenario::CreateWithTlsSecret(kanidm, secret) => {
1331                        let expected_hash = hash_secret_data(&secret);
1332                        self.handle_kanidm_get(kanidm.clone())
1333                            .await
1334                            .unwrap()
1335                            .handle_kanidm_status_patch(kanidm.clone())
1336                            .await
1337                            .unwrap()
1338                            .handle_tls_secret_get(kanidm.clone(), *secret)
1339                            .await
1340                            .unwrap()
1341                            .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1342                            .await
1343                            .unwrap()
1344                            .handle_service_patch(kanidm.clone())
1345                            .await
1346                    }
1347                }
1348                .expect("scenario completed without errors");
1349            })
1350        }
1351
1352        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1353            let (request, send) = self.0.next_request().await.expect("service not called");
1354            assert_eq!(request.method(), http::Method::GET);
1355            let expected_uri_prefix = format!(
1356                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1357                kanidm.name_any()
1358            );
1359            let uri = request.uri().to_string();
1360            assert!(
1361                uri.starts_with(&expected_uri_prefix),
1362                "expected uri to start with {}, got {}",
1363                expected_uri_prefix,
1364                uri
1365            );
1366            let response = serde_json::to_vec(&kanidm).unwrap();
1367            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1368            Ok(self)
1369        }
1370
1371        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1372            let (request, send) = self.0.next_request().await.expect("service not called");
1373            assert_eq!(request.method(), http::Method::PATCH);
1374            assert_eq!(
1375                request.uri().to_string(),
1376                format!(
1377                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1378                    kanidm.name_any()
1379                )
1380            );
1381
1382            let req_body = request.into_body().collect_bytes().await.unwrap();
1383            let json: serde_json::Value =
1384                serde_json::from_slice(&req_body).expect("patch object is json");
1385            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1386                .expect("valid kanidm status");
1387            let mut kanidm = Kanidm::test();
1388            kanidm.status = Some(status.clone());
1389            let response = serde_json::to_vec(&kanidm).unwrap();
1390            // pass through kanidm "patch accepted"
1391            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1392            Ok(self)
1393        }
1394
1395        async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1396            let (request, send) = self.0.next_request().await.expect("service not called");
1397            assert_eq!(request.method(), http::Method::GET);
1398            let expected_uri_prefix = format!(
1399                "/api/v1/namespaces/default/secrets/{}",
1400                kanidm.effective_tls_secret_name()
1401            );
1402            let uri = request.uri().to_string();
1403            assert!(
1404                uri.starts_with(&expected_uri_prefix),
1405                "expected uri to start with {expected_uri_prefix}, got {uri}"
1406            );
1407            let status = serde_json::json!({
1408                "kind": "Status",
1409                "apiVersion": "v1",
1410                "metadata": {},
1411                "status": "Failure",
1412                "message": "secret not found",
1413                "reason": "NotFound",
1414                "code": 404
1415            });
1416            send.send_response(
1417                Response::builder()
1418                    .status(404)
1419                    .body(Body::from(serde_json::to_vec(&status).unwrap()))
1420                    .unwrap(),
1421            );
1422            Ok(self)
1423        }
1424
1425        async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1426            let (request, send) = self.0.next_request().await.expect("service not called");
1427            assert_eq!(request.method(), http::Method::GET);
1428            let expected_uri_prefix = format!(
1429                "/api/v1/namespaces/default/secrets/{}",
1430                kanidm.effective_tls_secret_name()
1431            );
1432            let uri = request.uri().to_string();
1433            assert!(
1434                uri.starts_with(&expected_uri_prefix),
1435                "expected uri to start with {expected_uri_prefix}, got {uri}"
1436            );
1437            let response = serde_json::to_vec(&secret).unwrap();
1438            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1439            Ok(self)
1440        }
1441
1442        async fn handle_statefulset_patch(
1443            mut self,
1444            kanidm: Kanidm,
1445            expected_tls_hash: Option<String>,
1446        ) -> Result<Self> {
1447            for rg in kanidm.spec.replica_groups.iter() {
1448                let (request, send) = self.0.next_request().await.expect("service not called");
1449                assert_eq!(request.method(), http::Method::PATCH);
1450                assert_eq!(
1451                    request.uri().to_string(),
1452                    format!(
1453                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1454                        kanidm.statefulset_name(&rg.name)
1455                    )
1456                );
1457                let req_body = request.into_body().collect_bytes().await.unwrap();
1458                let json: serde_json::Value =
1459                    serde_json::from_slice(&req_body).expect("patch object is json");
1460                let statefulset: StatefulSet =
1461                    serde_json::from_value(json).expect("valid statefulset");
1462                assert_eq!(
1463                    statefulset.clone().spec.unwrap().replicas.unwrap(),
1464                    rg.replicas
1465                );
1466                let tls_hash_annotation = statefulset
1467                    .spec
1468                    .as_ref()
1469                    .unwrap()
1470                    .template
1471                    .metadata
1472                    .as_ref()
1473                    .unwrap()
1474                    .annotations
1475                    .as_ref()
1476                    .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
1477                    .cloned();
1478                assert_eq!(tls_hash_annotation, expected_tls_hash);
1479                let response = serde_json::to_vec(&statefulset).unwrap();
1480                // pass through kanidm "patch accepted"
1481                send.send_response(Response::builder().body(Body::from(response)).unwrap());
1482            }
1483
1484            Ok(self)
1485        }
1486
1487        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1488            let (request, send) = self.0.next_request().await.expect("service not called");
1489            assert_eq!(request.method(), http::Method::PATCH);
1490            assert_eq!(
1491                request.uri().to_string(),
1492                format!(
1493                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1494                    kanidm.name_any()
1495                )
1496            );
1497
1498            let req_body = request.into_body().collect_bytes().await.unwrap();
1499            let json: serde_json::Value =
1500                serde_json::from_slice(&req_body).expect("patch object is json");
1501            let service: Service = serde_json::from_value(json).expect("valid service");
1502            let response = serde_json::to_vec(&service).unwrap();
1503            // pass through kanidm "patch accepted"
1504            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1505            Ok(self)
1506        }
1507
1508        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1509            let (request, send) = self.0.next_request().await.expect("service not called");
1510            assert_eq!(request.method(), http::Method::PATCH);
1511            assert_eq!(
1512                request.uri().to_string(),
1513                format!(
1514                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
1515                    kanidm.name_any()
1516                )
1517            );
1518
1519            let req_body = request.into_body().collect_bytes().await.unwrap();
1520            let json: serde_json::Value =
1521                serde_json::from_slice(&req_body).expect("patch object is json");
1522            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
1523            let response = serde_json::to_vec(&ingress).unwrap();
1524            // pass through kanidm "patch accepted"
1525            send.send_response(Response::builder().body(Body::from(response)).unwrap());
1526            Ok(self)
1527        }
1528    }
1529
1530    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
1531        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
1532        let mock_client = Client::new(mock_service, "default");
1533        let stores = Stores {
1534            stateful_set_store: Writer::default().as_reader(),
1535            service_store: Writer::default().as_reader(),
1536            ingress_store: Writer::default().as_reader(),
1537            secret_store: Writer::default().as_reader(),
1538            http_route_store: Some(Writer::default().as_reader()),
1539            backend_tls_policy_store: Some(Writer::default().as_reader()),
1540            deployment_store: Writer::default().as_reader(),
1541            config_map_store: Writer::default().as_reader(),
1542        };
1543        let controller_id = "test";
1544
1545        // Create a test meter for metrics
1546        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
1547        let meter = provider.meter("test");
1548        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
1549
1550        let state = State::new(
1551            metrics,
1552            Writer::default().as_reader(),
1553            Writer::default().as_reader(),
1554            None,
1555        );
1556        let ctx = Arc::new(Context::new(
1557            state.to_context(mock_client, controller_id),
1558            stores,
1559        ));
1560        (ctx, ApiServerVerifier(handle))
1561    }
1562
1563    #[tokio::test]
1564    async fn kanidm_create() {
1565        let (testctx, fakeserver) = get_test_context();
1566        let kanidm = Kanidm::test();
1567        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
1568        reconcile_kanidm(Arc::new(kanidm), testctx)
1569            .await
1570            .expect("reconciler");
1571        timeout_after_1s(mocksrv).await;
1572    }
1573
1574    #[tokio::test]
1575    async fn kanidm_create_with_two_replicas() {
1576        let (testctx, fakeserver) = get_test_context();
1577        let kanidm = Kanidm::test().with_replicas(2);
1578        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
1579        reconcile_kanidm(Arc::new(kanidm), testctx)
1580            .await
1581            .expect("reconciler");
1582        timeout_after_1s(mocksrv).await;
1583    }
1584
1585    #[tokio::test]
1586    async fn kanidm_create_with_ingress() {
1587        let (testctx, fakeserver) = get_test_context();
1588        let kanidm = Kanidm::test().with_ingress();
1589        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
1590        reconcile_kanidm(Arc::new(kanidm), testctx)
1591            .await
1592            .expect("reconciler");
1593        timeout_after_1s(mocksrv).await;
1594    }
1595
1596    #[tokio::test]
1597    async fn kanidm_create_with_ingress_with_two_replicas() {
1598        let (testctx, fakeserver) = get_test_context();
1599        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
1600        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
1601        reconcile_kanidm(Arc::new(kanidm), testctx)
1602            .await
1603            .expect("reconciler");
1604        timeout_after_1s(mocksrv).await;
1605    }
1606
1607    fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
1608        Arc::new(StatefulSet {
1609            metadata: ObjectMeta {
1610                name: Some(format!("{cluster}-default")),
1611                namespace: Some(namespace.to_string()),
1612                labels: Some(BTreeMap::from([(
1613                    CLUSTER_LABEL.to_string(),
1614                    cluster.to_string(),
1615                )])),
1616                ..Default::default()
1617            },
1618            spec: Some(StatefulSetSpec {
1619                template: PodTemplateSpec {
1620                    metadata: Some(ObjectMeta {
1621                        annotations: tls_hash.map(|h| {
1622                            BTreeMap::from([(
1623                                TLS_SECRET_HASH_ANNOTATION.to_string(),
1624                                h.to_string(),
1625                            )])
1626                        }),
1627                        ..Default::default()
1628                    }),
1629                    ..Default::default()
1630                },
1631                ..Default::default()
1632            }),
1633            ..Default::default()
1634        })
1635    }
1636
1637    #[test]
1638    fn test_previous_tls_secret_hash() {
1639        let kanidm = Kanidm::test();
1640
1641        // annotation carried over from an existing statefulset
1642        let stss = vec![statefulset_of("test", "default", Some("abc123"))];
1643        assert_eq!(
1644            previous_tls_secret_hash(&kanidm, stss),
1645            Some("abc123".to_string())
1646        );
1647
1648        // no annotation on the existing statefulset
1649        let stss = vec![statefulset_of("test", "default", None)];
1650        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
1651
1652        // statefulsets from another cluster or namespace are ignored
1653        let stss = vec![
1654            statefulset_of("other", "default", Some("abc123")),
1655            statefulset_of("test", "other-ns", Some("abc123")),
1656        ];
1657        assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
1658    }
1659
1660    #[tokio::test]
1661    async fn kanidm_create_with_tls_secret() {
1662        let (testctx, fakeserver) = get_test_context();
1663        let kanidm = Kanidm::test();
1664        let secret = Secret {
1665            metadata: ObjectMeta {
1666                name: Some(kanidm.effective_tls_secret_name()),
1667                namespace: Some("default".to_string()),
1668                ..Default::default()
1669            },
1670            data: Some(BTreeMap::from([
1671                ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
1672                ("tls.key".to_string(), ByteString(b"key".to_vec())),
1673            ])),
1674            ..Default::default()
1675        };
1676        let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
1677            kanidm.clone(),
1678            Box::new(secret),
1679        ));
1680        reconcile_kanidm(Arc::new(kanidm), testctx)
1681            .await
1682            .expect("reconciler");
1683        timeout_after_1s(mocksrv).await;
1684    }
1685}